<tool id="ideas" name="IDEAS" version="1.2.0">
    <description>accounts for position dependent epigenetic events and detects local cell type relationships</description>
    <requirements>
        <requirement type="package" version="2.26.0">bedtools</requirement>
        <requirement type="package" version="332">ucsc-bedgraphtobigwig</requirement>
        <requirement type="package" version="332">ucsc-bedsort</requirement>
        <requirement type="package" version="332">ucsc-bigwigaverageoverbed</requirement>
        <requirement type="package" version="1.20">ideas</requirement>
        <requirement type="package" version="1.10.4">r-data.table</requirement>
        <requirement type="package" version="1.4.4">r-optparse</requirement>
    </requirements>
    <command detect_errors="exit_code"><![CDATA[
#set output_pdf_dir = "output_pdf_dir"
#set output_txt_dir = "output_txt_dir"
#set output_training_dir = "output_training_dir"
#set tmp_dir = "tmp"
#set prep_input_config = "prep_input_config.txt"
#set prep_output_config = "prep_output_config.txt"
#set specify_genomic_window = $specify_genomic_window_cond.specify_genomic_window
#set perform_training = $perform_training_cond.perform_training
##############################################
## Create the config file and prepare the data
##############################################
#if str($output_heatmaps) == "yes":
    mkdir '$output_pdf_dir' &&
#end if
#if str($perform_training) == "yes":
    mkdir '$output_training_dir' &&
#end if
mkdir '$output_txt_dir' &&
cp '$gen_prep_input_config' $prep_input_config &&
sort $prep_input_config -o $prep_input_config &&
prepMat
$prep_input_config
#if str($specify_genomic_window) == "yes":
    -bed '$specify_genomic_window_cond.bed_input'
#else:
    -gsz '$chromInfo'
    -wsz $specify_genomic_window_cond.window_size
    #set restrict_chromosomes = $specify_genomic_window_cond.restrict_chromosomes_cond.restrict_chromosomes
    #if str($restrict_chromosomes) == "yes":
        #set chroms = []
        #set chrom_repeat = $specify_genomic_window_cond.restrict_chromosomes_cond.chrom_repeat
        #for $i in $chrom_repeat.chrom
            $chroms.append($i)
        #end for
        -chr ",".join(chroms)
    #end if
#end if
$bychr
-c $reads_per_bp
#if str($blacklist_input) not in ["None", ""]:
    -exclude '$blacklist_input'
#end if
$norm
&>prepmat_log.txt;
if [[ $? -ne 0 ]]; then
    cp prepmat_log.txt '$output_txt_dir';
    exit 1;
fi
##############################################
## Coerce the prepMat config output to the
## format expected by IDEAS.
##############################################
&& cut -d' ' $prep_input_config -f1,2 > file1.txt
&& ls tmp/*.bed.gz > file2.txt
&& paste <(cat file1.txt) <(cat file2.txt) -d' ' > $prep_output_config
#if str($specify_genomic_window) == "yes":
    ##############################################
    ## Using a genomic window bed file, so categorize
    ## the window positions by chromosome to enable
    ## the IDEAS -inv option.
    ##############################################
    #import collections
    #set window_positions_by_chroms = collections.OrderedDict()
    #for count, line in enumerate(open($specify_genomic_window_cond.bed_input.file_name, 'r')):
        #set $line = $line.strip()
        #if not $line or $line.startswith('#'):
            #continue
        #end if
        #set items = $line.split('\t')
        #if $items[0] in $window_positions_by_chroms:
            #set tup = $window_positions_by_chroms[$items[0]]
            #set $tup[1] += 1
            #set $window_positions_by_chroms[$items[0]] = $tup
        #else:
            #set $window_positions_by_chroms[$items[0]] = [$count, $count+1]
        #end if
    #end for
    #for chrom, tup in $window_positions_by_chroms.items():
        && Rscript '$__tool_directory__/ideas.R'
        #if str($perform_training) == "yes":
            --training_iterations $perform_training_cond.training_iterations
            --training_windows $perform_training_cond.training_windows
        #end if
        --prep_output_config '$prep_output_config'
        --windows_bed '$specify_genomic_window_cond.bed_input'
        --hp $hp
        --window_start $tup[0]
        --window_end $tup[1]
        #if str($log2) != "0.0":
            --log2 $log2
        #end if
        #if str($max_states) != "0.0":
            --max_states $max_states
        #end if
        #if str($initial_states) != "0":
            --initial_states $initial_states
        #end if
        #if str($max_position_classes) != "0":
            --max_position_classes $max_position_classes
        #end if
        #if str($max_cell_type_clusters) != "0":
            --max_cell_type_clusters $max_cell_type_clusters
        #end if
        #if str($prior_concentration) != "0.0":
            --prior_concentration $prior_concentration
        #end if
        --burnin_num $burnin_num
        --mcmc_num $mcmc_num
        #if str($minerr) != "0.0":
            --minerr $minerr
        #end if
        #if str($maxerr) != "0.0":
            --maxerr $maxerr
        #end if
        --rseed $rseed
        --thread \${GALAXY_SLOTS:-4}
        --project_name '$project_name.$chrom'
        #if str($save_ideas_log) == "yes":
            --save_ideas_log $save_ideas_log
            --output_log '$output_log'
        #end if
    #end for
#else:
    ##############################################
    ## Not using a genomic window bed file.
    ##############################################
    && Rscript '$__tool_directory__/ideas.R'
    #if str($perform_training) == "yes":
        --training_iterations $perform_training_cond.training_iterations
        --training_windows $perform_training_cond.training_windows
    #end if
    --prep_output_config '$prep_output_config'
    --windows_bed $tmp_dir/*.bed
    --hp $hp
    #if str($log2) != "0.0":
        --log2 $log2
    #end if
    #if str($max_states) != "0.0":
        --max_states $max_states
    #end if
    #if str($initial_states) != "0":
        --initial_states $initial_states
    #end if
    #if str($max_position_classes) != "0":
        --max_position_classes $max_position_classes
    #end if
    #if str($max_cell_type_clusters) != "0":
        --max_cell_type_clusters $max_cell_type_clusters
    #end if
    #if str($prior_concentration) != "0.0":
        --prior_concentration $prior_concentration
    #end if
    --burnin_num $burnin_num
    --mcmc_num $mcmc_num
    #if str($minerr) != "0.0":
        --minerr $minerr
    #end if
    #if str($maxerr) != "0.0":
        --maxerr $maxerr
    #end if
    --rseed $rseed
    --thread \${GALAXY_SLOTS:-4}
    --project_name '$project_name'
    #if str($save_ideas_log) == "yes":
        --save_ideas_log $save_ideas_log
        --output_log '$output_log'
    #end if
#end if
#if str($perform_training) == "yes":
    #set heatmaps_input_dir = $output_training_dir
    && mv ./*.para0 '$output_training_dir'
    && for file in $output_training_dir/*.para0 do mv "$file" "${file/.para0/para}" done
    && mv ./*.profile0 '$output_training_dir'
    && for file in $output_training_dir/*.profile0 do mv "$file" "${file/.profile0/profile}" done
#else:
    #set heatmaps_input_dir = $output_txt_dir
    && mv ./*.cluster '$output_txt_dir'
    && mv ./*.para '$output_txt_dir'
    && mv ./*.profile '$output_txt_dir'
    && mv ./*.state '$output_txt_dir'
#end if
#if str($output_heatmaps) == "yes":
    && Rscript '$__tool_directory__/create_heatmaps.R'
    --input_dir '$heatmaps_input_dir'
    --output_dir '$output_pdf_dir'
    --script_dir '$__tool_directory__'
#end if
    ]]></command>
    <configfiles>
        <configfile name="gen_prep_input_config"><![CDATA[#if str($cell_type_epigenetic_factor_cond.cell_type_epigenetic_factor) == "extract":
    #set input_name_positions = $cell_type_epigenetic_factor_cond.input_name_positions
    #for $i in $cell_type_epigenetic_factor_cond.input:
        #set file_name_with_ext = $i.name
        #if str($file_name_with_ext).find("http") >= 0 or str($file_name_with_ext).find("ftp") >= 0:
             #set file_name_with_ext = $file_name_with_ext.split('/')[-1]
        #end if
        #assert str($file_name_with_ext).find("-") >= 0, "The selected input '%s' is invalid because it does not include the '-' character which is required when setting cell type and epigenetic factor names by extracting them from the input file names." % $file_name_with_ext
        #set file_name = $file_name_with_ext.split(".")[0]
        #if str($input_name_positions) == "cell_first":
            #set cell_type_name = $file_name.split("-")[0]
            #set epigenetic_factor_name = $file_name.split("-")[1]
        #else:
            #set cell_type_name = $file_name.split("-")[1]
            #set epigenetic_factor_name = $file_name.split("-")[0]
        #end if
${cell_type_name} ${epigenetic_factor_name} ${i}
    #end for
#else:
    #for $input_items in $cell_type_epigenetic_factor_cond.input_repeat:
${input_items.cell_type_name} ${input_items.epigenetic_factor_name} ${input_items.input}
    #end for
#end if]]></configfile>
    </configfiles>
    <inputs>
        <conditional name="perform_training_cond">
            <param name="perform_training" type="select" label="Perform training?">
                <option value="yes" selected="true">Yes</option>
                <option value="no">No</option>
            </param>
            <when value="yes">
                <param name="training_iterations" type="integer" value="20" min="2" label="Number of training iterations"/>
                <param name="training_windows" type="integer" value="10000" min="2" label="Number of randomly selected windows for training"/>
            </when>
            <when value="no"/>
        </conditional>
        <conditional name="cell_type_epigenetic_factor_cond">
            <param name="cell_type_epigenetic_factor" type="select" label="Set cell type and epigenetic factor names by">
                <option value="extract" selected="true">extracting them from the selected input file names</option>
                <option value="manual">manually setting them for each selected input</option>
            </param>
            <when value="extract">
                <param name="input" type="data" format="bigwig,bam" multiple="True" label="BAM or BigWig files">
                    <validator type="empty_field"/>
                    <validator type="unspecified_build"/>
                </param>
                <param name="input_name_positions" type="select" display="radio" label="Selected input file name pattern is" help="A '-' character must separate cell type and epigenetic factor names within the selected input file names">
                    <option value="cell_first" selected="true">Cell type name - Epigenetic factor name</option>
                    <option value="cell_last">Epigenetic factor name - Cell type name</option>
                </param>
            </when>
            <when value="manual">
                <repeat name="input_repeat" title="Cell type, Epigenetic factor and Input" min="1">
                    <param name="cell_type_name" type="text" value="" label="Cell type name">
                        <validator type="empty_field"/>
                    </param>
                    <param name="epigenetic_factor_name" type="text" value="" label="Epigenetic factor name">
                        <validator type="empty_field"/>
                    </param>
                    <param name="input" type="data" format="bigwig,bam" label="BAM or BigWig file">
                        <validator type="empty_field"/>
                        <validator type="unspecified_build"/>
                    </param>
                </repeat>
            </when>
        </conditional>
        <param name="project_name" type="text" value="myProject" label="Project name" help="Outputs will have this base name">
            <validator type="empty_field"/>
        </param>
        <param name="rseed" type="integer" value="1234" min="0" max="1000000" label="Seed for IDEAS model initialization" help="Zero value generates a random seed, and this seed will be different for each job run."/>
        <conditional name="specify_genomic_window_cond">
            <param name="specify_genomic_window" type="select" label="Select Bed file that defines genomic windows on which to process the data">
                <option value="no" selected="true">No</option>
                <option value="yes">Yes</option>
            </param>
            <when value="no">
                <param name="window_size" type="integer" value="200" label="Window size in base pairs"/>
                <conditional name="restrict_chromosomes_cond">
                    <param name="restrict_chromosomes" type="select" label="Restrict processing to specified chromosomes">
                        <option value="no" selected="true">No</option>
                        <option value="yes">Yes</option>
                    </param>
                    <when value="no"/>
                    <when value="yes">
                        <repeat name="chrom_repeat" title="Chromosomes" min="1">
                            <param name="chrom" type="text" value="" label="Chromosome" help="One chromosome (e.g., chr1, chr2, chrX) per text field"/>
                        </repeat>
                    </when>
                </conditional>
            </when>
            <when value="yes">
                <param name="bed_input" type="data" format="bed" label="Bed file specifying the genomic windows"/>
            </when>
        </conditional>
        <param name="bychr" type="boolean" truevalue="true" falsevalue="" checked="False" label="Output chromosomes in separate files"/>
        <param name="reads_per_bp" type="select" display="radio" label="Calculate the signal in each genomic window using">
            <option value="6" selected="true">mean</option>
            <option value="8">max</option>
        </param>
        <param name="blacklist_input" type="data" format="bed" optional="True" multiple="True" label="Select file(s) containing regions to exclude"/>
        <param name="norm" type="boolean" truevalue="true" falsevalue="" checked="False" label="Standardize all datasets"/>
        <param name="hp" type="boolean" truevalue="true" falsevalue="" checked="False" label="Discourage state transition across chromosomes"/>
        <param name="log2" type="float" value="0" min="0" label="Use log2(x+number) transformation" help="Zero means no log2 transformation"/>
        <param name="max_states" type="float" value="0" min="0" label="Maximum number of states to be inferred" help="Zero sets the maximum to a large number"/>
        <param name="initial_states" type="integer" value="20" min="0" label="Initial number of states" help="Positive integer"/>
        <param name="max_position_classes" type="integer" value="0" min="0" label="Maximum number of position classes to be inferred" help="Zero sets the maximum to a large number"/>
        <param name="max_cell_type_clusters" type="integer" value="0" min="0" label="Maximum number of cell type clusters allowed" help="Zero sets the maximum to a large number"/>
        <param name="prior_concentration" type="float" value="1" min="0" label="Prior concentration" help="Zero value results in the default: sqrt(number of cell types)"/>
        <param name="burnin_num" type="integer" value="20" min="1" label="Number of burnin steps"/>
        <param name="mcmc_num" type="integer" value="20" min="1" label="Number of maximization steps"/>
        <param name="minerr" type="float" value="0.5" min="0" label="Minimum standard deviation for the emission Gaussian distribution" help="Zero value results in the default: 0.5"/>
        <param name="maxerr" type="float" value="1000000" min="0" label="Maximum standard deviation for the emission Gaussian distribution" help="Zero sets the maximum to a large number"/>
        <param name="output_heatmaps" type="select" display="radio" label="Output heatmaps?">
            <option value="yes" selected="true">Yes</option>
            <option value="no">No</option>
        </param>
        <param name="save_ideas_log" type="select" display="radio" label="Save IDEAS log in an additional history item">
            <option value="no" selected="true">No</option>
            <option value="yes">Yes</option>
        </param>
    </inputs>
    <outputs>
        <data name="output_log" format="txt" label="${tool.name} (output log) on ${on_string}">
            <filter>save_ideas_log == 'yes'</filter>
        </data>
        <collection name="output_pdf_collection" type="list" label="${tool.name} (heatmaps) on ${on_string}">
            <discover_datasets pattern="__name__" directory="output_pdf_dir" format="pdf"/>
            <filter>output_heatmaps == 'yes'</filter>
        </collection>
        <collection name="output_txt_collection" type="list">
            <discover_datasets pattern="__name__" directory="output_txt_dir" format="txt"/>
            <filter>perform_training_cond['perform_training'] == 'no'</filter>
        </collection>
        <collection name="output_ttraining_collection" type="list">
            <discover_datasets pattern="__name__" directory="output_training_dir" format="txt"/>
            <filter>perform_training_cond['perform_training'] == 'yes'</filter>
        </collection>
    </outputs>
    <tests>
        <test>
            <param name="cell_type_epigenetic_factor" value="extract"/>
            <param name="input" value="e001-h3k4me3.bigwig" ftype="bigwig" dbkey="hg19"/>
            <param name="input_name_positions" value="cell_first"/>
            <param name="specify_genomic_window" value="yes"/>
            <param name="bed_input" value="genomic_windows.bed" ftype="bed" dbkey="hg19"/>
            <param name="project_name" value="IDEAS_out"/>
            <output_collection name="output_txt_collection" type="list">
                <element name="IDEAS_out.cluster" file="IDEAS_out.cluster" ftype="txt"/>
                <element name="IDEAS_out.para" file="IDEAS_out.para" ftype="txt"/>
                <element name="IDEAS_out.profile" file="IDEAS_out.profile" ftype="txt"/>
                <element name="IDEAS_out.state" file="IDEAS_out.state" ftype="txt"/>
            </output_collection>
            <output_collection name="output_pdf_collection" type="list">
                <element name="IDEAS_out.pdf" file="IDEAS_out.pdf" compare="contains"/>
            </output_collection>
        </test>
        <test>
            <param name="cell_type_epigenetic_factor" value="manual"/>
            <repeat name="input_repeat">
                <param name="cell_type_name" value="e001" />
                <param name="epigenetic_factor_name" value="h3k4me3"/>
                <param name="input" value="e001-h3k4me3.bigwig" ftype="bigwig" dbkey="hg19"/>
            </repeat>
            <param name="specify_genomic_window" value="yes"/>
            <param name="bed_input" value="genomic_windows.bed" ftype="bed" dbkey="hg19"/>
            <param name="project_name" value="IDEAS_out"/>
            <output_collection name="output_txt_collection" type="list">
                <element name="IDEAS_out.cluster" file="IDEAS_out.cluster" ftype="txt"/>
                <element name="IDEAS_out.para" file="IDEAS_out.para" ftype="txt"/>
                <element name="IDEAS_out.profile" file="IDEAS_out.profile" ftype="txt"/>
                <element name="IDEAS_out.state" file="IDEAS_out.state" ftype="txt"/>
            </output_collection>
            <output_collection name="output_pdf_collection" type="list">
                <element name="IDEAS_out.pdf" file="IDEAS_out.pdf" compare="contains"/>
            </output_collection>
        </test>
    </tests>
    <help>
**What it does**

IDEAS (an **I**\ ntegrative and **D**\ iscriminative **E**\ pigenome **A**\ nnotation **S**\ ystem) identifies
de novo regulatory functions from epigenetic data in multiple cell types jointly. It is a full probabilistic
model defined on all data, and it combines signals across both the genome and cell types to boost power. The
underlying assumption of IDEAS is that, because all cell types share the same underlying DNA sequences,
**functions of each DNA segment should be correlated**. Also, cell type specific regulation is locus-dependent,
and thus IDEAS uses local epigenetic landscape to **identify de novo and local cell type clusters** without
assuming or requiring a known global cell type relationship.

IDEAS takes as input a list of epigenetic data sets (histones, chromatin accessibility, CpG methylation, TFs, etc)
or any other whole-genome data sets (e.g., scores). Currently the supported data formats include BigWig and BAM.
All data sets will first be mapped by IDEAS to a common genomic coordinate in a selected assembly (200bp windows
by default, or user-provided). The user can specify regions to be considered or removed from the analysis. The
input data may come from one cell type/condition/individual/time point (although it does not fully utilize the
advantage of IDEAS), or from multiple cell types/conditions/individuals/time points. The same set of epigenetic
features may not be present in all cell types, for which IDEAS will do imputation of the missing tracks if
specified.

.. image:: $PATH_TO_IMAGES/ideas.png

IDEAS predicts regulatory functions, denoted by epigenetic states, at each position in each cell type by
**combining information simultaneously learned from other cell types** at the same positions in cell types with
similar local epigenetic landscapes. Size of genomic intervals for determining the similarity are also learned.
All of the inferences are done through parallel infinite-state hidden Markov models (iHMM), which is a Bayesian
non-parametric technique to automatically determine the number of local cell type clusters and the number of
epigenetic states.

In addition to its improved power, IDEAS has two unique advantages:

 1) applies **linear time inference** with respect to the number of cell types, which allows it to study hundreds or more cell types jointly
 2) uses mini-batch training to **improve reproducibility** of the predicted epigenetic states, which is important because genome segmentation is not convex and hence cannot guarantee a global optimal solution.

-----

**Options**

* **Set cell type and epigenetic factor names by** - cell type and epigenetic factor names can be set manually or by extracting them from the names of the selected input datasets.  The latter case requires all selected datasets to have names that contain a "-" character.

 * **BAM or BigWig files** - select one or more Bam or Bigwig files from your history, making sure that the name of every selected input include a "-" character (e.g., e001-h3k4me3.bigwig).
 * **Cell type, Epigenetic factor and Input** - manually select any number of inputs, setting the cell type and epigenetic factor name for each.  The combination of "cell type name" and "epigenetic factor name" must be unique for each input.  For example, if you have replicate data you may want to specify the cell name as "rep1", "rep2", etc and the factor name as "rep1", "rep2", etc.

  * **Cell type name** - cell type name
  * **Epigenetic factor name** - epigenetic factor name
  * **BAM or BigWig file** - BAM or BigWig file

* **Project name** - datasets produced by IDEAS will have this base name.
* **Seed for IDEAS model initialization** - enter an integer to be used as the seed for the IDEAS model initialization.  A zero value causes IDEAS to automatically generate a random seed, and this seed will be different for each job run.
* **Select Bed file that defines genomic windows on which to process the data** - if "No" is selected, IDEAS will run whole genome segmentation.  If "Yes" is selected, IDEAS will segment genomes in the unit of the windows defined by the bed file.  This file can be in BED3, BED4 or BED5 format, but only the first three columns (chr posst posed) will be used.

 * **Window size in base pairs** - Window size in base pairs (if "No" is selected)
 * **Restrict processing to specified chromosomes** - If "Yes" is selected, processing will be restricted to specified chromosomes

  * **Chromosomes** - processing will be restricted to specified chromosomes (if "Yes" is selected)

 * **Bed file specifying the genomic windows** - bed file specifying the genomic windows (if "Yes" is selected)

* **Output chromosomes in separate files** - select "Yes" to produce separate files for each chromosome, allowing you to run IDEAS on different chromosomes separately.
* **Calculate the signal in each genomic window using** - use the bigWigAverageOverBed utility from the UCSC genome browser to calculate the signal (i.e., the number of reads per bp) in each genomic window.
* **Select file(s) containing regions to exclude** - select one or more bed files that contains regions you'd like excluded from your datasets.
* **Standardize all datasets** - select "Yes" to standardize all datasets (e.g., reads / total_reads * 20 million) so that the signals from different cell types become comparable - your datasets can be read counts, logp-values or fold change.

* **Discourage state transition across chromosomes** - select "Yes" to produce similar states in adjacent windows, making the annotation smoother, but at risk of reducing precision.
* **Use log2(x+number) transformation** - perform Log2-transformation of the input data by log2(x+number) (recommended for read count data to reduce skewness). You can enter a number that is representative of the noise level in your data (e.g., a number less than 1). If this number is at a similar scale or larger than the signal in your data, it will lose power.  For example, if your input data is mean read count per window, using 0.1 may produce better results.
* **Maximum number of states to be inferred** - restrict the maximum number of states to be generated by IDEAS; the final number of inferred states may be smaller than the number you specified
* **Initial number of states** - while IDEAS may infer 30 states or more by starting from just 20 states, it may not do so if it is trapped in a local mode. We recommend setting the initial number of states slightly larger than the number of states you expect.
* **Maximum number of position classes to be inferred** - Set this value only if:

 * you do not want position classes (e.g., for testing purposes), in this case set the value to 1
 * IDEAS runs slow because there are too many position classes, generally less than 100 position classes will run fine

* **Maximum number of cell type clusters allowed** - If you set the value to 1, then all cell types will be clustered in one group, which may be desirable if all cell types are homogeneous and you want IDEAS to use information in all cell types equally.
* **Prior concentration** - specify the prior concentration parameter; default is A=sqrt(number of cell types).  A smaller concentration parameter (e.g., 1 or less) will emphasize more on position specificity and a larger concentration parameter (e.g., 10 * number of cell types) will emphasize more on global homogeneity.
* **Number of burnin steps** - specify the number of burnin steps; default is 20.  Increasing the burnin and maximization steps will increase computing and only slightly increase accuracy, while decreasing them will reduce computing resources but may also reduce accuracy.  We recommend to run IDEAS with at least 20 burnins and 20 maximizations.  IDEAS will not stop even if it reaches a maximum mode.
* **Number of maximization steps** - specify the number of maximization steps; default is 20.
* **Minimum standard deviation for the emission Gaussian distribution** - This number multiplied by the overall standard deviation of your data will be used as a lower bound for the standard deviation for each factor in each epigenetic state (the default is 0.5). This number is useful for removing very subtle clusters in the data.  Setting this value near 0 will allow IDEAS to discover many subtle states, while setting it greater than 1 will result in IDEAS losing the ability to detect meaningful states.
* **Maximim standard deviation for the emission Gaussian distribution** - if you want to find fine-grained states you may use this option (if not used, IDEAS uses infinity), but it is rearely used unless you need more states to be inferred.

    </help>
    <citations>
        <citation type="doi">10.1093/nar/gkw278</citation>
    </citations>
</tool>