|
150
|
1 #!/usr/bin/env Rscript
|
|
|
2
|
|
|
3 suppressPackageStartupMessages(library("data.table"))
|
|
|
4 suppressPackageStartupMessages(library("optparse"))
|
|
|
5
|
|
|
6 option_list <- list(
|
|
|
7 make_option(c("--burnin_num"), action="store", dest="burnin_num", type="integer", help="Number of burnin steps"),
|
|
|
8 make_option(c("--bychr"), action="store_true", dest="bychr", default=FALSE, help="Output chromosomes in separate files"),
|
|
|
9 make_option(c("--hp"), action="store_true", dest="hp", default=FALSE, help="Discourage state transition across chromosomes"),
|
|
|
10 make_option(c("--initial_states"), action="store", dest="initial_states", type="integer", default=NULL, help="Initial number of states"),
|
|
|
11 make_option(c("--log2"), action="store", dest="log2", type="double", default=NULL, help="log2 transformation"),
|
|
|
12 make_option(c("--maxerr"), action="store", dest="maxerr", type="double", default=NULL, help="Maximum standard deviation for the emission Gaussian distribution"),
|
|
|
13 make_option(c("--max_cell_type_clusters"), action="store", dest="max_cell_type_clusters", type="integer", default=NULL, help="Maximum number of cell type clusters allowed"),
|
|
|
14 make_option(c("--max_position_classes"), action="store", dest="max_position_classes", type="integer", default=NULL, help="Maximum number of position classes to be inferred"),
|
|
|
15 make_option(c("--max_states"), action="store", dest="max_states", type="double", default=NULL, help="Maximum number of states to be inferred"),
|
|
|
16 make_option(c("--mcmc_num"), action="store", dest="mcmc_num", type="integer", help="Number of maximization steps"),
|
|
|
17 make_option(c("--minerr"), action="store", dest="minerr", type="double", default=NULL, help="Minimum standard deviation for the emission Gaussian distribution"),
|
|
|
18 make_option(c("--norm"), action="store_true", dest="norm", default=FALSE, help="Standardize all datasets"),
|
|
|
19 make_option(c("--output_log"), action="store", dest="output_log", default=NULL, help="Output log file path"),
|
|
|
20 make_option(c("--prep_output_config"), action="store", dest="prep_output_config", help="prepMat output config file"),
|
|
|
21 make_option(c("--prior_concentration"), action="store", dest="prior_concentration", type="double", default=NULL, help="Prior concentration"),
|
|
|
22 make_option(c("--project_name"), action="store", dest="project_name", help="Outputs will have this base name"),
|
|
|
23 make_option(c("--rseed"), action="store", dest="rseed", type="integer", help="Seed for IDEAS model initialization"),
|
|
|
24 make_option(c("--save_ideas_log"), action="store", dest="save_ideas_log", default=NULL, help="Flag to save IDEAS process log"),
|
|
|
25 make_option(c("--script_dir"), action="store", dest="script_dir", help="R script source directory"),
|
|
|
26 make_option(c("--thread"), action="store", dest="thread", type="integer", help="Process threads"),
|
|
|
27 make_option(c("--tmp_dir"), action="store", dest="tmp_dir", help="Directory of bed files"),
|
|
|
28 make_option(c("--training_iterations"), action="store", dest="training_iterations", type="integer", default=NULL, help="Number of training iterations"),
|
|
|
29 make_option(c("--training_windows"), action="store", dest="training_windows", type="integer", default=NULL, help="Number of training iterations"),
|
|
|
30 make_option(c("--windows_bed"), action="store", dest="windows_bed", default=NULL, help="Bed file containing bed windows"),
|
|
|
31 make_option(c("--window_end"), action="store", dest="window_end", type="integer", default=NULL, help="Windows positions by chromosome end value"),
|
|
|
32 make_option(c("--window_start"), action="store", dest="window_start", type="integer", default=NULL, help="Windows positions by chromosome start value")
|
|
|
33 )
|
|
|
34
|
|
|
35 parser <- OptionParser(usage="%prog [options] file", option_list=option_list)
|
|
|
36 args <- parse_args(parser, positional_arguments=TRUE)
|
|
|
37 opt <- args$options
|
|
|
38
|
|
|
39 add_output_redirect <- function(cmd, save_ideas_log, output_log, default_log_name) {
|
|
|
40 if (is.null(save_ideas_log)) {
|
|
|
41 cmd = paste(cmd, "&>>", default_log_name, sep=" ");
|
|
|
42 }else {
|
|
|
43 cmd = paste(cmd, "&>>", output_log, sep=" ");
|
|
|
44 }
|
|
|
45 return(cmd);
|
|
|
46 }
|
|
|
47
|
|
|
48 combine_state <- function(parafiles, method="ward.D", mycut=0.9, pcut=1.0) {
|
|
|
49 X = NULL;
|
|
|
50 K = NULL;
|
|
|
51 I = NULL;
|
|
|
52 myheader = NULL;
|
|
|
53 p = NULL;
|
|
|
54 for(i in 1:length(parafiles)) {
|
|
|
55 x = fread(parafiles[i]);
|
|
|
56 t = max(which(is.na(x[1,])==F));
|
|
|
57 x = as.matrix(x[,1:t]);
|
|
|
58 if(i==1) {
|
|
|
59 myheader = colnames(x);
|
|
|
60 p = sqrt(9/4-2*(1-length(myheader))) - 3 / 2;
|
|
|
61 }
|
|
|
62 m = match(myheader[1:p+1], colnames(x)[1:p+1]);
|
|
|
63 v = NULL;
|
|
|
64 for(ii in 1:p) {
|
|
|
65 for(jj in 1:ii) {
|
|
|
66 a = max(m[ii],m[jj]);
|
|
|
67 b = min(m[ii],m[jj]);
|
|
|
68 v = c(v, a*(a+1)/2+b-a);
|
|
|
69 }
|
|
|
70 }
|
|
|
71 X = rbind(X, array(as.matrix(x[, c(1, 1+m, 1+p+v)]), dim=c(length(x) / (1+p+length(v)), 1 + p + length(v))));
|
|
|
72 K = c(K, dim(x)[1]);
|
|
|
73 I = c(I, rep(i, dim(x)[1]));
|
|
|
74 }
|
|
|
75 N = length(parafiles);
|
|
|
76 p = (sqrt(1 + dim(X)[2] * 8) - 3) / 2;
|
|
|
77 omycut = mycut;
|
|
|
78 mycut = round(length(parafiles) * mycut);
|
|
|
79 M = array(X[,1:p+1] / X[,1], dim=c(dim(X)[1], p));
|
|
|
80 V = array(0, dim=c(dim(X)[1] * p, p));
|
|
|
81 for(i in 1:dim(X)[1]) {
|
|
|
82 t = (i - 1) * p;
|
|
|
83 l = 1;
|
|
|
84 for(j in 1:p) {
|
|
|
85 for(k in 1:j) {
|
|
|
86 V[t+j, k] = V[t+k, j] = X[i,1+p+l] / X[i,1] - M[i,j] * M[i,k];
|
|
|
87 l = l + 1;
|
|
|
88 }
|
|
|
89 }
|
|
|
90 V[t+1:p,] = t(solve(chol(V[t+1:p,] + diag(1e-1,p))));
|
|
|
91 }
|
|
|
92 D = array(0, dim=rep(dim(X)[1],2));
|
|
|
93 for(i in 2:dim(X)[1]) {
|
|
|
94 for(j in 1:(i-1)) {
|
|
|
95 D[i,j] = D[j,i] = sqrt((sum((V[(i-1)*p+1:p,]%*%(M[i,]-M[j,]))^2) + sum((V[(j-1)*p+1:p,]%*%(M[i,]-M[j,]))^2)));
|
|
|
96 }
|
|
|
97 }
|
|
|
98 MM = NULL;
|
|
|
99 kk = NULL;
|
|
|
100 for(i in 1:N) {
|
|
|
101 t = 1:K[i];
|
|
|
102 if(i > 1) {
|
|
|
103 t = t + sum(K[1:(i-1)]);
|
|
|
104 }
|
|
|
105 t = (1:dim(D)[1])[-t];
|
|
|
106 h = hclust(as.dist(D[t,t]), method=method);
|
|
|
107 k = -1;
|
|
|
108 tM = NULL;
|
|
|
109 for(j in min(K):(min(length(t), max(K)*2))) {
|
|
|
110 m = cutree(h,k=j);
|
|
|
111 tt = NULL;
|
|
|
112 for(l in 1:j) {
|
|
|
113 tt[l] = length(unique(I[t[which(m==l)]]));
|
|
|
114 }
|
|
|
115 tk = length(which(tt>=mycut));
|
|
|
116 if(tk > k) {
|
|
|
117 k = tk;
|
|
|
118 tM = make_parameter(1:j, I[t], m, mycut, X[t,]);
|
|
|
119 } else if(tk < k) {
|
|
|
120 break;
|
|
|
121 }
|
|
|
122 }
|
|
|
123 kk[i] = k;
|
|
|
124 MM = rbind(MM, cbind(i, tM));
|
|
|
125 }
|
|
|
126 mysel = median(kk);
|
|
|
127 h = hclust(as.dist(D), method=method);
|
|
|
128 rt = rep(0, max(K)*2);
|
|
|
129 k = -1;
|
|
|
130 for(i in min(K):min(dim(D)[1], max(K)*2)) {
|
|
|
131 m = cutree(h,k=i);
|
|
|
132 tt = NULL;
|
|
|
133 for(j in 1:i) {
|
|
|
134 tt[j] = length(unique(I[which(m==j)]));
|
|
|
135 }
|
|
|
136 tk = length(which(tt>=mycut));
|
|
|
137 if(tk==mysel | tk<k) {
|
|
|
138 break;
|
|
|
139 }
|
|
|
140 k = tk;
|
|
|
141 rt[i] = length(which(tt>=mycut));
|
|
|
142 }
|
|
|
143 mysel = max(k,tk);
|
|
|
144 m = cutree(h, k=mysel);
|
|
|
145 nn = NULL;
|
|
|
146 for(i in 1:mysel) {
|
|
|
147 t = which(m==i);
|
|
|
148 nn[i] = sum(X[t,1]);
|
|
|
149 }
|
|
|
150 oo = order(nn, decreasing=T);
|
|
|
151 rt = make_parameter(oo, I, m, mycut, X);
|
|
|
152 onstate = max(rt[,1]) + 1;
|
|
|
153 ooo = NULL;
|
|
|
154 for(i in oo) {
|
|
|
155 t = which(m==i);
|
|
|
156 if(length(unique(I[t])) >= mycut) {
|
|
|
157 ooo = c(ooo, i);
|
|
|
158 }
|
|
|
159 }
|
|
|
160 d = NULL;
|
|
|
161 for(i in 1:N) {
|
|
|
162 d = rbind(d, compare_two(rt, MM[MM[,1]==i,-1])[1:onstate]);
|
|
|
163 }
|
|
|
164 dd = array(cutree(hclust(dist(c(d))), k=2), dim=dim(d));
|
|
|
165 kk = table(c(dd));
|
|
|
166 kk = which(as.integer(kk)==max(as.integer(kk)))[1];
|
|
|
167 pp = apply(dd, 2, function(x){length(which(x!=kk))/length(x)});
|
|
|
168 pp0 = apply(d, 2, function(x){length(which(x>0.5))/length(x)});
|
|
|
169 pp[pp0<pp] = pp0[pp0<pp];
|
|
|
170 t = which(pp > pcut);
|
|
|
171 if(length(t) > 0) {
|
|
|
172 j = 0;
|
|
|
173 nrt = NULL;
|
|
|
174 for(i in (1:onstate-1)[-t]) {
|
|
|
175 nrt = rbind(nrt, cbind(j, rt[rt[,1]==i,-1]));
|
|
|
176 j = j + 1;
|
|
|
177 }
|
|
|
178 rt = nrt;
|
|
|
179 ooo = ooo[-t];
|
|
|
180 }
|
|
|
181 nrt = NULL;
|
|
|
182 for(i in 0:max(rt[,1])) {
|
|
|
183 t = which(rt[,1]==i);
|
|
|
184 nrt = rbind(nrt, apply(array(rt[t,], dim=c(length(t), dim(rt)[2])), 2, sum)[-1]);
|
|
|
185 }
|
|
|
186 rt = nrt;
|
|
|
187 colnames(rt) = myheader;
|
|
|
188 O = NULL;
|
|
|
189 Ip = NULL;
|
|
|
190 Xp = NULL;
|
|
|
191 k = 0;
|
|
|
192 for(i in 1:length(parafiles)) {
|
|
|
193 str = gsub(".para", ".profile", parafiles[i]);
|
|
|
194 p = as.matrix(read.table(str));
|
|
|
195 u = array(0, dim=c(dim(p)[1], length(ooo)));
|
|
|
196 for(j in 1:length(ooo)) {
|
|
|
197 t = which(m[k+1:K[i]] == ooo[j]);
|
|
|
198 u[,j] = apply(array(p[,1+t], dim=c(dim(p)[1], length(t))), 1, sum);
|
|
|
199 }
|
|
|
200 k = k + K[i];
|
|
|
201 u = u / (apply(u, 1, sum) + 1e-10);
|
|
|
202 Xp = rbind(Xp, cbind(p[,1], u));
|
|
|
203 Ip = c(Ip, rep(i,dim(u)[1]));
|
|
|
204 }
|
|
|
205 hp = hclust(dist(((Xp[,-1]+min(1e-3, min(Xp[,-1][Xp[,-1]>0]))))), method=method);
|
|
|
206 ocut = min(mycut/2, length(parafiles)/2);
|
|
|
207 t = range(as.integer(table(Ip)));
|
|
|
208 Kp = NULL;
|
|
|
209 for(i in t[1]:(t[2]*2)) {
|
|
|
210 m = cutree(hp, k=i);
|
|
|
211 tt = table(Ip,m);
|
|
|
212 ll = apply(tt, 2, function(x){length(which(x>0))});
|
|
|
213 Kp = c(Kp, length(which(ll>=ocut)));
|
|
|
214 }
|
|
|
215 oN = (t[1]:(t[2]*2))[which(Kp==max(Kp))[1]];
|
|
|
216 m = cutree(hp, k=oN);
|
|
|
217 tt = table(Ip,m);
|
|
|
218 ll = apply(tt, 2, function(x){length(which(x>0))});
|
|
|
219 tt = which(ll>=ocut);
|
|
|
220 for(i in tt) {
|
|
|
221 t = which(m==i);
|
|
|
222 O = rbind(O, c(sum(Xp[t, 1]), apply(array(Xp[t,-1]*Xp[t,1], dim=c(length(t), dim(Xp)[2]-1)), 2, sum)/sum(Xp[t, 1])));
|
|
|
223 }
|
|
|
224 nrt = NULL;
|
|
|
225 nrt$para = rt;
|
|
|
226 nrt$profile = O;
|
|
|
227 return(nrt);
|
|
|
228 }
|
|
|
229
|
|
|
230 compare_two <- function(n, m) {
|
|
|
231 NN = get_mean(n);
|
|
|
232 MM = get_mean(m);
|
|
|
233 p = (-3 + sqrt(9 + 8 * (dim(n)[2] - 2))) / 2;
|
|
|
234 dd = NULL;
|
|
|
235 for (i in 1:dim(NN)[1]) {
|
|
|
236 dd[i] = min(apply(array(MM[,1:p], dim=c(dim(MM)[1],p)), 1, function(x){sqrt(sum((x-NN[i,1:p])^2))}));
|
|
|
237 }
|
|
|
238 for (i in 1:dim(MM)[1]) {
|
|
|
239 dd[i+dim(NN)[1]] = min(apply(array(NN[,1:p], dim=c(dim(NN)[1],p)), 1, function(x){sqrt(sum((x-MM[i,1:p])^2))}));
|
|
|
240 }
|
|
|
241 return(dd);
|
|
|
242 }
|
|
|
243
|
|
|
244 get_mean <- function(n) {
|
|
|
245 N = NULL;
|
|
|
246 for(i in sort(unique(n[,1]))) {
|
|
|
247 t = which(n[,1]==i);
|
|
|
248 N = rbind(N, apply(array(n[t,], dim=c(length(t), dim(n)[2])), 2, sum)[-1]);
|
|
|
249 }
|
|
|
250 NN = N[,-1] / N[,1];
|
|
|
251 return(array(NN, dim=c(length(NN)/(dim(n)[2]-2), dim(n)[2]-2)));
|
|
|
252 }
|
|
|
253
|
|
|
254 make_parameter <- function(myorder, id, mem, mycut, para) {
|
|
|
255 rt = NULL;
|
|
|
256 j = 0;
|
|
|
257 for(i in myorder) {
|
|
|
258 t = which(mem==i);
|
|
|
259 if (length(unique(id[t])) >= mycut) {
|
|
|
260 rt = rbind(rt, cbind(j, array(para[t,], dim=c(length(t), dim(para)[2]))));
|
|
|
261 j = j + 1;
|
|
|
262 }
|
|
|
263 }
|
|
|
264 return(rt);
|
|
|
265 }
|
|
|
266
|
|
|
267 run_cmd <- function(cmd, save_ideas_log, output_log, default_log_name) {
|
|
|
268 cat("\n\n >>>>> cmd:\n", cmd, "\n\n");
|
|
|
269 rc = system(cmd);
|
|
|
270 if (rc != 0) {
|
|
|
271 if (is.null(save_ideas_log)) {
|
|
|
272 file.rename(default_log_name, output_log);
|
|
|
273 }
|
|
|
274 quit(rc);
|
|
|
275 }
|
|
|
276 }
|
|
|
277
|
|
|
278 default_log_name = "ideas_log.txt";
|
|
|
279 output_base_name = opt$project_name;
|
|
|
280 cmd = paste("ideas", opt$prep_output_config, sep=" ");
|
|
|
281 if (!is.null(opt$windows_bed)) {
|
|
|
282 cmd = paste(cmd, opt$windows_bed, sep=" ");
|
|
|
283 }
|
|
|
284 if (!is.null(opt$training_iterations)) {
|
|
|
285 cmd = paste(cmd, "-impute none", sep=" ");
|
|
|
286 }
|
|
|
287 if (opt$bychr) {
|
|
|
288 cmd = paste(cmd, "-bychr", sep=" ");
|
|
|
289 }
|
|
|
290 if (opt$hp) {
|
|
|
291 cmd = paste(cmd, "-hp", sep=" ");
|
|
|
292 }
|
|
|
293 if (opt$norm) {
|
|
|
294 cmd = paste(cmd, "-norm", sep=" ");
|
|
|
295 }
|
|
|
296 if (!is.null(opt$window_start) && !is.null(opt$window_end)) {
|
|
|
297 cmd = paste(cmd, "-inv", opt$window_start, opt$window_end, sep=" ");
|
|
|
298 }
|
|
|
299 if (!is.null(opt$log2)) {
|
|
|
300 cmd = paste(cmd, "-log2", opt$log2, sep=" ");
|
|
|
301 }
|
|
|
302 if (!is.null(opt$max_states)) {
|
|
|
303 cmd = paste(cmd, "-G", opt$max_states, sep=" ");
|
|
|
304 }
|
|
|
305 if (!is.null(opt$initial_states)) {
|
|
|
306 cmd = paste(cmd, "-C", opt$initial_states, sep=" ");
|
|
|
307 }
|
|
|
308 if (!is.null(opt$max_position_classes)) {
|
|
|
309 cmd = paste(cmd, "-P", opt$max_position_classes, sep=" ");
|
|
|
310 }
|
|
|
311 if (!is.null(opt$max_cell_type_clusters)) {
|
|
|
312 cmd = paste(cmd, "-K", opt$max_cell_type_clusters, sep=" ");
|
|
|
313 }
|
|
|
314 if (!is.null(opt$prior_concentration)) {
|
|
|
315 cmd = paste(cmd, "-A", opt$prior_concentration, sep=" ");
|
|
|
316 }
|
|
|
317 cmd = paste(cmd, "-sample", opt$burnin_num, opt$mcmc_num, sep=" ");
|
|
|
318 if (!is.null(opt$minerr)) {
|
|
|
319 cmd = paste(cmd, "-minerr", opt$minerr, sep=" ");
|
|
|
320 }
|
|
|
321 if (!is.null(opt$maxerr)) {
|
|
|
322 cmd = paste(cmd, "-maxerr", opt$maxerr, sep=" ");
|
|
|
323 }
|
|
|
324 cmd = paste(cmd, "-rseed", opt$rseed, sep=" ");
|
|
|
325 cmd = paste(cmd, "-thread", opt$thread, sep=" ");
|
|
|
326
|
|
|
327 if (is.null(opt$training_iterations)) {
|
|
152
|
328 final_cmd = paste(cmd, "-o", output_base_name, sep=" ");
|
|
|
329 final_cmd = add_output_redirect(final_cmd, opt$save_ideas_log, opt$output_log, default_log_name);
|
|
|
330 run_cmd(final_cmd, opt$save_ideas_log, opt$output_log, default_log_name);
|
|
150
|
331 } else {
|
|
152
|
332 output_para0 = paste(output_base_name, ".para0", sep="");
|
|
|
333 output_profile0 = paste(output_base_name, ".profile0", sep="");
|
|
150
|
334 for (i in 1:opt$training_iterations) {
|
|
152
|
335 final_cmd = paste(cmd, "-o", paste(output_base_name, ".tmp.", i, sep=""), sep=" ");
|
|
|
336 final_cmd = add_output_redirect(final_cmd, opt$save_ideas_log, opt$output_log, default_log_name);
|
|
|
337 run_cmd(final_cmd, opt$save_ideas_log, opt$output_log, default_log_name);
|
|
150
|
338 }
|
|
|
339 tpara = combine_state(paste(output_base_name, ".tmp.", (1:opt$training_iterations), ".para", sep=""), mycut=0.5);
|
|
|
340 para = tpara$para;
|
|
152
|
341 write.table(tpara$profile, output_profile0, quote=F, row.names=F, col.names=F);
|
|
150
|
342 para = apply(para, 1, function(x){paste(x, collapse=" ")});
|
|
|
343 para = c(readLines(paste(output_base_name, ".tmp.1.para", sep=""), n=1), para);
|
|
|
344 writeLines(para, output_para0);
|
|
152
|
345 cmd = c(cmd, "-otherpara", output_para0, output_profile0);
|
|
|
346 if (length(which(cmd == "-G")) == 0) {
|
|
|
347 cmd = c(cmd, "-G", length(para)-1);
|
|
|
348 } else {
|
|
|
349 tt = which(cmd == "-G");
|
|
|
350 cmd[tt + 1] = length(para)-1;
|
|
|
351 }
|
|
|
352 tt = which(cmd == '-C');
|
|
|
353 if(length(tt)>0) {
|
|
|
354 cmd = cmd[-c(tt, tt+1)];
|
|
|
355 }
|
|
150
|
356 }
|
