Mercurial > repos > bgruening > create_tool_recommendation_model

comparison optimise_hyperparameters.py @ 4:f0da532be419 draft

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
"planemo upload for repository https://github.com/bgruening/galaxytools/tree/recommendation_training/tools/tool_recommendation_model commit 5eebc0cb44e71f581d548b7e842002705dd155eb"
author bgruening
date Fri, 06 May 2022 09:04:44 +0000
parents 98bc44d17561
children
comparison
equal deleted inserted replaced
3:98bc44d17561 4:f0da532be419
1 """ 1 """
2 Find the optimal combination of hyperparameters 2 Find the optimal combination of hyperparameters
3 """ 3 """
4 4
5 import numpy as np 5 import numpy as np
6 from hyperopt import fmin, tpe, hp, STATUS_OK, Trials
7
8 from keras.models import Sequential
9 from keras.layers import Dense, GRU, Dropout
10 from keras.layers.embeddings import Embedding
11 from keras.layers.core import SpatialDropout1D
12 from keras.optimizers import RMSprop
13 from keras.callbacks import EarlyStopping
14
15 import utils 6 import utils
7 from hyperopt import fmin, hp, STATUS_OK, tpe, Trials
8 from tensorflow.keras.callbacks import EarlyStopping
9 from tensorflow.keras.layers import Dense, Dropout, Embedding, GRU, SpatialDropout1D
10 from tensorflow.keras.models import Sequential
11 from tensorflow.keras.optimizers import RMSprop
16 12
17 13
18 class HyperparameterOptimisation: 14 class HyperparameterOptimisation:
19
20 def __init__(self): 15 def __init__(self):
21 """ Init method. """ 16 """ Init method. """
22 17
23 def train_model(self, config, reverse_dictionary, train_data, train_labels, test_data, test_labels, tool_tr_samples, class_weights): 18 def train_model(
19 self,
20 config,
21 reverse_dictionary,
22 train_data,
23 train_labels,
24 test_data,
25 test_labels,
26 tool_tr_samples,
27 class_weights,
28 ):
24 """ 29 """
25 Train a model and report accuracy 30 Train a model and report accuracy
26 """ 31 """
27 # convert items to integer 32 # convert items to integer
28 l_batch_size = list(map(int, config["batch_size"].split(","))) 33 l_batch_size = list(map(int, config["batch_size"].split(",")))
38 optimize_n_epochs = int(config["optimize_n_epochs"]) 43 optimize_n_epochs = int(config["optimize_n_epochs"])
39 44
40 # get dimensions 45 # get dimensions
41 dimensions = len(reverse_dictionary) + 1 46 dimensions = len(reverse_dictionary) + 1
42 best_model_params = dict() 47 best_model_params = dict()
43 early_stopping = EarlyStopping(monitor='val_loss', mode='min', verbose=1, min_delta=1e-1, restore_best_weights=True) 48 early_stopping = EarlyStopping(
49 monitor="val_loss",
50 mode="min",
51 verbose=1,
52 min_delta=1e-1,
53 restore_best_weights=True,
54 )
44 55
45 # specify the search space for finding the best combination of parameters using Bayesian optimisation 56 # specify the search space for finding the best combination of parameters using Bayesian optimisation
46 params = { 57 params = {
47 "embedding_size": hp.quniform("embedding_size", l_embedding_size[0], l_embedding_size[1], 1), 58 "embedding_size": hp.quniform(
59 "embedding_size", l_embedding_size[0], l_embedding_size[1], 1
60 ),
48 "units": hp.quniform("units", l_units[0], l_units[1], 1), 61 "units": hp.quniform("units", l_units[0], l_units[1], 1),
49 "batch_size": hp.quniform("batch_size", l_batch_size[0], l_batch_size[1], 1), 62 "batch_size": hp.quniform(
50 "learning_rate": hp.loguniform("learning_rate", np.log(l_learning_rate[0]), np.log(l_learning_rate[1])), 63 "batch_size", l_batch_size[0], l_batch_size[1], 1
64 ),
65 "learning_rate": hp.loguniform(
66 "learning_rate", np.log(l_learning_rate[0]), np.log(l_learning_rate[1])
67 ),
51 "dropout": hp.uniform("dropout", l_dropout[0], l_dropout[1]), 68 "dropout": hp.uniform("dropout", l_dropout[0], l_dropout[1]),
52 "spatial_dropout": hp.uniform("spatial_dropout", l_spatial_dropout[0], l_spatial_dropout[1]), 69 "spatial_dropout": hp.uniform(
53 "recurrent_dropout": hp.uniform("recurrent_dropout", l_recurrent_dropout[0], l_recurrent_dropout[1]) 70 "spatial_dropout", l_spatial_dropout[0], l_spatial_dropout[1]
71 ),
72 "recurrent_dropout": hp.uniform(
73 "recurrent_dropout", l_recurrent_dropout[0], l_recurrent_dropout[1]
74 ),
54 } 75 }
55 76
56 def create_model(params): 77 def create_model(params):
57 model = Sequential() 78 model = Sequential()
58 model.add(Embedding(dimensions, int(params["embedding_size"]), mask_zero=True)) 79 model.add(
80 Embedding(dimensions, int(params["embedding_size"]), mask_zero=True)
81 )
59 model.add(SpatialDropout1D(params["spatial_dropout"])) 82 model.add(SpatialDropout1D(params["spatial_dropout"]))
60 model.add(GRU(int(params["units"]), dropout=params["dropout"], recurrent_dropout=params["recurrent_dropout"], return_sequences=True, activation="elu")) 83 model.add(
84 GRU(
85 int(params["units"]),
86 dropout=params["dropout"],
87 recurrent_dropout=params["recurrent_dropout"],
88 return_sequences=True,
89 activation="elu",
90 )
91 )
61 model.add(Dropout(params["dropout"])) 92 model.add(Dropout(params["dropout"]))
62 model.add(GRU(int(params["units"]), dropout=params["dropout"], recurrent_dropout=params["recurrent_dropout"], return_sequences=False, activation="elu")) 93 model.add(
94 GRU(
95 int(params["units"]),
96 dropout=params["dropout"],
97 recurrent_dropout=params["recurrent_dropout"],
98 return_sequences=False,
99 activation="elu",
100 )
101 )
63 model.add(Dropout(params["dropout"])) 102 model.add(Dropout(params["dropout"]))
64 model.add(Dense(2 * dimensions, activation="sigmoid")) 103 model.add(Dense(2 * dimensions, activation="sigmoid"))
65 optimizer_rms = RMSprop(lr=params["learning_rate"]) 104 optimizer_rms = RMSprop(lr=params["learning_rate"])
66 batch_size = int(params["batch_size"]) 105 batch_size = int(params["batch_size"])
67 model.compile(loss=utils.weighted_loss(class_weights), optimizer=optimizer_rms) 106 model.compile(
107 loss=utils.weighted_loss(class_weights), optimizer=optimizer_rms
108 )
68 print(model.summary()) 109 print(model.summary())
69 model_fit = model.fit_generator( 110 model_fit = model.fit(
70 utils.balanced_sample_generator( 111 utils.balanced_sample_generator(
71 train_data, 112 train_data,
72 train_labels, 113 train_labels,
73 batch_size, 114 batch_size,
74 tool_tr_samples, 115 tool_tr_samples,
75 reverse_dictionary 116 reverse_dictionary,
76 ), 117 ),
77 steps_per_epoch=len(train_data) // batch_size, 118 steps_per_epoch=len(train_data) // batch_size,
78 epochs=optimize_n_epochs, 119 epochs=optimize_n_epochs,
79 callbacks=[early_stopping], 120 callbacks=[early_stopping],
80 validation_data=(test_data, test_labels), 121 validation_data=(test_data, test_labels),
81 verbose=2, 122 verbose=2,
82 shuffle=True 123 shuffle=True,
83 ) 124 )
84 return {'loss': model_fit.history["val_loss"][-1], 'status': STATUS_OK, 'model': model} 125 return {
126 "loss": model_fit.history["val_loss"][-1],
127 "status": STATUS_OK,
128 "model": model,
129 }
130
85 # minimize the objective function using the set of parameters above 131 # minimize the objective function using the set of parameters above
86 trials = Trials() 132 trials = Trials()
87 learned_params = fmin(create_model, params, trials=trials, algo=tpe.suggest, max_evals=int(config["max_evals"])) 133 learned_params = fmin(
88 best_model = trials.results[np.argmin([r['loss'] for r in trials.results])]['model'] 134 create_model,
135 params,
136 trials=trials,
137 algo=tpe.suggest,
138 max_evals=int(config["max_evals"]),
139 )
140 best_model = trials.results[np.argmin([r["loss"] for r in trials.results])][
141 "model"
142 ]
89 # set the best params with respective values 143 # set the best params with respective values
90 for item in learned_params: 144 for item in learned_params:
91 item_val = learned_params[item] 145 item_val = learned_params[item]
92 best_model_params[item] = item_val 146 best_model_params[item] = item_val
93 return best_model_params, best_model 147 return best_model_params, best_model