Coverage for mlair/model_modules/model_class.py: 61%

1""" 

2Module for neural models to use during experiment. 

3 

4To work properly, each customised model needs to inherit from AbstractModelClass and needs an implementation of the 

5set_model method. 

6 

7In this module, you can find some exemplary model classes that have been build and were running in a experiment. 

8 

9* `MyLittleModel`: small model implementation with a single 1x1 Conv, and 4 Dense layers (64, 32, 16, window_lead_time). 

10* `MyBranchedModel`: a model with single 1x1 Conv, and 4 Dense layers (64, 32, 16, window_lead_time), it has three 

11 output branches from different layers of the model. 

12* `MyTowerModel`: a more complex model with inception blocks (called towers) 

13* `MyPaperModel`: A model used for the publication: <Add Publication Title / Citation> 

14 

15In addition, a short introduction how to create your own model is given hereinafter. 

16 

17How to create a customised model? 

18################################# 

19 

20* Create a new class: 

21 

22 .. code-block:: python 

23 

24 class MyCustomisedModel(AbstractModelClass): 

25 

26 def __init__(self, input_shape: list, output_shape: list): 

27 

28 super().__init__(input_shape[0], output_shape[0]) 

29 

30 # settings 

31 self.dropout_rate = 0.1 

32 self.activation = keras.layers.PReLU 

33 

34 # apply to model 

35 self.set_model() 

36 self.set_compile_options() 

37 self.set_custom_objects(loss=self.compile_options['loss']) 

38 

39* Make sure to add the `super().__init__()` and at least `set_model()` and `set_compile_options()` to your custom init 

40 method. 

41* If you have custom objects in your model, that are not part of keras, you need to add them to custom objects. To do 

42 this, call `set_custom_objects` with arbitrarily kwargs. In the shown example, the loss has been added, because it 

43 wasn't a standard loss. Apart from this, we always encourage you to add the loss as custom object, to prevent 

44 potential errors when loading an already created model instead of training a new one. 

45* Build your model inside `set_model()`, e.g. 

46 

47 .. code-block:: python 

48 

49 class MyCustomisedModel(AbstractModelClass): 

50 

51 def set_model(self): 

52 x_input = keras.layers.Input(shape=self._input_shape) 

53 x_in = keras.layers.Conv2D(32, (1, 1), padding='same', name='{}_Conv_1x1'.format("major"))(x_input) 

54 x_in = self.activation(name='{}_conv_act'.format("major"))(x_in) 

55 x_in = keras.layers.Flatten(name='{}'.format("major"))(x_in) 

56 x_in = keras.layers.Dropout(self.dropout_rate, name='{}_Dropout_1'.format("major"))(x_in) 

57 x_in = keras.layers.Dense(16, name='{}_Dense_16'.format("major"))(x_in) 

58 x_in = self.activation()(x_in) 

59 x_in = keras.layers.Dense(self._output_shape, name='{}_Dense'.format("major"))(x_in) 

60 out_main = self.activation()(x_in) 

61 self.model = keras.Model(inputs=x_input, outputs=[out_main]) 

62 

63* Your are free, how to design your model. Just make sure to save it in the class attribute model. 

64* Additionally, set your custom compile options including the loss. 

65 

66 .. code-block:: python 

67 

68 class MyCustomisedModel(AbstractModelClass): 

69 

70 def set_compile_options(self): 

71 self.initial_lr = 1e-2 

72 self.optimizer = keras.optimizers.SGD(lr=self.initial_lr, momentum=0.9) 

73 self.lr_decay = mlair.model_modules.keras_extensions.LearningRateDecay(base_lr=self.initial_lr, 

74 drop=.94, 

75 epochs_drop=10) 

76 self.loss = keras.losses.mean_squared_error 

77 self.compile_options = {"metrics": ["mse", "mae"]} 

78 

79* If you have a branched model with multiple outputs, you need either set only a single loss for all branch outputs or 

80 to provide the same number of loss functions considering the right order. E.g. 

81 

82 .. code-block:: python 

83 

84 class MyCustomisedModel(AbstractModelClass): 

85 

86 def set_model(self): 

87 ... 

88 self.model = keras.Model(inputs=x_input, outputs=[out_minor_1, out_minor_2, out_main]) 

89 

90 def set_compile_options(self): 

91 self.loss = [keras.losses.mean_absolute_error] + # for out_minor_1 

92 [keras.losses.mean_squared_error] + # for out_minor_2 

93 [keras.losses.mean_squared_error] # for out_main 

94 

95 

96How to access my customised model? 

97################################## 

98 

99If the customised model is created, you can easily access the model with 

100 

101>>> MyCustomisedModel().model 

102<your custom model> 

103 

104The loss is accessible via 

105 

106>>> MyCustomisedModel().loss 

107<your custom loss> 

108 

109You can treat the instance of your model as instance but also as the model itself. If you call a method, that refers to 

110the model instead of the model instance, you can directly apply the command on the instance instead of adding the model 

111parameter call. 

112 

113>>> MyCustomisedModel().model.compile(**kwargs) == MyCustomisedModel().compile(**kwargs) 

114True 

115 

116""" 

117 

118import mlair.model_modules.keras_extensions 

119 

120__author__ = "Lukas Leufen, Felix Kleinert" 

121__date__ = '2020-05-12' 

122 

123import tensorflow.keras as keras 

124 

125from mlair.model_modules import AbstractModelClass 

126from mlair.model_modules.inception_model import InceptionModelBase 

127from mlair.model_modules.flatten import flatten_tail 

128from mlair.model_modules.advanced_paddings import PadUtils, Padding2D, SymmetricPadding2D 

129from mlair.model_modules.loss import l_p_loss 

130 

131 

132class MyLittleModelHourly(AbstractModelClass): 

133 """ 

134 A customised model with a 1x1 Conv, and 4 Dense layers (64, 32, 16, window_lead_time), where the last layer is the 

135 output layer depending on the window_lead_time parameter. Dropout is used between the Convolution and the first 

136 Dense layer. 

137 """ 

138 

139 def __init__(self, input_shape: list, output_shape: list): 

140 """ 

141 Sets model and loss depending on the given arguments. 

142 

143 :param shape_inputs: list of input shapes (expect len=1 with shape=(window_hist, station, variables)) 

144 :param shape_outputs: list of output shapes (expect len=1 with shape=(window_forecast)) 

145 """ 

146 

147 assert len(input_shape) == 1 

148 assert len(output_shape) == 1 

149 super().__init__(input_shape[0], output_shape[0]) 

150 

151 # settings 

152 self.dropout_rate = 0.1 

153 self.regularizer = keras.regularizers.l2(0.001) 

154 self.activation = keras.layers.PReLU 

155 

156 # apply to model 

157 self.set_model() 

158 self.set_compile_options() 

159 self.set_custom_objects(loss=self.compile_options['loss']) 

160 

161 def set_model(self): 

162 """ 

163 Build the model. 

164 """ 

165 

166 # add 1 to window_size to include current time step t0 

167 x_input = keras.layers.Input(shape=self._input_shape) 

168 x_in = keras.layers.Conv2D(128, (1, 1), padding='same', name='{}_Conv_1x1_128'.format("major"))(x_input) 

169 x_in = self.activation()(x_in) 

170 x_in = keras.layers.Conv2D(64, (1, 1), padding='same', name='{}_Conv_1x1_64'.format("major"))(x_in) 

171 x_in = self.activation()(x_in) 

172 x_in = keras.layers.Conv2D(32, (1, 1), padding='same', name='{}_Conv_1x1_32'.format("major"))(x_in) 

173 x_in = self.activation()(x_in) 

174 x_in = keras.layers.Flatten(name='{}'.format("major"))(x_in) 

175 x_in = keras.layers.Dropout(self.dropout_rate, name='{}_Dropout_1'.format("major"))(x_in) 

176 x_in = keras.layers.Dense(128, name='{}_Dense_128'.format("major"))(x_in) 

177 x_in = self.activation()(x_in) 

178 x_in = keras.layers.Dense(64, name='{}_Dense_64'.format("major"))(x_in) 

179 x_in = self.activation()(x_in) 

180 x_in = keras.layers.Dense(self._output_shape, name='{}_Dense'.format("major"))(x_in) 

181 out_main = self.activation()(x_in) 

182 self.model = keras.Model(inputs=x_input, outputs=[out_main]) 

183 

184 def set_compile_options(self): 

185 self.initial_lr = 1e-2 

186 self.optimizer = keras.optimizers.SGD(lr=self.initial_lr, momentum=0.9) 

187 self.compile_options = {"loss": [keras.losses.mean_squared_error], "metrics": ["mse", "mae"]} 

188 

189 

190class MyBranchedModel(AbstractModelClass): 

191 """ 

192 A customised model 

193 

194 with a 1x1 Conv, and 4 Dense layers (64, 32, 16, window_lead_time), where the last layer is the 

195 output layer depending on the window_lead_time parameter. Dropout is used between the Convolution and the first 

196 Dense layer. 

197 """ 

198 

199 def __init__(self, input_shape: list, output_shape: list): 

200 """ 

201 Sets model and loss depending on the given arguments. 

202 

203 :param input_shape: list of input shapes (expect len=1 with shape=(window_hist, station, variables)) 

204 :param output_shape: list of output shapes (expect len=1 with shape=(window_forecast)) 

205 """ 

206 

207 assert len(input_shape) == 1 

208 assert len(output_shape) == 1 

209 super().__init__(input_shape[0], output_shape[0]) 

210 

211 # settings 

212 self.dropout_rate = 0.1 

213 self.regularizer = keras.regularizers.l2(0.1) 

214 self.activation = keras.layers.PReLU 

215 

216 # apply to model 

217 self.set_model() 

218 self.set_compile_options() 

219 self.set_custom_objects(loss=self.compile_options["loss"]) 

220 

221 def set_model(self): 

222 """ 

223 Build the model. 

224 """ 

225 

226 # add 1 to window_size to include current time step t0 

227 x_input = keras.layers.Input(shape=self._input_shape) 

228 x_in = keras.layers.Conv2D(32, (1, 1), padding='same', name='{}_Conv_1x1'.format("major"))(x_input) 

229 x_in = self.activation(name='{}_conv_act'.format("major"))(x_in) 

230 x_in = keras.layers.Flatten(name='{}'.format("major"))(x_in) 

231 x_in = keras.layers.Dropout(self.dropout_rate, name='{}_Dropout_1'.format("major"))(x_in) 

232 x_in = keras.layers.Dense(64, name='{}_Dense_64'.format("major"))(x_in) 

233 x_in = self.activation()(x_in) 

234 out_minor_1 = keras.layers.Dense(self._output_shape, name='{}_Dense'.format("minor_1"))(x_in) 

235 out_minor_1 = self.activation(name="minor_1")(out_minor_1) 

236 x_in = keras.layers.Dense(32, name='{}_Dense_32'.format("major"))(x_in) 

237 x_in = self.activation()(x_in) 

238 out_minor_2 = keras.layers.Dense(self._output_shape, name='{}_Dense'.format("minor_2"))(x_in) 

239 out_minor_2 = self.activation(name="minor_2")(out_minor_2) 

240 x_in = keras.layers.Dense(16, name='{}_Dense_16'.format("major"))(x_in) 

241 x_in = self.activation()(x_in) 

242 x_in = keras.layers.Dense(self._output_shape, name='{}_Dense'.format("major"))(x_in) 

243 out_main = self.activation(name="main")(x_in) 

244 self.model = keras.Model(inputs=x_input, outputs=[out_minor_1, out_minor_2, out_main]) 

245 

246 def set_compile_options(self): 

247 self.initial_lr = 1e-2 

248 self.optimizer = keras.optimizers.SGD(lr=self.initial_lr, momentum=0.9) 

249 self.lr_decay = mlair.model_modules.keras_extensions.LearningRateDecay(base_lr=self.initial_lr, drop=.94, 

250 epochs_drop=10) 

251 self.compile_options = {"loss": [keras.losses.mean_absolute_error] + [keras.losses.mean_squared_error] + [ 

252 keras.losses.mean_squared_error], "metrics": ["mse", "mae"]} 

253 

254 

255class MyTowerModel(AbstractModelClass): 

256 

257 def __init__(self, input_shape: list, output_shape: list): 

258 """ 

259 Sets model and loss depending on the given arguments. 

260 

261 :param input_shape: list of input shapes (expect len=1 with shape=(window_hist, station, variables)) 

262 :param output_shape: list of output shapes (expect len=1 with shape=(window_forecast)) 

263 """ 

264 

265 assert len(input_shape) == 1 

266 assert len(output_shape) == 1 

267 super().__init__(input_shape[0], output_shape[0]) 

268 

269 # settings 

270 self.dropout_rate = 1e-2 

271 self.regularizer = keras.regularizers.l2(0.1) 

272 self.initial_lr = 1e-2 

273 self.lr_decay = mlair.model_modules.keras_extensions.LearningRateDecay(base_lr=self.initial_lr, drop=.94, 

274 epochs_drop=10) 

275 self.activation = keras.layers.PReLU 

276 

277 # apply to model 

278 self.set_model() 

279 self.set_compile_options() 

280 self.set_custom_objects(loss=self.compile_options["loss"]) 

281 

282 def set_model(self): 

283 """ 

284 Build the model. 

285 """ 

286 activation = self.activation 

287 conv_settings_dict1 = { 

288 'tower_1': {'reduction_filter': 8, 'tower_filter': 8 * 2, 'tower_kernel': (3, 1), 'activation': activation}, 

289 'tower_2': {'reduction_filter': 8, 'tower_filter': 8 * 2, 'tower_kernel': (5, 1), 'activation': activation}, 

290 'tower_3': {'reduction_filter': 8, 'tower_filter': 8 * 2, 'tower_kernel': (1, 1), 'activation': activation}, 

291 } 

292 

293 pool_settings_dict1 = {'pool_kernel': (3, 1), 'tower_filter': 8 * 2, 'activation': activation} 

294 

295 conv_settings_dict2 = { 

296 'tower_1': {'reduction_filter': 8 * 2, 'tower_filter': 16 * 2 * 2, 'tower_kernel': (3, 1), 

297 'activation': activation}, 

298 'tower_2': {'reduction_filter': 8 * 2, 'tower_filter': 16 * 2 * 2, 'tower_kernel': (5, 1), 

299 'activation': activation}, 

300 'tower_3': {'reduction_filter': 8 * 2, 'tower_filter': 16 * 2 * 2, 'tower_kernel': (1, 1), 

301 'activation': activation}, 

302 } 

303 pool_settings_dict2 = {'pool_kernel': (3, 1), 'tower_filter': 16, 'activation': activation} 

304 

305 conv_settings_dict3 = {'tower_1': {'reduction_filter': 16 * 4, 'tower_filter': 32 * 2, 'tower_kernel': (3, 1), 

306 'activation': activation}, 

307 'tower_2': {'reduction_filter': 16 * 4, 'tower_filter': 32 * 2, 'tower_kernel': (5, 1), 

308 'activation': activation}, 

309 'tower_3': {'reduction_filter': 16 * 4, 'tower_filter': 32 * 2, 'tower_kernel': (1, 1), 

310 'activation': activation}, 

311 } 

312 

313 pool_settings_dict3 = {'pool_kernel': (3, 1), 'tower_filter': 32, 'activation': activation} 

314 

315 ########################################## 

316 inception_model = InceptionModelBase() 

317 

318 X_input = keras.layers.Input(shape=self._input_shape) 

319 

320 X_in = inception_model.inception_block(X_input, conv_settings_dict1, pool_settings_dict1, 

321 regularizer=self.regularizer, 

322 batch_normalisation=True) 

323 

324 X_in = keras.layers.Dropout(self.dropout_rate)(X_in) 

325 

326 X_in = inception_model.inception_block(X_in, conv_settings_dict2, pool_settings_dict2, 

327 regularizer=self.regularizer, 

328 batch_normalisation=True) 

329 

330 X_in = keras.layers.Dropout(self.dropout_rate)(X_in) 

331 

332 X_in = inception_model.inception_block(X_in, conv_settings_dict3, pool_settings_dict3, 

333 regularizer=self.regularizer, 

334 batch_normalisation=True) 

335 ############################################# 

336 

337 # out_main = flatten_tail(X_in, 'Main', activation=activation, bound_weight=True, dropout_rate=self.dropout_rate, 

338 # reduction_filter=64, inner_neurons=64, output_neurons=self.window_lead_time) 

339 

340 out_main = flatten_tail(X_in, inner_neurons=64, activation=activation, output_neurons=self._output_shape, 

341 output_activation='linear', reduction_filter=64, 

342 name='Main', bound_weight=True, dropout_rate=self.dropout_rate, 

343 kernel_regularizer=self.regularizer 

344 ) 

345 

346 self.model = keras.Model(inputs=X_input, outputs=[out_main]) 

347 

348 def set_compile_options(self): 

349 self.optimizer = keras.optimizers.Adam(lr=self.initial_lr) 

350 self.compile_options = {"loss": [keras.losses.mean_squared_error], "metrics": ["mse"]} 

351 

352 

353class IntelliO3_ts_architecture(AbstractModelClass): 

354 

355 def __init__(self, input_shape: list, output_shape: list): 

356 """ 

357 Sets model and loss depending on the given arguments. 

358 

359 :param input_shape: list of input shapes (expect len=1 with shape=(window_hist, station, variables)) 

360 :param output_shape: list of output shapes (expect len=1 with shape=(window_forecast)) 

361 """ 

362 

363 assert len(input_shape) == 1 

364 assert len(output_shape) == 1 

365 super().__init__(input_shape[0], output_shape[0]) 

366 

367 from mlair.model_modules.keras_extensions import LearningRateDecay 

368 

369 # settings 

370 self.dropout_rate = .35 

371 self.regularizer = keras.regularizers.l2(0.01) 

372 self.initial_lr = 1e-4 

373 self.lr_decay = LearningRateDecay(base_lr=self.initial_lr, drop=.94, epochs_drop=10) 

374 self.activation = keras.layers.ELU 

375 self.padding = "SymPad2D" 

376 

377 # apply to model 

378 self.set_model() 

379 self.set_compile_options() 

380 self.set_custom_objects(loss=self.compile_options["loss"], 

381 SymmetricPadding2D=SymmetricPadding2D, 

382 LearningRateDecay=LearningRateDecay) 

383 

384 def set_model(self): 

385 """ 

386 Build the model. 

387 

388 :param activation: activation function 

389 :param window_history_size: number of historical time steps included in the input data 

390 :param channels: number of variables used in input data 

391 :param dropout_rate: dropout rate used in the model [0, 1) 

392 :param window_lead_time: number of time steps to forecast in the output layer 

393 :return: built keras model 

394 """ 

395 activation = self.activation 

396 first_kernel = (3, 1) 

397 first_filters = 16 

398 

399 conv_settings_dict1 = { 

400 'tower_1': {'reduction_filter': 8, 'tower_filter': 16 * 2, 'tower_kernel': (3, 1), 

401 'activation': activation}, 

402 'tower_2': {'reduction_filter': 8, 'tower_filter': 16 * 2, 'tower_kernel': (5, 1), 

403 'activation': activation}, 

404 'tower_3': {'reduction_filter': 8, 'tower_filter': 16 * 2, 'tower_kernel': (1, 1), 

405 'activation': activation} 

406 } 

407 pool_settings_dict1 = {'pool_kernel': (3, 1), 'tower_filter': 16, 'activation': activation} 

408 

409 conv_settings_dict2 = { 

410 'tower_1': {'reduction_filter': 64, 'tower_filter': 32 * 2, 'tower_kernel': (3, 1), 

411 'activation': activation}, 

412 'tower_2': {'reduction_filter': 64, 'tower_filter': 32 * 2, 'tower_kernel': (5, 1), 

413 'activation': activation}, 

414 'tower_3': {'reduction_filter': 64, 'tower_filter': 32 * 2, 'tower_kernel': (1, 1), 

415 'activation': activation} 

416 } 

417 pool_settings_dict2 = {'pool_kernel': (3, 1), 'tower_filter': 32, 'activation': activation} 

418 

419 ########################################## 

420 inception_model = InceptionModelBase() 

421 

422 X_input = keras.layers.Input(shape=self._input_shape) 

423 

424 pad_size = PadUtils.get_padding_for_same(first_kernel) 

425 X_in = Padding2D("SymPad2D")(padding=pad_size, name="SymPad")(X_input) 

426 X_in = keras.layers.Conv2D(filters=first_filters, 

427 kernel_size=first_kernel, 

428 kernel_regularizer=self.regularizer, 

429 name="First_conv_{}x{}".format(first_kernel[0], first_kernel[1]))(X_in) 

430 X_in = self.activation(name='FirstAct')(X_in) 

431 

432 X_in = inception_model.inception_block(X_in, conv_settings_dict1, pool_settings_dict1, 

433 regularizer=self.regularizer, 

434 batch_normalisation=True, 

435 padding=self.padding) 

436 

437 out_minor1 = flatten_tail(X_in, inner_neurons=64, activation=activation, output_neurons=self._output_shape, 

438 output_activation='linear', reduction_filter=32 * 2, 

439 name='minor_1', bound_weight=False, dropout_rate=self.dropout_rate, 

440 kernel_regularizer=self.regularizer 

441 ) 

442 

443 X_in = keras.layers.Dropout(self.dropout_rate)(X_in) 

444 

445 X_in = inception_model.inception_block(X_in, conv_settings_dict2, pool_settings_dict2, 

446 regularizer=self.regularizer, 

447 batch_normalisation=True, padding=self.padding) 

448 

449 ############################################# 

450 

451 out_main = flatten_tail(X_in, inner_neurons=64 * 2, activation=activation, output_neurons=self._output_shape, 

452 output_activation='linear', reduction_filter=64 * 2, 

453 name='Main', bound_weight=False, dropout_rate=self.dropout_rate, 

454 kernel_regularizer=self.regularizer 

455 ) 

456 

457 self.model = keras.Model(inputs=X_input, outputs=[out_minor1, out_main]) 

458 

459 def set_compile_options(self): 

460 self.compile_options = {"optimizer": keras.optimizers.Adam(lr=self.initial_lr, amsgrad=True), 

461 "loss": [l_p_loss(4), keras.losses.mean_squared_error], 

462 "metrics": ['mse'], 

463 "loss_weights": [.01, .99] 

464 }