DL之DNN：自定义MultiLayerNet【6*100+ReLU,SGD】对MNIST数据集训练进而比较【多个超参数组合最优化】性能
DL之DNN：自定义MultiLayerNet【6*100+ReLU,SGD】对MNIST数据集训练进而比较【多个超参数组合最优化】性能
输出结果
val_acc:0.14 | lr:4.370890470178883e-06, weight_decay:1.3601071813862507e-08
val_acc:0.09 | lr:0.00014631102405898786, weight_decay:1.1349746520024042e-06
val_acc:0.03 | lr:2.010303146711775e-06, weight_decay:2.3839639499335248e-05
val_acc:0.57 | lr:0.004194705876330481, weight_decay:7.275840690546594e-08
val_acc:0.16 | lr:0.00038584771807663687, weight_decay:1.564048542830317e-08
val_acc:0.11 | lr:1.3228406839207872e-05, weight_decay:1.6377230486052654e-08
val_acc:0.74 | lr:0.009429984488065873, weight_decay:1.9935852692855053e-05
val_acc:0.77 | lr:0.008214361491759711, weight_decay:1.1407085706746392e-06
val_acc:0.06 | lr:2.1752807956494228e-05, weight_decay:6.977690310157555e-05
val_acc:0.09 | lr:0.00019133856821239577, weight_decay:2.976345671479303e-05
val_acc:0.12 | lr:3.016900313611944e-06, weight_decay:2.2027187591768506e-05
val_acc:0.08 | lr:1.0214432461948464e-06, weight_decay:4.649923123908956e-05
val_acc:0.09 | lr:9.767546641051179e-06, weight_decay:1.0359511501077427e-06
val_acc:0.16 | lr:0.0015355393894511675, weight_decay:2.7097408814814655e-08
val_acc:0.08 | lr:5.4342212986763476e-05, weight_decay:5.23859951921203e-08
val_acc:0.65 | lr:0.006014462135048698, weight_decay:3.9418873067436714e-05
val_acc:0.1 | lr:1.3996453691094716e-05, weight_decay:1.3409964033073044e-05
val_acc:0.18 | lr:0.0007649457731714201, weight_decay:5.015510823338895e-08
val_acc:0.7 | lr:0.008168506325658352, weight_decay:1.0658855818651342e-06
val_acc:0.1 | lr:5.614195975029914e-06, weight_decay:5.559021054670591e-07
val_acc:0.35 | lr:0.001299225969580492, weight_decay:1.0346743582021148e-08
val_acc:0.64 | lr:0.006086420186505023, weight_decay:2.159424730589368e-07
val_acc:0.19 | lr:0.0004990433329386827, weight_decay:2.2090856059786305e-06
val_acc:0.05 | lr:3.5739139537042676e-06, weight_decay:2.561460621807675e-06
val_acc:0.08 | lr:7.448534945424167e-05, weight_decay:1.472970864043896e-06
val_acc:0.26 | lr:0.0007897244496348546, weight_decay:1.6718419245733562e-06
val_acc:0.58 | lr:0.004612375959216581, weight_decay:2.3591718261430373e-08
val_acc:0.08 | lr:6.619302392889255e-05, weight_decay:1.3233299986991992e-05
val_acc:0.54 | lr:0.005887131600710247, weight_decay:1.6028359838515335e-05
val_acc:0.11 | lr:8.419431520071244e-06, weight_decay:1.6978835122739706e-08
val_acc:0.14 | lr:1.6223362387224087e-05, weight_decay:2.653683122557726e-06
val_acc:0.08 | lr:4.62163920927252e-06, weight_decay:4.216528178334854e-05
val_acc:0.18 | lr:0.0006777392606545035, weight_decay:8.407616649296984e-05
val_acc:0.09 | lr:2.205707304320701e-06, weight_decay:1.99590931015699e-07
val_acc:0.15 | lr:4.7148321283130434e-05, weight_decay:1.1732299915050255e-05
val_acc:0.12 | lr:0.00025581006916690986, weight_decay:4.305256448920019e-06
val_acc:0.07 | lr:5.194264564497702e-05, weight_decay:8.596731993140124e-08
val_acc:0.07 | lr:1.1256846967789416e-05, weight_decay:4.432904277423879e-07
val_acc:0.09 | lr:7.448991735220237e-06, weight_decay:8.373785215977307e-07
val_acc:0.36 | lr:0.00251398773518266, weight_decay:1.4031617168449831e-06
val_acc:0.21 | lr:0.00010284549477549629, weight_decay:8.113121916297787e-05
val_acc:0.19 | lr:0.0004961619619840043, weight_decay:5.883256859922363e-06
val_acc:0.05 | lr:8.163246476050396e-05, weight_decay:4.498986377409097e-07
val_acc:0.2 | lr:0.0007080318500965934, weight_decay:8.846619605809187e-06
val_acc:0.11 | lr:3.30784766264413e-05, weight_decay:3.3826389977440746e-08
val_acc:0.14 | lr:8.844997695528248e-06, weight_decay:6.2687740564102e-07
val_acc:0.7 | lr:0.007358969112499609, weight_decay:1.3510111118037593e-06
val_acc:0.06 | lr:1.1433587684467216e-05, weight_decay:4.44478652661818e-07
val_acc:0.06 | lr:1.0108656834025462e-05, weight_decay:4.982986041003918e-06
val_acc:0.07 | lr:0.00011781337491423188, weight_decay:6.173214313495265e-07
val_acc:0.11 | lr:1.1073006300871238e-05, weight_decay:1.3060132953611249e-05
val_acc:0.14 | lr:2.473188605569667e-06, weight_decay:1.214266085210085e-07
val_acc:0.14 | lr:2.026269549322074e-05, weight_decay:2.776062411345026e-07
val_acc:0.18 | lr:1.272934437738808e-05, weight_decay:2.642825687963657e-05
val_acc:0.58 | lr:0.0033951762862381306, weight_decay:7.973957963920187e-08
val_acc:0.12 | lr:1.0664639723330058e-06, weight_decay:3.9316812476327313e-07
val_acc:0.11 | lr:3.3615262816564584e-06, weight_decay:1.0653765745678783e-05
val_acc:0.09 | lr:2.1638868431682165e-06, weight_decay:1.4864784633618121e-08
val_acc:0.7 | lr:0.0074464975100202585, weight_decay:3.122919333714384e-08
val_acc:0.22 | lr:0.000745833941756555, weight_decay:1.4366994559218038e-07
val_acc:0.39 | lr:0.002301183444562202, weight_decay:1.2864600979854382e-05
val_acc:0.1 | lr:2.5610635228178507e-05, weight_decay:2.1476760582230525e-07
val_acc:0.12 | lr:0.0004690505499031566, weight_decay:2.521807169874246e-06
val_acc:0.12 | lr:1.145363854399871e-06, weight_decay:3.807788578798886e-08
val_acc:0.1 | lr:5.728982199161293e-06, weight_decay:5.3993056547633026e-06
val_acc:0.11 | lr:9.28017641045176e-06, weight_decay:5.182821461162384e-07
val_acc:0.18 | lr:1.6072016105181504e-05, weight_decay:3.505635913560728e-05
val_acc:0.23 | lr:0.0013079631640391708, weight_decay:8.178372416835232e-07
val_acc:0.11 | lr:1.303555183537829e-06, weight_decay:7.241478675103921e-08
val_acc:0.19 | lr:0.0007753891771349916, weight_decay:4.174489463838404e-06
val_acc:0.08 | lr:5.271622365349557e-05, weight_decay:8.491182848529751e-05
val_acc:0.08 | lr:0.00014039062201923758, weight_decay:6.366073730895015e-07
val_acc:0.09 | lr:3.925888248811256e-06, weight_decay:4.131097819019027e-06
val_acc:0.11 | lr:0.0003914236447760262, weight_decay:1.2409606802589577e-08
val_acc:0.12 | lr:0.00024812485239091074, weight_decay:1.3343145720947007e-05
val_acc:0.11 | lr:1.0581942665909421e-06, weight_decay:2.4747970863380575e-08
val_acc:0.11 | lr:1.056576970136167e-06, weight_decay:4.949021089487696e-08
val_acc:0.26 | lr:0.0007207554378117602, weight_decay:8.860239826440169e-05
val_acc:0.29 | lr:0.0014882676876609164, weight_decay:9.203022616531917e-07
val_acc:0.47 | lr:0.0030166755193361586, weight_decay:1.0451914730874759e-08
val_acc:0.14 | lr:0.00024298690911699048, weight_decay:5.485462689913924e-06
val_acc:0.14 | lr:6.839952333174794e-06, weight_decay:7.870450187037055e-05
val_acc:0.07 | lr:0.000180976217729757, weight_decay:2.7515827866498776e-05
val_acc:0.41 | lr:0.0030021728745674658, weight_decay:2.468045607486459e-05
val_acc:0.1 | lr:4.8746279465277163e-05, weight_decay:1.044127698285955e-06
val_acc:0.09 | lr:4.449907824680105e-05, weight_decay:7.328779436667631e-08
val_acc:0.18 | lr:8.976903743901344e-05, weight_decay:7.334032597679438e-06
val_acc:0.12 | lr:4.653201426561944e-05, weight_decay:1.1421969224324098e-06
val_acc:0.16 | lr:0.0007233551163996372, weight_decay:2.42088254480196e-08
val_acc:0.13 | lr:4.121193446313957e-06, weight_decay:8.909554775787621e-06
val_acc:0.07 | lr:2.350402404330748e-06, weight_decay:1.4779315408050775e-05
val_acc:0.07 | lr:0.00020349921247596708, weight_decay:2.9820461267728694e-05
val_acc:0.43 | lr:0.0016163207456264563, weight_decay:9.505666196332687e-07
val_acc:0.07 | lr:3.0170517986137628e-06, weight_decay:7.370756010313667e-06
val_acc:0.13 | lr:2.5145108621558854e-06, weight_decay:9.32932311979866e-06
val_acc:0.55 | lr:0.003856533085564158, weight_decay:8.439817906586993e-06
val_acc:0.12 | lr:4.463330713338443e-06, weight_decay:3.6825946020691855e-08
val_acc:0.13 | lr:5.930576454986149e-06, weight_decay:5.5823171433583716e-05
val_acc:0.1 | lr:1.3703760858297245e-05, weight_decay:4.0877300200370504e-07
val_acc:0.17 | lr:0.00011614339077294565, weight_decay:3.499743193885368e-05
=========== Hyper-Parameter Optimization Result ===========
Best-1(val acc:0.77) | lr:0.008214361491759711, weight decay:1.1407085706746392e-06
Best-2(val acc:0.74) | lr:0.009429984488065873, weight decay:1.9935852692855053e-05
Best-3(val acc:0.7) | lr:0.008168506325658352, weight decay:1.0658855818651342e-06
Best-4(val acc:0.7) | lr:0.007358969112499609, weight decay:1.3510111118037593e-06
Best-5(val acc:0.7) | lr:0.0074464975100202585, weight decay:3.122919333714384e-08
Best-6(val acc:0.65) | lr:0.006014462135048698, weight decay:3.9418873067436714e-05
Best-7(val acc:0.64) | lr:0.006086420186505023, weight decay:2.159424730589368e-07
Best-8(val acc:0.58) | lr:0.004612375959216581, weight decay:2.3591718261430373e-08
Best-9(val acc:0.58) | lr:0.0033951762862381306, weight decay:7.973957963920187e-08
Best-10(val acc:0.57) | lr:0.004194705876330481, weight decay:7.275840690546594e-08
Best-11(val acc:0.55) | lr:0.003856533085564158, weight decay:8.439817906586993e-06
Best-12(val acc:0.54) | lr:0.005887131600710247, weight decay:1.6028359838515335e-05
Best-13(val acc:0.47) | lr:0.0030166755193361586, weight decay:1.0451914730874759e-08
Best-14(val acc:0.43) | lr:0.0016163207456264563, weight decay:9.505666196332687e-07
Best-15(val acc:0.41) | lr:0.0030021728745674658, weight decay:2.468045607486459e-05
Best-16(val acc:0.39) | lr:0.002301183444562202, weight decay:1.2864600979854382e-05
Best-17(val acc:0.36) | lr:0.00251398773518266, weight decay:1.4031617168449831e-06
Best-18(val acc:0.35) | lr:0.001299225969580492, weight decay:1.0346743582021148e-08
Best-19(val acc:0.29) | lr:0.0014882676876609164, weight decay:9.203022616531917e-07
Best-20(val acc:0.26) | lr:0.0007897244496348546, weight decay:1.6718419245733562e-06
设计思路
190417更新
核心代码
'超参数的随机采样的代码如下所示'
weight_decay = 10 ** np.random.uniform(-8, -4) #权值衰减系数的初始范围为[10−8，10−4]，
lr = 10 ** np.random.uniform(-6, -2) #学习率的初始范围为[10−6，10−2]
x_train, t_train = shuffle_dataset(x_train, t_train)
x_val = x_train[:validation_num]
t_val = t_train[:validation_num]
x_train = x_train[validation_num:]
t_train = t_train[validation_num:]
def __train(lr, weight_decay, epocs=50):
network = MultiLayerNet(input_size=784, hidden_size_list=[100, 100, 100, 100, 100, 100],
output_size=10, weight_decay_lambda=weight_decay)
trainer = Trainer(network, x_train, t_train, x_val, t_val,
epochs=epocs, mini_batch_size=100,
optimizer='sgd', optimizer_param={'lr': lr}, verbose=False)
trainer.train()
return trainer.test_acc_list, trainer.train_acc_list
optimization_trial = 100
results_val = {}
results_train = {}
for _ in range(optimization_trial):
weight_decay = 10 ** np.random.uniform(-8, -4)
lr = 10 ** np.random.uniform(-6, -2)
val_acc_list, train_acc_list = __train(lr, weight_decay)
key = "lr:" + str(lr) + ", weight decay:" + str(weight_decay)
results_val[key] = val_acc_list
results_train[key] = train_acc_list相关文章
CSDN：2019.04.09起
赞 (0)