From b234f7edfdf44c91c288ca80fbcc8b6d62448222 Mon Sep 17 00:00:00 2001 From: zhujian <505169307@qq.com> Date: Fri, 1 May 2020 21:16:11 +0800 Subject: [PATCH 1/4] refactor(userwarning): 1: scheduler.step(epoch) -> scheduler.step(); 2: get_lr() -> get_last_lr() --- warmup_scheduler/run.py | 2 +- warmup_scheduler/scheduler.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/warmup_scheduler/run.py b/warmup_scheduler/run.py index a1dbf3d..d20d3e1 100644 --- a/warmup_scheduler/run.py +++ b/warmup_scheduler/run.py @@ -18,7 +18,7 @@ optim.step() for epoch in range(1, 20): - scheduler_warmup.step(epoch) + scheduler_warmup.step() print(epoch, optim.param_groups[0]['lr']) optim.step() # backward pass (update network) diff --git a/warmup_scheduler/scheduler.py b/warmup_scheduler/scheduler.py index 8b84082..2ebdc70 100644 --- a/warmup_scheduler/scheduler.py +++ b/warmup_scheduler/scheduler.py @@ -28,7 +28,7 @@ def get_lr(self): if not self.finished: self.after_scheduler.base_lrs = [base_lr * self.multiplier for base_lr in self.base_lrs] self.finished = True - return self.after_scheduler.get_lr() + return self.after_scheduler.get_last_lr() return [base_lr * self.multiplier for base_lr in self.base_lrs] if self.multiplier == 1.0: From f4b25f07291d280f90b0f8df7d4eac3f8ba04331 Mon Sep 17 00:00:00 2001 From: zhujian <505169307@qq.com> Date: Fri, 1 May 2020 21:26:02 +0800 Subject: [PATCH 2/4] perf(run): print and plot lr_list --- warmup_scheduler/run.py | 36 ++++++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/warmup_scheduler/run.py b/warmup_scheduler/run.py index d20d3e1..5e1a384 100644 --- a/warmup_scheduler/run.py +++ b/warmup_scheduler/run.py @@ -1,24 +1,44 @@ import torch -from torch.optim.lr_scheduler import StepLR, ExponentialLR +from torch.optim.lr_scheduler import CosineAnnealingLR from torch.optim.sgd import SGD from warmup_scheduler import GradualWarmupScheduler +def plot(lr_list): + import matplotlib.pyplot as plt + from matplotlib.pyplot import MultipleLocator + + f = plt.figure() + + x_major_locator = MultipleLocator(1) + ax = plt.gca() + ax.xaxis.set_major_locator(x_major_locator) + + x = range(1, len(lr_list) + 1) + plt.plot(x, lr_list) + plt.show() + + if __name__ == '__main__': model = [torch.nn.Parameter(torch.randn(2, 2, requires_grad=True))] optim = SGD(model, 0.1) - # scheduler_warmup is chained with schduler_steplr - scheduler_steplr = StepLR(optim, step_size=10, gamma=0.1) - scheduler_warmup = GradualWarmupScheduler(optim, multiplier=1, total_epoch=5, after_scheduler=scheduler_steplr) + epochs = 20 + # scheduler_warmup is chained with lr_schduler + lr_schduler = CosineAnnealingLR(optim, T_max=epochs - 5, eta_min=0.02) + scheduler_warmup = GradualWarmupScheduler(optim, multiplier=1, total_epoch=5, after_scheduler=lr_schduler) # this zero gradient update is needed to avoid a warning message, issue #8. optim.zero_grad() - optim.step() - for epoch in range(1, 20): + lr_list = list() + for epoch in range(epochs): + optim.step() scheduler_warmup.step() - print(epoch, optim.param_groups[0]['lr']) - optim.step() # backward pass (update network) + current_lr = optim.param_groups[0]['lr'] + print(epoch + 1, current_lr) + lr_list.append(current_lr) + + plot(lr_list) From e7c60e0fd33d652233fc9401a8a00b803916f349 Mon Sep 17 00:00:00 2001 From: zhujian <505169307@qq.com> Date: Fri, 1 May 2020 21:30:48 +0800 Subject: [PATCH 3/4] docs(readme): update sample picture --- README.md | 32 ++----------------------------- asset/warmup_cosineannealing.png | Bin 0 -> 22507 bytes 2 files changed, 2 insertions(+), 30 deletions(-) create mode 100644 asset/warmup_cosineannealing.png diff --git a/README.md b/README.md index 72a5fac..f211699 100644 --- a/README.md +++ b/README.md @@ -2,9 +2,6 @@ Gradually warm-up(increasing) learning rate for pytorch's optimizer. Proposed in 'Accurate, Large Minibatch SGD: Training ImageNet in 1 Hour'. -example tensorboard -Example : Gradual Warmup for 100 epoch, after that, use cosine-annealing. - ## Install ``` @@ -13,31 +10,6 @@ $ pip install git+https://github.com/ildoonet/pytorch-gradual-warmup-lr.git ## Usage -See [run.py](warmup_scheduler/run.py) file. - -```python -import torch -from torch.optim.lr_scheduler import StepLR, ExponentialLR -from torch.optim.sgd import SGD - -from warmup_scheduler import GradualWarmupScheduler - - -if __name__ == '__main__': - model = [torch.nn.Parameter(torch.randn(2, 2, requires_grad=True))] - optim = SGD(model, 0.1) +Example : Gradual Warmup for 5 epoch, after that, use cosine-annealing. See [run.py](warmup_scheduler/run.py) file. - # scheduler_warmup is chained with schduler_steplr - scheduler_steplr = StepLR(optim, step_size=10, gamma=0.1) - scheduler_warmup = GradualWarmupScheduler(optim, multiplier=1, total_epoch=5, after_scheduler=scheduler_steplr) - - # this zero gradient update is needed to avoid a warning message, issue #8. - optim.zero_grad() - optim.step() - - for epoch in range(1, 20): - scheduler_warmup.step(epoch) - print(epoch, optim.param_groups[0]['lr']) - - optim.step() # backward pass (update network) -``` +![](./asset/warmup_cosineannealing.png) \ No newline at end of file diff --git a/asset/warmup_cosineannealing.png b/asset/warmup_cosineannealing.png new file mode 100644 index 0000000000000000000000000000000000000000..9b462e9b8fd7c2f031bc522f571ce2877cc5818e GIT binary patch literal 22507 zcmbrmcRXI*wl=&FM2jT41Zkr8h+cw3^xlbE1ks7!C4wMGbV(4sM2p^QkfQfaNc8R| zx^J%K*?WKcoPGA$@B99hpIGLaYtAvpxW+ZEF@uy9rHSyV@gWEzl6fel3PG5@5QH&( z9uxdeRk2(#1Zfq?NZnI+eY7%xr%z&bBDPV6?D+$%t*&+lHDWLY$_a%PEbuSN^*joKe6RaLIN(1q z|DS&_z1V>7pHfp(^ZD~PZJWER$w{D z2_{dwkwJz@2|?n-u6_6zgb?%*x5nHb{-AEG$r1?&eCGnje>6;V7)FhG-)#>Mk1JQM zP*B9ae0i>Hye~2#;dtp&!RW|H&nKOFZxJ<`jm3AaTJY!Xx%SD%LKpDyi;9Z2c6K^C zIt&U?&-3!Eb#$nBlyr4RGn2flJpQazuh+fDLs=*02f0LpslN{A%X7q={uas5l^~?` z!SUI%XD%*7S%!g%FXEv&cX#4bX7I_Bbhf(oxOQe{^Myvu5|WZF(sNx&VkX~7Lv`|- z&lScf#eRFDpoJ%S)v{g|buONQy=y9AGx?Os>*U;rE&MJbSr#E3)v4~$Zzquul+iJb zp|7HH?^Ik>HDd2oY;3HbpIN)1_?RoDE-6q34syq)+Mu z7PqH)^kxZ+pM8?G(5wE9zv~1+afCOq?Vdd=+Iy9mnVFiJ3PxGv^~LoR?DSR8QIj6R z3Ta}x>GO(ZJYGMgAc>A7M8x}>_i|f*TfL~eF>PPhLd(yt{Ln*`*1^)(8n4CQnH>5V zT1|EQzziR6K!aO6Pu|+l-LsrApyw=UHsesaD!QYEqst+)okLN4B}}BU&h`)#(Dq()9Ai9n?LM#Vq_BPO z$Vg?1pgrS4wyKLM>PLVqnUs zwIpWCUH?BpGw$h^-3XypV1HHT3hij#CM6l#*sL}A6IfeY%gMvW zRn8|Wi5*9sFfuYOoGyC(1mD8=y|}H$m~sAbmqsyfs^!wc_ZmBaU+Vi{5s01B#md=j z+8^&q?f+g6mwsXpF zk0s`uM_$hBO6vsc%+K%r*5Xym5nv5kScoX*CFy3IPzEMQk7J4e?Gl&qH7%GPBy~t#O!}xx*pKxLslxtU67&E+KCHw^T7_qugsY~R++zJ z@8_sj4V%1ltnjr`dHx@nNnCO_#w9yR2jdJ~cp%El$SkZo_L)K-o2#uv3%%1}u9R@w zv--{QK8%3NYMplXBtLR7ooK}qf`(>cZ}#;5AR9#;V@Lj*FHSD+l~kND>K=8@JHfiJuUE=hdlkLtRwfv}r#WB4gs^6i7yEY}F?x7^%S92Wst(c8oXvoM$+$eGx*Zkwo_bAJ6z-6E<;lA>jIvy z%2bQH@f;t{THVX(mAmpXnWmLBY&c&3MR+R4$3QXQ!8r?wkCjx0bA_HIvOOIBUDo-> zIgwN}>;$FdZY(4khVw=cUg2h_>UXf1Tb3YQ0&{a|;ZQQk)M{IZ|Ak(qy{jD!r^LP` zo3PKvdGkouHnUDB^=fRx?wtJoZ?OVF)66=V`Lewmo>z&k=^(Sh$PE>gUffb?U@C*O zc%2-*C_f%VC3;VeybZ{=fO@~cRz=zH;|d9r)^1`=1GzX>!rnn|m*tEt|;r3caCDjFW8 zx-*)@l1a~W4i%a}g`{Vr)t1r6Kc1bBQZ0i%1Y784SUGLhS{!Mqpihrt zc|S*oenmmWAR2(3@wb(@gxx&m%`l9)FyMbs#^4#&>$B58(8kW2qDSuQcDB(od30(_ zWFVl7U@qtZ_YwI?ok|#+iV!u29Kl==j<`ZJnwQ81OA2{8eM4f$S_oTz%RqA>@JBZY z#%L)RZM^JKNxxjqarg4~X<(ZVPFK-~H@GPqgc0-ArmcUXcA51;$72H2L&r5avj()7 zl36pDr~|U|sAH2*i--GJE3#z}j&t^=E>0}pG|m&qGyuc}!fA!~cSpv?#)gM0Dk^T> zLW}Ai)_EO%NJz-b%Ug7L`ZR0LRYPN7Wu$l=1S7-aC9VAY{QdoXYNaLYb>>D?)L^M9 z^jep6^2V~MCCW9bQEh>lv>T1h=4b%3x zpL27!6?@~?+T(c^!?)+V`j&rmc9PS^{`l=O>BeAE$v+R;+BS^s2?0v# zuhj!Ve`G`^v+Nxm697#<-=mnQi?d!+ay_1?a+R7Xy=jC@0SK+Ysz^zBaq>JZ?bhZc z4h{}nT-;C{_1xTCOiau#G6!R$qwh`@mX?Z&i#=9Lhp4Hk8GqED99Zg(Zg+|Ktj1e# z$LB8QY;8Tu*T77mC+k-Y5&OnNO>r&$g~j)2TXm|VI2S@-AZRKk-^j=UFX@HF#l?k% zEUplIY&#U{Bmd(1x~;MC&7*|GL{X!rbGqjfWU8yG6xB%PNnQVj;vDB_66HFM89aDK zFbOYxpUwn3o8=;JZs21;523zN3H##iUnD_8MkC$?bz1kUu0xt98*?iXCq@{rR!UMy!kp=>tSqc91yeuqHwHI=DXenox%jGf*c z#{=+7Lqo%qlknk#goMPGw)*4UF|nhc#OS@RZ*Ru38;p(!H0+bZFAznG-CsRCSlG70 zkjFCFa9GZNVt1+a8^kFBzQP!~LINJTfJ7@P)!9Uq!G^(xC;eq#6Ad#D5>Je}p1#(= zMp+;pgLyH1r$1Loo0eW{8uM19X6D;Q>;Yh|BuKna*16&oHi{0%;u*$_qy3Ibu0Oc- zgwvPlCmf!$YHGONQlqEQ&&r_N*>$CAU#EjunEc+4a6^RH2qam0{e4p8weI#(#)UGR zj9~6zYsnXH8uvB6qI{FjyqMf9DRy=hX;3N zTypYP;7KBjknLeD$6Y#llATlS=BJZN=q=+VFp@gSHPErpeF0?!hk=-qjIexMa(}f* zuVmWUT0Kl)-~GlEil-;8(y<@duzat>zE!*TLQ<_%=uY3%mm+LVy_V9v+4|s=awns1 z^YqJn$1Dq>O^x&5z7cQM0xg2EcH3@qfZ0rFYOg|-(Z{YiJ3H<=OCU}M@?aH0(=c# z6AJRlkA(sO{D}+8dI#d}m@s1)-71+`9u{b-&;tJ%H7Pq@mUZK02vO zq{mBhw{-YK=OIp3WGHn#vp;bym-_Ee45Gw!zV$~H5L|8 zNgD#e4@PF_e+`{aNGbn9R>TW+(;;I-wQ-E}-{3&vPLQ{3RWH-QLq#$@EJy@4wn8a9 zL!udKvLI#VXvf0)>w9$d!mowyW1S)8^sJKn=+POKkS4t+6IMj5>aqG<4$IMsJ*8|5 zq(mQmkY&5^lbr~_4pRbD54EmPX0GT1+vBMoCh7(VJ_xM$9lT{zHC-K8tA0IHyyvuC zVUnZRleE?$bt(iM!_cDeS9f;!c5GlaUMo>V8q==+4sa$V7*-)1k1Zk2^T<0sq`lC! z@MLg%FnjJ{5s1?MiapZX-AvTtw#YjKCVy7;?|B#TMY8!m>HZwcd2w*qYci_k@ zyo8N8Vzx>&QFqcWp3Dj(Ev;COYc}x7=;cIxr;@tTNH#1;3|T8V6Jy-c%0JbCydOKF zg-3BtYGenP4oc}Fn8LwGcF1f?RsifJS_7!Q82e4yQ4UK&iWsq|pC|_8hs1I7{q9rJ z(z4~$F7W#tV=bZkhqphf-D&uUoE(X&-}pwT53h@G%5W~uzDjz#1OGvSD;nKUJL>VEr^}m}9U9c%gp(OtjP0A4SVcZUA}Vc85dU1U2NFcKU0r$GqF~KZZKA(% z5bE#LxO=MN^NYi;a zPI>*Aa9_?-R8)M=XFKSs$^pWPO`0+zL%y&?o`tLmjUHA-qH6Y$V?mW#iS!dV!NlmNWm0;cx;r3O@8nO1 zOi72lf!E5jD}wj2p8Ee2iIY~W?c(ARTJ`-k70>wi_!k*1t)X}xGkl+L8X@PFV6w+S zoyj1uFe^^Ja*3XPFh$4mu@zmLz%YU-PG_O_B-|$t9QM|-D0YFcF$kv>p)CM{na}+n z5RAyPUk`t-6N~PldedYmI3B4pN4JI1RD(E1LPF9afc+1g@!|?ztSd-pim-j_em}B! zrs)#=sOpqe56kxorLZpxGxGx7Ls{9&bejN`*x7Z*-EaB+o%z}|@egVB080ATGc`?l zA6n~v-T2wO*l1Ag98?@6EG%qOmAL!q-0Q!bF^_w4U1%N#dih}|@CAXLs3Tcl3`!Wo zM7Br^i->?=%WJBV3|$~3BxN{0KDM^9dRae|t0Vc`Yh${R$E?$8yy9)AFJP0v)cLUn zQ{j{W>j2I7*$P9P;M2`*b(*4>`!NGQp$i|yaEPbxd7qn@oKzgSN=|+|X0APcc4_Hs zWqp4FLWjEhH?G4lrY|{&%UT08MUN)qUMYG)$AB?hgVt1Bn%?~b$K;y#Wdudq*w{!g zu>Rb+FD3OV{m+!oDZtDm47OjYWk1SftBcOjNaiu*FP=*|Fdyh zYilbx1;zUC=h;C$+tpK5)T9Qc0!Ur@Z3O?KZJ@mSrdquLQeu=v7e8_!Xz5Ru?H406 zGc#-J)tQ;gbaWiT55Eb*kTr378YL|)9c;nCkoo4##m&v{-@p4DFB|@u^x(M9(NXBU zI(o7$_yhhB%B3CadJ-!P-@YO&Gjl!nr!8-p#}%)B@%MdzTccE^o_+v(q_QX02@}TT(#jz^+J0B?U(j|3+|s8yvX&Rs!+IkO`%Jwe zW4D2RP0^nYC8xyu7$xN#wd0B+>XOWBEEnuVkSDkSQ}%w9LDJT*{J=yDnC-o%(N1FNlO=G8!48jY5Cm%rI>RaV)r@4w9jz8&fpw*pU-(na+dPbim(L#gkkp&yHGWm^ zD>;O|m#tn0aEZ0CnkJp)+QQ=>P%ko*AGelc2x3EisZXC05&nLrm&s|maWp}-$4!Y) z(lJcgA7kFN$RzeWC2#zw7}HPSXRs#V_8h!~5P3;hsSUPE zI>#BdG#3`c5xvfn^&rl;y=`9SrA_?iV)oCxkU7kf7h=Dg zPf}c+Mo?_S8^8jk64`!`{^fsgw+!bR;Jm@M-kErYUFzF^!8D4ukQb6%iePy(;`pJg zc5#X>vwnp8jQ%K07tm2vVw-;XTZQs7+3WC5uhTTqlkKptCYDePlI+LKz0Y#!Kk~X7 z*GLY%zHwtYV#CC4e4n}u2h{@yW-IeYqj3+d4ei_oz+7YK4~w7Pbkwm=tIGDv!_a0& ztRrfaJl<~Wc>r;MvUKdYQ|*#0ny)N<`qRZX7tVfvOhu7B@V@N&b2f-l>z-3cpX$KG zS3cSfGfRj(7>@1aCIIMJYMzrGtx?&? zO!1;*)9u`!SFgWdI%Iz^pww-?__OI9IoO|*v)^|D0=asw@K#EyT<0JpAE2M?X{?8D z{Rt4PPbP*n8+(z7Tn1n)<-scAPa3|c^EsU_4{3DuN4cF1GKp45C-306G^F!_>ecuB zw2pyE5yJAUa|FndDjpgfVH;n7ranEd+;gT_c z>R3w!Fdfs^T+f~{@a{wqOX{*7m*!<{OxTFtmq(0hS~NXi!*6pjM>zw=s4F)#mB=;G zN3c#Dg8&fSf;bx)3h!4|S3#`O*Vn&%`SNhl=zqa0dkTEoweF>@+uAjsZZQc(%4Hq9 zds%V=%f{9D9h%zzhf@qdCNL-1#mYgr*w{KcI{OC)b0F*0E^Gi}-i_)#lyRG26QRxs;TYhfNsWjo7w0sU*;0YBH*8Q`5K&Lg znDOzqn_H^JLf{=N$UBr0h27T1D`(O`=&i1p;pCo#` zn3-fsS4+FjEEAZkw^oAYFhH*bC!2R?K_qCw?LiH?_5}<13JD1<_G@n6 z{JpNt{x<%7f5R!R!8D)wMc8(9`9Xxf!1Vm~9*A$`zPEqnP=l;-UZ z>;O|2lRe8BSM)MC90>`vB5L@x3Da<_`T@V@=VJi}XZKSAaB#i7s_veHxjK(8Gtkrb z1QmscD?NHddpuY}m@R$GY+Jor#->;HDje>^oaUw@6T@?@W zdP|6Ei6yX6T$+!;+rS9{oD+j}`%?OO|hK$d1 zB|L{Z71T9Az$eeq%Xi&BoGlrtUwog_;J=P)LcYrslyri0Z=9+R)QVx;N-3QC?%OED zh2o6kM4VG(>ux@7|5TtsmaQ>}Y1VB67=9rZx?nPA@mwJ2Yn7CBH^Oc}a)6bY0{(W@ z=im<|C#5_oj2Jq;Q5<$EW0f4CF6cpTAp3^tQ0Z+DCGe0aF#>XXP}xvkCUL0?07ymh zDGm-HP9KuR72DV4#J{5jd0*{zTZlJh6I)SP>jPWHo>8v2n1vVxtz^E~4G#bx`k zaHRX{rv1SJeL4H}c@brVH~0M_Wk|!xPxi+TOASFf+uegBBBr001iwn?`(bqVZMzlb zNY0Rm!qY|yaUIP)!UNaEc=;;mMa0A18&IP(Pcz|=R))hVg{&NHov>ZZ=)0C=9grF1 zaDpYHMwTp)=~K9hBuO-=_CdGWc?hv^{P2*lO2wFMO_uNVWyk<}2R}}VJpMc>LN+VN z4RO|QYKke~wZ}jEjnDeS)Ljz?tLu7@?$EU@#(e8Q z4C=0DubgRFzidwDW+qK7`YPZ$lJl#@#u>`7{kYZjw1gG%5-=QK)n}~#^9vhzKUTy_ zoFv4$ly*&vu%Yfk;LmrX?0?+g;4LfWg;J3%%J#5H$cdb9Y{xHyAZn1vz(6d)BxVZF zQ~#Xg3m8${&YO8A#LEbi6Vu$%zRyY3%U|oH99(n%6T}VPgA|7Dn)in(TW)>Nn6<)y z+;0+6-*wh`;wE_o9N9BdyJNJ_x6?B}f(fA?O1$)`Jg_8gL;U3kAXS;PX8-;z;3oZ7ly(?xz2pb{$$He zheq@j&MdMMi@GFkVbpICiW5T1B`bICU5|IBI|wCCX?%)oZq>{8hv#650@P&_4tuFM zSMLdhQGhjRM%E-K$NH_x*gdgzh4<`&EcTmP+$by(gDc3&!g0& zq}Pb^_$BflAWQ>9+caTyHAV_7FF?k%fbyZia_!4d*xlW|d-pCoJ3Ag;t&e<_?UX3U z&p_llprNMD(fQud;kB3^>kU#dkUp=53vWHt(ee2cI%_~rV3%=JlyG1pnlOQ1h64#l z$^K@oCn=Fs032rdRUuN!i?+IpmmBWgyQitCSyxvVl}bR)T3A|I%XJR~^+2L4bE~Tf zFgG4ZOqclX9di^u0l|9Bc4wF9A?wYXsaaW!tgN|JRR;%VhJh~8tPL(p7_C!dZV_w{ zR4*60IxDPBCk0na!1n?;_U~#hRaI4;9|Y9)4tItOeU|gezx4XLYJI~z|EG6bsm03^ zLK}msWrz5aXr6!Q?y)26*%2N%}RsAdBbHMz(I8`AA28NJuS9fX^N?L!}@)H2(4#&)=_5Nt(1C9$_*>7Y$ zFk7E&K(xQ4yU}P+JuA?!RAu!#*`E&=JE`DzyE&=BZ7G|+b*X^T;wD^_<%KA(j@D0( zxq-5M$QAv^ko(^>EieDBX&F+~1_nvOFae0CzVS9%5BXw}L6(A=6gLS^WWagm_x_gT zLU3Ia!pY0f*`0nifZMYihKjT92KCo}O=A=Y{Yx4{%B%~GwjL`ZBPE4d8&f%Xc~S~* zIs^_H%;;)Bh?;0e6`1)yg@7Cj?OEe6f|mjiG_RNX^agZWK0z*obn8B zwAx~Degs;Ap0K$p(MN-!;qPGPSGEnhdmF5vVO|la&fd&2V9ae4*@g(S^6$^3)Qv4K zJsh{8L4*1!@SeABmj~;<1jS$GXv#|LhQbR;8V2hw^&Z-D{y(gU&QvLPuN%_x%ds^f zc?N{TX(%!`%wsv*J%0&(=7OJmUo!5$Dh1ON5T`GwssJLOq|Z(I7r>(4h`S-9FU^za zYA4s>0}OjJe?;oR(<;*@5x!Y5rYuV=wcKInWauCBsdEQs=D3C!y4#(au$?0g8{SGR--Zd zkyXepCJ*CjAhKpZ0SqXar)vi~5eUVKx6fO0rB_MSh)oZx8&UiOK5F^x{8Qppz@Q@V zY!6EK@BC2o;uF*-Ixh%&@Qv;hxeK(`mMmG+BJ=Y@|V{#MI*>1aU%=^XfXyAWbGt zxmg4_6PQRQ!*qRByJv7?^;hHKZ*YPGqU`LBrnj<|=0JI^`w9#ch|>X0(LIq`h3CQV zYreRmDG#`^>UkldUb_&%Zlp96s0yOd>SReI2xy!b;0(60hJ8%9!q`{w!;$ZY~OuO@p5Axwp@`5Ts~0&AI!<0$W0 zRqO?z61?sk*dD+2qeLs*fP7>m53XNve_tA1AdK4`wBEcAj2RI~1h%_y?U%@%Ro@_z zo8uhL?K7_&qC2dJU@jAUQhdFwVv512SZNd+Pn=A_i$eSJ* z>&4o|i@7^PasGgb!i|;g5j|As7bwH|2uIK9)>w^H#fy#L~5py9y>1XHa zU#k|{<=54nScBkFh~&1u;2t3QRui6;;Y7f7%IWJL)tTuwgOext*MwVzus9h&C9J8l zdZj_er;Uu44Fdhjj;gPFRI5Wz8ax^ zNQun8LCj0->H3Zwt0|b=g;Am3O4+!Cj|#6~1vT*^=d4Uqr`*ot4?>ARrZ;And>f6- z0B+PrG*HGN-||utb^$d(IZfD*>qX+^)PN#nz_&pD_U_NIR0Nnp?XqU&Gyb+3SU9Iw z01~B!-yLcVd__qcLTQ3D6V#&GSXI1IOAgNg?7aq-|B8l;HOuNJph7GES_SHhamirw zoWUpjShCa8$%f6lP~|vEUE*+QSuYYGwm60X-$pT1145UH zXV12PAho8pc6)1U;SZ<^sHv&>U&bzc|6cF6#?H>pkXUp}we0Orsd8&2>ZP%B>T$hiQ<1Nl`=<{f(gj6$jr;r6l$k9N_J5vvAeNIQIary1s;Mlo9TI%ZEVsE0NZi&wP{{4G;`Xg6JaCEeWxVSIBjE|n49)S;_bNv1NLG0R{ zZuBF$El(-roGX7DC^5QkDriN=#yaFhd~>%kxVriQmxpB-6ViTpo{*B&`T_p*uV0qO zt|I@kEOKW_OTWr7XpRsM&xrQV#>T~}r$Hr#^nL2~l?KB+S9FDdVQ2^Y-^(;tN~${0 z)?;Ajjk@ZGyD=^H!t^z*$Y@(zpj-D=r>{$r@=lMdNuNDPAIQ@_AAJ)#ur6?<#LSiCV+VSH@ z>eP(MLpCHe^V}Pz@qi(6 zBfqpOW#dkke&EF8ECJzxaB#FE1WxDas5j=H`bdOxyrK_>gXbKcV~20ZAXFn*O)Ow; z=a8g%ntY}6FMzNX=KHt}o+)+to@DTeI?@s3rr0*YWpRJDH{G!$P zAvaVJtQ$a#rKs`)2kQ9)k92x6)^or6AgPY*p{_OBT!&=>$JS$4Hq_|$Pc zrT2_aRxWF$=)G{);}8$LD#GQU>9@+#=ZGUUxe@9(He^}8BT-@0yCx3gNNBWGh}J|( zuaS(@qx5)Qn{Eo^8L3aX9_^1c4BJv*wjBCccmMawD#J?x`cc8^5$?F_XAjBSkgaOL zY&M50^pOLQVu2#5*?D`c{E>OjsSvfmJx{l3)>U=6T{BRd0No+}rMpXGJ6HL=-sU~8 zMDpZ&@;Y9xce8E^XaM9f?QR+VA#-jf+>n+%+uFnur3rs|u1^U?&)OY#d)E-csb#1E z*2{vTe=Ez;vI}03jehCV*0aaN=%c&q?l@TDgZ*CUNN?Q*y34EyZSRYGojk_yb`I9V zduX6V<+ZnN5xo?Sl^PT@Wb>)0?L3Z3$!f+>1stgX?QUpAF5*!;dB8=mMEkR4_uHWBja0P@w7 zv0erYT7N*U$g{+}=YKsRQm)85y_?Jf6d8~>0T8FA-Tg8t!kS$ellR;l8AMb=-zHN8 z#?L>jfwl}17fQl5!=HLGQ2KS4`iwkV?z7oZ9`hM<1e_uCG%v6CFupXWtJl<4ELr&@ zU(g7`B#3q#3nc8u6ZPzaSNTdmGKp-yJmAYdHWLDdL9o!W0`N(S6CN{YI2(X`BYiBG zN*O?GHcOG=B0*N}eMyA-s2;68pgg1>K%7s3h&Qo*NwSndi^h5h6cADF$R6-n-stMd z-vVN4V16HAY7gWbs?A$Q4NT*@4QLqSY1@K~_U&#Y=!SqlT?-H`;7w{{mt9j{=RFTX ztil_p%2xXS*yq`-0e}bjZAj{0FJ%L2S5UAi=H&!_3~Xk2_v>e)l8^T8g3=g72>6c= zzg=GF#+ePZmbyHykvL=cjJs-d`_&N%!NCW&JEgRVI}IYjFY6&QAr=eJeMw*E1Y7hNS=Ul#;a9)!-+%in8@ zxn*Gu3WviZq0L^A2x(!bto$ZAI=Z017BpeJz$UP=w*K*b_+<0jw{Oh>MBBT&op9`F z4kT7mP#E}Nbb4|OB6J0)inCqwJ+0Fzl{=SKbP9TS4m9j5y?{)(e>ztU1J zZ~e^-Qbc|N6(ul_22Nq`Z)Ivzd%F@MVTTH@K&9b-QKahb0iEz6ENQU$G^hprt=;F_ zS5ZJxcrwOul~J!V<8-kMo*od{3*HkbF&U}SijSE zk#h#H7NbNnT`+3_`DIs(i`u6zeV+!U%r9|`x0|LhgujM=WU_&aT1HwumbJfQ^cVwL z1Yn&-t0Ro+p|*5@>MKNuAWDq~I?a?jagCxC*l%Fg``;-y0NYhnOubh7XOg3CZ)UvG zVJOiJ3&7iqyv-bslFmu*7vMNhP69=~{{@}vnVf{}-N2pvuLwq;jxMRDM)<&M*5u)f zI?)${@I283Hlaykx>K|l{a3v!3+CIhMgv^xw_Q&RjLtSCF{{(G!){rdHp*>~-N zzIrATB|Uu^hU$GygP=D|B3T*MA@E9(Xwg=>Rwl6dT>>T=reiRzAmRchVsSfr$?Ko^kOP;`nI8hQbvf6K7OW@}QUU>% zy6)@5XR^aJIv)OO+Vl>@iH7?$I9fENQ`FGih;Ix7tzPDmbhZ3fwHmQ}gU$Jb(r{4G6g>+nl!AuRSwi%>BpS zY}Vt%9uH$?{|`Uv|1f8!8jgDk1hg%TtraocS?9Z?J$#W-3^#@1Dj>Q zRY4hFScGFoSwN17AkvSv@NDMF4yCB9yZ;I;ag>!3`(XzWp&-JuToD@PUO?FLN;tBV z&7DP-2J{^lN4%+bfJ%7m!SH~c?;7kkw-+N=TG>L<6sVHsA>Yah<*D5_Rch$KlT5Ha zcF24f^3oFT&Vbzd+viJIzWe+tp6P06SiV7oM#PJrFn__eC-$5Qb_gY)@1~bYaHXBT z4Cesp(GrD`>ihLGq>9?oq*a}0Zm>6((saA2=XVd@B+fusWpSiE@w)RRg!2A{52Kl^ zOYc|63ih;>W*Ba;aFu8Xlb{BnyU6Tetiy%Z#D1-7E z#OWi0T`G~e5FPmJO`{;Zx*H8o9X!9V1l;WaY8(I`8g86RH&`UC#iYt*flYvzwV+65 zf$t~IImE(BdsM^95Jjz$do1)u2=2tqY%$4Jc+!Oz!=3#4#wXK@BOusIEQX3B{OM>uHNz7*eNcCU-bB1ndyJEG z59bf8viz^TF1HFzO1%zu1%!k^9^SX!-L1T7PfScKE!{R|Q@zSpzu&>rB^>$gUCw6O zqmBHCI!DKEKp@bK6YKG9`1B6sn~m4ONP**j_?V8n!Ua)H{3~U~YZ`jK|EA1%+>LYY zoRqXQ1sNF#{ax`q^F#F~P6`S>f3)We=)GjV79=aI0@=UgDHe1MAXrlFy=~MJI}U_V z0n&^u5S&_MKrP{4iA62?3obsgk(Vb#a_@tpE#(W=W5%a zFu}XTT$06WIXG_LMgwa5@-nw0c?~okJ?bo#J=j_qacKd7g)e3J~kFfuwi>ABqj;FPVd?z_E;ug%S~8O!GNvhMbqz>_{IdHa_m?RDlX zp267JYdp0j=H8?Jubsopqz%uXKc7jfudfdc4dn;J>H=IB5wX=Nv_3pGrlqA7Lf`l% z2?ICB*8iX!?(zU#-ERHyEL2BRBA|f+(+{_hR_p~i%zTWEz4ubbT4!Et!L9B!Zyvpv z&;{_LINM5lIt$g+mxP)CM*q(s0`vsmNPQ5xwXuPs8vx9Q-w?glYdvxqy@5r7SireJ z`ecbt^nMgK{Py9VZ*q3Yt?lg}py9j1E*Ug@o0~IKvL`iy*SMrqB$j0YPjBRb2`v~(4W_Fs0YXwpa>n}h-@(NH&U^qg z(FC`&_yEBSz(J4P*F^q|Z*TqJ_%n(p2YOV9CYQbnkH(#lz`8tW8Yi$z6Q4@h$K^q} zeq#*1AAHDA#xn-#jfWtCETLbO;{b#Y)cyYJ3Rr6UkB&TXFvF1h&Xmep60J5d4m9E* zDNq@SPNk3&^mBt_D87J(;+1ZKsaN@Ij2VxAEe&Pi8Bk;a6!-pbC!+00i9B44j@AKROVUh^vGeDRA6qke7SX)+4d=r zU_cE8sd9CUcjPL{wt@Jc{Hm08w~l?p)m`(ptzk$?D7u9{2lVF8RQU43L(6 ztNIQ40|}^(Km15ll0y*e)0Z;iRUKQ2bXCNME>p=@~S4XhDN17=O zH$iS_eX+Pa2|nQ;wLpVrhL4XvKiSC=>TUv6u|pgK^3KiHGSX4fvNI_A(bnLZzl&@B zn?`^gv~`ERcNzsCp{|#CncU`)?ioC7;0&A7z;ijJsb<~kj>j*vvO%7?oS=4JuOadi zsHMpQ&kx}^WPd&ecL_9QZ-7qi;x&H+szs+CcJ+Gv1x+EWp6Hp4 zv)USDoNcpq@u0c43v|AX33lWvz-DLH6*k*~HtiaJ*8~i|p#lJ%a<2U*>wUmYxqy@- zq4^~YQ7YyQ$CSE-YU%TVv<_-ee#n$VVxzE7c#XWPR!?Ot zVG(HB?!IRbvSKYZFmma*eJa6--0;#?Msz7W zw&~?`+W8E+3PuFvfjC%3#1}NNCYSH_)yBhiAusN3-aKpgiCvwlnEx^8mSjp|1<=1A zYi|U!&(-NR&}c&t>i}q)1_>2tnuae`0O@((Yop1M2uH}UW?a#B9(2l}*Ff#z!L38i%9%%dR ztMs4;6BP6n8;(}A{TZ;}SqAQ{$jJHIy#VT|{N3T8Ic%!+7hlPRW8C-}bNUHr4aT)< zMEG6@O(w7vWB~TXqiyqXvJd$&Kby$Prra)&96O8N1jld@X9HZR;NAu36BCQ68{i&~ z7{FnH+Z_OTaqrbnQ1cr{pj-m%)-N2`kG&}rn|&Os-`Hse zT8im=pnUTfJX<9v2V8(SAqUOL?%jb#E6}_}uzECWNC&D!HoQk2tYFDfFs?q!`7jwx zQKx{+;1WA>MF7WZ@P5!)1`As{>xTEyph|j+x?$VJto!iuk{vBTG7v#;4?Xciw!a5o+X-S9i72aCd!J6VV?hmX)R?1z@CGzgkvxw3>z)AFHwzd{TR-o;ZkXWBeTN^9y8W{m$_MLZNKmb0mvy&4nJ(z6`1-ij8_}U6^iG_wb zA?HO*5Hl9Qy$=UInay$3{abek69>HnejUM^qwEJT=l%0 z`KqVXV^wP|S~Ocxe1yMgegIo1UD!0GsWES%X#k=}M++A3H2J^Zs-phjHRWfU2|@da z6Iy|1X%CKr7Zw)Y+tyW{#)AHdV*(+Epx!@c91MiI_-{0<|LC3pFbo7`5(Y&8miXe^ zg5+;oR^`;xe6Wl@_@iTY8++rgkbt`QnzR8>S2H&g6B9u-12iiuL%29N!I38?CqQLH z?WAFwyV#d0hVGWEtNd|BrsEeRY*M0x|yWT(1T92s2ygdyJ z*0pA+*9hZEzE;^|Tn1q|_J00Y0#T611R8THPgp3A$6oji`sW|+ zY+|b__N4s%kG*;Ay&n`1)pDr4AXwpQ;HIx=pv7YSryu&z6koA1a&g@aGtN;A8~d8b z+C3M+(f;Qj7CGJ!U}_Vi>)@BRAushfo54SR=rg=V@eR9B{`U69#>Xiuql^E+)9}4Z zOyUsmnuHkeB}@$$|LKP}%1~QdTm9Zl$e(eCE(N}c5}#A=vi7dAF%QtKRt^Mlpgzaz zb%1u5@BHkrF;?TYopgfEJXP^P_ekHoabsA(@qR&!rf1^|Y^&#yfq`ub)ZV9uQ^gJ& z)vlW};7Gd7C%Bv}Y1O~uG?D<4EFoM5piu9ikP!uq?9R7oCnhH3VpuaEwG zp8Wn1Yw1(`zU~emU*8xPXr0rY12xE)n3&d1t~Yge&C^w7Y)M2E;C)~I+@QF8ZD6@`yxVLw!N16^?Sm0H{3+_r#A<5iK!$6m| z=j1RSB&D@wx7ay3b#!$NxvC}|n143m9%-ZVk$=x;Erw3g$=|8Hg)1Y0A#e1+nz8g| zyGJ&U`>e|f=+@UWHJvj&JWPcH8x z*V=O%!=H^=3UacujpnEOySv5R_6tDA*_R*^&_l_^#RY2OEAk%}G=d5^1qI(tZVnG? zgDnAmt;8X~-@k{2@WO=*FIPa)gQvhH51VabCvV@yy?B9nV3>{*bwO%4{$OC}_Ui1c zB(5EL4_+vrV<(GD#ucyvNp&%mmP71>!SQDHwg zG-N59E#}4c#+h93+3`wo7dJQe(+P8uy!`w@*GEYQx*zvOzn*nunCj-Xg6$r{g!@!g zQXlkOczD=3n5PG%na@+vJ zcO_3ll|*((YiFNT9vzrndHE7kyMNi{?!%H#eUG!bObGn8$EHqp!QMTg`z&4XYvmM=2k3bk~(*x*RrUj5q%AG*_AO(kSx0UEP@YgD_e19d^?w{J$4 zA0~m8)}Mf)m+k-`_0w2VFg+u|@JH&bH@}cj-;YCJO7=qrE+Tu=eq&|U#`BafJ3AEx z9d@?19us`11J`g^yfE*|tY6&NKRsDL#l*rIiFfU0Vv4-zsH|k$T`J)E)TS35)_sfS zYsf&Xp?7>4hP|p>?b>Y|e3YPI8V}oK^`)IsarZvBb2Sj=Rjl$1+cC-Tu@eaK=$sa~ zd>8$DG8EgsI9g==9?7}gf!)he13Geoiw*ALR|e%cYSRk(_(y9%HEa?6%DI1I!>+77NELQg0EdXU z<4?bO^$JjDJt3ru**?If4h;-^=GiP%(JOg+$pj}M3vRmxx3@eo>m5iG@mTOtJzS?a zl(79=T`lB)j#mZ^3Q6x1{HB2Od~o_4J-UEeo=@`0Xc6S+FXS zVRI&!0uQ26?E@Nv@knlWk6d3DiMvCgt)n9o=_pq12e+u*3$=# ze{~AsE8tR?K6ezBYV$!~TNQGA@EQ1uP`A?b%J%$xc6Xsi9;>{;^1A$ayc(5db{Q=h zeuVBu?|8ZoPfiZkEkz$+JC73C+h`^>_v;?|vs)t5(=!6@=*w<7omltgi{qIlpkVK= zq)zyZ7wL|NQUJ}-qz#rs`6dLy001)vfBANLG%x1Jo~1%ZI?4vl=lxw3Z^x_KfnrC` zz5CwcmR6RqkOgz~|mx+&FGoJ!d8aNDs4mGzf4ql45#6-ohGS{oA1<;_> z`X%clKnYnT8@?k*ifA;pFM8Uzkxzcl79&ZXXmlxO?C@DR9u#UU?gzMXbiPJASb1hVx zi*71G@@*8IOiiBo>^+oB87&hC_URx+2wf+HxiCytwG-z|3_T!lL@WxSKl;k>xvR33JoE^yLbF1(8Zs**_Yq zV6fRt;D+&iaUv(*Hn_fW8qceDd%oGyjtsEj=_bd9NHwTRlGX&n{{$q>&H#dxSg&SU zdNGcT>eYE)d;V9pqw>nnM^@L3yJ5aX6BL76^~y_oDoSnTw$2o!$e!+=5` z*b;bAxsm7;r2BVmf6GKHB`PisTN@7UGx3u70+g&8yw?J8*~HiPJ;MfYSEgb`%LKZp zVsv}v+WzwJ5US2YKFbi6GUo{+|UP%TsS^yry8A+G}RPNS6kj&TdQ9NJK z)`C1~MAyxM`@*lesf?@&J<=GrVn-Z7VEfXirJmv?00Z1-eAf3CpJjAROiYUGVmQLb zEr*1*TXr09w;Tk~<`pB=I=&0W%v#IpZz;Vcw=M&vrq3*?&$|6GddxOc(+))3tKb}a z`qL2+neBvM-#&Alp9Vx*RRc4UZBpk+^`-8j1M#T+u{0X(f{@`l7D0a3MCtp^oP8R)>AW zgoFg=SZ`nt@i(W*pSm=teR3qVbUbn+8|dHrG$Ps*&h@n7eonyoO|$-0R%%x0!1#5M8_xg7Kc60q+nWk;+?8SD?rBcQD?L(oD#cKekrlbIk+63tl{MDdc_QRv z7tBuML__cSj96H+RelR`aaZsd3W(R_BH zNcytOU`QEU6LD@Uex6ZSUM(X!6n0$Tx{z+|VGyDBwm0J1iI2vVI!_ltzbZ+bdol-( znF_MWeSoMdE{ZnS#D6If6ePjl{dQXY8si(v_|W%zfJ7}pHA^=m=(beCzmk!*40hM< z9_RVWg{tKM6)MdnsNv2d-%$u`@QrGmSYP+-n4~gTvnXDZ8@FX>2whXOUrO$}m|A@K z_3>5fkb?Iq?s21ezR-UXYX^>d^42b5%RepZzFjdWi9*5FzvqwUij@vu!?ee@jLU~U zmiLmT%Al2(yjLF6wQOJndrg)JF^054fEDbub$Vl1Lp?(=-Vg@J$`7qT2e1nm$~x)! gO4zmk^<(R%5Ys)4P;yWvAb$|Nx1U%2(XfpF0MM}o%m4rY literal 0 HcmV?d00001 From 4a1658ab25063b56fec4c7acdcb2bf4a8e8376dd Mon Sep 17 00:00:00 2001 From: zhujian <505169307@qq.com> Date: Tue, 5 May 2020 00:12:04 +0800 Subject: [PATCH 4/4] perf(run): before train should run step() once --- warmup_scheduler/run.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/warmup_scheduler/run.py b/warmup_scheduler/run.py index 5e1a384..6d11432 100644 --- a/warmup_scheduler/run.py +++ b/warmup_scheduler/run.py @@ -31,13 +31,16 @@ def plot(lr_list): # this zero gradient update is needed to avoid a warning message, issue #8. optim.zero_grad() + optim.step() + scheduler_warmup.step() lr_list = list() for epoch in range(epochs): + current_lr = optim.param_groups[0]['lr'] + optim.step() scheduler_warmup.step() - current_lr = optim.param_groups[0]['lr'] print(epoch + 1, current_lr) lr_list.append(current_lr)