From b234f7edfdf44c91c288ca80fbcc8b6d62448222 Mon Sep 17 00:00:00 2001
From: zhujian <505169307@qq.com>
Date: Fri, 1 May 2020 21:16:11 +0800
Subject: [PATCH 1/4] refactor(userwarning): 1: scheduler.step(epoch) ->
scheduler.step(); 2: get_lr() -> get_last_lr()
---
warmup_scheduler/run.py | 2 +-
warmup_scheduler/scheduler.py | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/warmup_scheduler/run.py b/warmup_scheduler/run.py
index a1dbf3d..d20d3e1 100644
--- a/warmup_scheduler/run.py
+++ b/warmup_scheduler/run.py
@@ -18,7 +18,7 @@
optim.step()
for epoch in range(1, 20):
- scheduler_warmup.step(epoch)
+ scheduler_warmup.step()
print(epoch, optim.param_groups[0]['lr'])
optim.step() # backward pass (update network)
diff --git a/warmup_scheduler/scheduler.py b/warmup_scheduler/scheduler.py
index 8b84082..2ebdc70 100644
--- a/warmup_scheduler/scheduler.py
+++ b/warmup_scheduler/scheduler.py
@@ -28,7 +28,7 @@ def get_lr(self):
if not self.finished:
self.after_scheduler.base_lrs = [base_lr * self.multiplier for base_lr in self.base_lrs]
self.finished = True
- return self.after_scheduler.get_lr()
+ return self.after_scheduler.get_last_lr()
return [base_lr * self.multiplier for base_lr in self.base_lrs]
if self.multiplier == 1.0:
From f4b25f07291d280f90b0f8df7d4eac3f8ba04331 Mon Sep 17 00:00:00 2001
From: zhujian <505169307@qq.com>
Date: Fri, 1 May 2020 21:26:02 +0800
Subject: [PATCH 2/4] perf(run): print and plot lr_list
---
warmup_scheduler/run.py | 36 ++++++++++++++++++++++++++++--------
1 file changed, 28 insertions(+), 8 deletions(-)
diff --git a/warmup_scheduler/run.py b/warmup_scheduler/run.py
index d20d3e1..5e1a384 100644
--- a/warmup_scheduler/run.py
+++ b/warmup_scheduler/run.py
@@ -1,24 +1,44 @@
import torch
-from torch.optim.lr_scheduler import StepLR, ExponentialLR
+from torch.optim.lr_scheduler import CosineAnnealingLR
from torch.optim.sgd import SGD
from warmup_scheduler import GradualWarmupScheduler
+def plot(lr_list):
+ import matplotlib.pyplot as plt
+ from matplotlib.pyplot import MultipleLocator
+
+ f = plt.figure()
+
+ x_major_locator = MultipleLocator(1)
+ ax = plt.gca()
+ ax.xaxis.set_major_locator(x_major_locator)
+
+ x = range(1, len(lr_list) + 1)
+ plt.plot(x, lr_list)
+ plt.show()
+
+
if __name__ == '__main__':
model = [torch.nn.Parameter(torch.randn(2, 2, requires_grad=True))]
optim = SGD(model, 0.1)
- # scheduler_warmup is chained with schduler_steplr
- scheduler_steplr = StepLR(optim, step_size=10, gamma=0.1)
- scheduler_warmup = GradualWarmupScheduler(optim, multiplier=1, total_epoch=5, after_scheduler=scheduler_steplr)
+ epochs = 20
+ # scheduler_warmup is chained with lr_schduler
+ lr_schduler = CosineAnnealingLR(optim, T_max=epochs - 5, eta_min=0.02)
+ scheduler_warmup = GradualWarmupScheduler(optim, multiplier=1, total_epoch=5, after_scheduler=lr_schduler)
# this zero gradient update is needed to avoid a warning message, issue #8.
optim.zero_grad()
- optim.step()
- for epoch in range(1, 20):
+ lr_list = list()
+ for epoch in range(epochs):
+ optim.step()
scheduler_warmup.step()
- print(epoch, optim.param_groups[0]['lr'])
- optim.step() # backward pass (update network)
+ current_lr = optim.param_groups[0]['lr']
+ print(epoch + 1, current_lr)
+ lr_list.append(current_lr)
+
+ plot(lr_list)
From e7c60e0fd33d652233fc9401a8a00b803916f349 Mon Sep 17 00:00:00 2001
From: zhujian <505169307@qq.com>
Date: Fri, 1 May 2020 21:30:48 +0800
Subject: [PATCH 3/4] docs(readme): update sample picture
---
README.md | 32 ++-----------------------------
asset/warmup_cosineannealing.png | Bin 0 -> 22507 bytes
2 files changed, 2 insertions(+), 30 deletions(-)
create mode 100644 asset/warmup_cosineannealing.png
diff --git a/README.md b/README.md
index 72a5fac..f211699 100644
--- a/README.md
+++ b/README.md
@@ -2,9 +2,6 @@
Gradually warm-up(increasing) learning rate for pytorch's optimizer. Proposed in 'Accurate, Large Minibatch SGD: Training ImageNet in 1 Hour'.
-
-Example : Gradual Warmup for 100 epoch, after that, use cosine-annealing.
-
## Install
```
@@ -13,31 +10,6 @@ $ pip install git+https://github.com/ildoonet/pytorch-gradual-warmup-lr.git
## Usage
-See [run.py](warmup_scheduler/run.py) file.
-
-```python
-import torch
-from torch.optim.lr_scheduler import StepLR, ExponentialLR
-from torch.optim.sgd import SGD
-
-from warmup_scheduler import GradualWarmupScheduler
-
-
-if __name__ == '__main__':
- model = [torch.nn.Parameter(torch.randn(2, 2, requires_grad=True))]
- optim = SGD(model, 0.1)
+Example : Gradual Warmup for 5 epoch, after that, use cosine-annealing. See [run.py](warmup_scheduler/run.py) file.
- # scheduler_warmup is chained with schduler_steplr
- scheduler_steplr = StepLR(optim, step_size=10, gamma=0.1)
- scheduler_warmup = GradualWarmupScheduler(optim, multiplier=1, total_epoch=5, after_scheduler=scheduler_steplr)
-
- # this zero gradient update is needed to avoid a warning message, issue #8.
- optim.zero_grad()
- optim.step()
-
- for epoch in range(1, 20):
- scheduler_warmup.step(epoch)
- print(epoch, optim.param_groups[0]['lr'])
-
- optim.step() # backward pass (update network)
-```
+
\ No newline at end of file
diff --git a/asset/warmup_cosineannealing.png b/asset/warmup_cosineannealing.png
new file mode 100644
index 0000000000000000000000000000000000000000..9b462e9b8fd7c2f031bc522f571ce2877cc5818e
GIT binary patch
literal 22507
zcmbrmcRXI*wl=&FM2jT41Zkr8h+cw3^xlbE1ks7!C4wMGbV(4sM2p^QkfQfaNc8R|
zx^J%K*?WKcoPGA$@B99hpIGLaYtAvpxW+ZEF@uy9rHSyV@gWEzl6fel3PG5@5QH&(
z9uxdeRk2(#1Zfq?NZnI+eY7%xr%z&bBDPV6?D+$%t*&+lHDWLY$_a%PEbuSN^*joKe6RaLIN(1q
z|DS&_z1V>7pHfp(^ZD~PZJWER$w{D
z2_{dwkwJz@2|?n-u6_6zgb?%*x5nHb{-AEG$r1?&eCGnje>6;V7)FhG-)#>Mk1JQM
zP*B9ae0i>Hye~2#;dtp&!RW|H&nKOFZxJ<`jm3AaTJY!Xx%SD%LKpDyi;9Z2c6K^C
zIt&U?&-3!Eb#$nBlyr4RGn2flJpQazuh+fDLs=*02f0LpslN{A%X7q={uas5l^~?`
z!SUI%XD%*7S%!g%FXEv&cX#4bX7I_Bbhf(oxOQe{^Myvu5|WZF(sNx&VkX~7Lv`|-
z&lScf#eRFDpoJ%S)v{g|buONQy=y9AGx?Os>*U;rE&MJbSr#E3)v4~$Zzquul+iJb
zp|7HH?^Ik>HDd2oY;3HbpIN)1_?RoDE-6q34syq)+Mu
z7PqH)^kxZ+pM8?G(5wE9zv~1+afCOq?Vdd=+Iy9mnVFiJ3PxGv^~LoR?DSR8QIj6R
z3Ta}x>GO(ZJYGMgAc>A7M8x}>_i|f*TfL~eF>PPhLd(yt{Ln*`*1^)(8n4CQnH>5V
zT1|EQzziR6K!aO6Pu|+l-LsrApyw=UHsesaD!QYEqst+)okLN4B}}BU&h`)#(Dq()9Ai9n?LM#Vq_BPO
z$Vg?1pgrS4wyKLM>PLVqnUs
zwIpWCUH?BpGw$h^-3XypV1HHT3hij#CM6l#*sL}A6IfeY%gMvW
zRn8|Wi5*9sFfuYOoGyC(1mD8=y|}H$m~sAbmqsyfs^!wc_ZmBaU+Vi{5s01B#md=j
z+8^&q?f+g6mwsXpF
zk0s`uM_$hBO6vsc%+K%r*5Xym5nv5kScoX*CFy3IPzEMQk7J4e?Gl&qH7%GPBy~t#O!}xx*pKxLslxtU67&E+KCHw^T7_qugsY~R++zJ
z@8_sj4V%1ltnjr`dHx@nNnCO_#w9yR2jdJ~cp%El$SkZo_L)K-o2#uv3%%1}u9R@w
zv--{QK8%3NYMplXBtLR7ooK}qf`(>cZ}#;5AR9#;V@Lj*FHSD+l~kND>K=8@JHfiJuUE=hdlkLtRwfv}r#WB4gs^6i7yEY}F?x7^%S92Wst(c8oXvoM$+$eGx*Zkwo_bAJ6z-6E<;lA>jIvy
z%2bQH@f;t{THVX(mAmpXnWmLBY&c&3MR+R4$3QXQ!8r?wkCjx0bA_HIvOOIBUDo->
zIgwN}>;$FdZY(4khVw=cUg2h_>UXf1Tb3YQ0&{a|;ZQQk)M{IZ|Ak(qy{jD!r^LP`
zo3PKvdGkouHnUDB^=fRx?wtJoZ?OVF)66=V`Lewmo>z&k=^(Sh$PE>gUffb?U@C*O
zc%2-*C_f%VC3;VeybZ{=fO@~cRz=zH;|d9r)^1`=1GzX>!rnn|m*tEt|;r3caCDjFW8
zx-*)@l1a~W4i%a}g`{Vr)t1r6Kc1bBQZ0i%1Y784SUGLhS{!Mqpihrt
zc|S*oenmmWAR2(3@wb(@gxx&m%`l9)FyMbs#^4#&>$B58(8kW2qDSuQcDB(od30(_
zWFVl7U@qtZ_YwI?ok|#+iV!u29Kl==j<`ZJnwQ81OA2{8eM4f$S_oTz%RqA>@JBZY
z#%L)RZM^JKNxxjqarg4~X<(ZVPFK-~H@GPqgc0-ArmcUXcA51;$72H2L&r5avj()7
zl36pDr~|U|sAH2*i--GJE3#z}j&t^=E>0}pG|m&qGyuc}!fA!~cSpv?#)gM0Dk^T>
zLW}Ai)_EO%NJz-b%Ug7L`ZR0LRYPN7Wu$l=1S7-aC9VAY{QdoXYNaLYb>>D?)L^M9
z^jep6^2V~MCCW9bQEh>lv>T1h=4b%3x
zpL27!6?@~?+T(c^!?)+V`j&rmc9PS^{`l=O>BeAE$v+R;+BS^s2?0v#
zuhj!Ve`G`^v+Nxm697#<-=mnQi?d!+ay_1?a+R7Xy=jC@0SK+Ysz^zBaq>JZ?bhZc
z4h{}nT-;C{_1xTCOiau#G6!R$qwh`@mX?Z&i#=9Lhp4Hk8GqED99Zg(Zg+|Ktj1e#
z$LB8QY;8Tu*T77mC+k-Y5&OnNO>r&$g~j)2TXm|VI2S@-AZRKk-^j=UFX@HF#l?k%
zEUplIYU{Bmd(1x~;MC&7*|GL{X!rbGqjfWU8yG6xB%PNnQVj;vDB_66HFM89aDK
zFbOYxpUwn3o8=;JZs21;523zN3H##iUnD_8MkC$?bz1kUu0xt98*?iXCq@{rR!UMy!kp=>tSqc91yeuqHwHI=DXenox%jGf*c
z#{=+7Lqo%qlknk#goMPGw)*4UF|nhc#OS@RZ*Ru38;p(!H0+bZFAznG-CsRCSlG70
zkjFCFa9GZNVt1+a8^kFBzQP!~LINJTfJ7@P)!9Uq!G^(xC;eq#6Ad#D5>Je}p1#(=
zMp+;pgLyH1r$1Loo0eW{8uM19X6D;Q>;Yh|BuKna*16&oHi{0%;u*$_qy3Ibu0Oc-
zgwvPlCmf!$YHGONQlqEQ&&r_N*>$CAU#EjunEc+4a6^RH2qam0{e4p8weI#(#)UGR
zj9~6zYsnXH8uvB6qI{FjyqMf9DRy=hX;3N
zTypYP;7KBjknLeD$6Y#llATlS=BJZN=q=+VFp@gSHPErpeF0?!hk=-qjIexMa(}f*
zuVmWUT0Kl)-~GlEil-;8(y<@duzat>zE!*TLQ<_%=uY3%mm+LVy_V9v+4|s=awns1
z^YqJn$1Dq>O^x&5z7cQM0xg2EcH3@qfZ0rFYOg|-(Z{YiJ3H<=OCU}M@?aH0(=c#
z6AJRlkA(sO{D}+8dI#d}m@s1)-71+`9u{b-&;tJ%H7Pq@mUZK02vO
zq{mBhw{-YK=OIp3WGHn#vp;bym-_Ee45Gw!zV$~H5L|8
zNgD#e4@PF_e+`{aNGbn9R>TW+(;;I-wQ-E}-{3&vPLQ{3RWH-QLq#$@EJy@4wn8a9
zL!udKvLI#VXvf0)>w9$d!mowyW1S)8^sJKn=+POKkS4t+6IMj5>aqG<4$IMsJ*8|5
zq(mQmkY&5^lbr~_4pRbD54EmPX0GT1+vBMoCh7(VJ_xM$9lT{zHC-K8tA0IHyyvuC
zVUnZRleE?$bt(iM!_cDeS9f;!c5GlaUMo>V8q==+4sa$V7*-)1k1Zk2^T<0sq`lC!
z@MLg%FnjJ{5s1?MiapZX-AvTtw#YjKCVy7;?|B#TMY8!m>HZwcd2w*qYci_k@
zyo8N8Vzx>&QFqcWp3Dj(Ev;COYc}x7=;cIxr;@tTNH#1;3|T8V6Jy-c%0JbCydOKF
zg-3BtYGenP4oc}Fn8LwGcF1f?RsifJS_7!Q82e4yQ4UK&iWsq|pC|_8hs1I7{q9rJ
z(z4~$F7W#tV=bZkhqphf-D&uUoE(X&-}pwT53h@G%5W~uzDjz#1OGvSD;nKUJL>VEr^}m}9U9c%gp(OtjP0A4SVcZUA}Vc85dU1U2NFcKU0r$GqF~KZZKA(%
z5bE#LxO=MN^NYi;a
zPI>*Aa9_?-R8)M=XFKSs$^pWPO`0+zL%y&?o`tLmjUHA-qH6Y$V?mW#iS!dV!NlmNWm0;cx;r3O@8nO1
zOi72lf!E5jD}wj2p8Ee2iIY~W?c(ARTJ`-k70>wi_!k*1t)X}xGkl+L8X@PFV6w+S
zoyj1uFe^^Ja*3XPFh$4mu@zmLz%YU-PG_O_B-|$t9QM|-D0YFcF$kv>p)CM{na}+n
z5RAyPUk`t-6N~PldedYmI3B4pN4JI1RD(E1LPF9afc+1g@!|?ztSd-pim-j_em}B!
zrs)#=sOpqe56kxorLZpxGxGx7Ls{9&bejN`*x7Z*-EaB+o%z}|@egVB080ATGc`?l
zA6n~v-T2wO*l1Ag98?@6EG%qOmAL!q-0Q!bF^_w4U1%N#dih}|@CAXLs3Tcl3`!Wo
zM7Br^i->?=%WJBV3|$~3BxN{0KDM^9dRae|t0Vc`Yh${R$E?$8yy9)AFJP0v)cLUn
zQ{j{W>j2I7*$P9P;M2`*b(*4>`!NGQp$i|yaEPbxd7qn@oKzgSN=|+|X0APcc4_Hs
zWqp4FLWjEhH?G4lrY|{&%UT08MUN)qUMYG)$AB?hgVt1Bn%?~b$K;y#Wdudq*w{!g
zu>Rb+FD3OV{m+!oDZtDm47OjYWk1SftBcOjNaiu*FP=*|Fdyh
zYilbx1;zUC=h;C$+tpK5)T9Qc0!Ur@Z3O?KZJ@mSrdquLQeu=v7e8_!Xz5Ru?H406
zGc#-J)tQ;gbaWiT55Eb*kTr378YL|)9c;nCkoo4##m&v{-@p4DFB|@u^x(M9(NXBU
zI(o7$_yhhB%B3CadJ-!P-@YO&Gjl!nr!8-p#}%)B@%MdzTccE^o_+v(q_QX02@}TT(#jz^+J0B?U(j|3+|s8yvX&Rs!+IkO`%Jwe
zW4D2RP0^nYC8xyu7$xN#wd0B+>XOWBEEnuVkSDkSQ}%w9LDJT*{J=yDnC-o%(N1FNlO=G8!48jY5Cm%rI>RaV)r@4w9jz8&fpw*pU-(na+dPbim(L#gkkp&yHGWm^
zD>;O|m#tn0aEZ0CnkJp)+QQ=>P%ko*AGelc2x3EisZXC05&nLrm&s|maWp}-$4!Y)
z(lJcgA7kFN$RzeWC2#zw7}HPSXRs#V_8h!~5P3;hsSUPE
zI>#BdG#3`c5xvfn^&rl;y=`9SrA_?iV)oCxkU7kf7h=Dg
zPf}c+Mo?_S8^8jk64`!`{^fsgw+!bR;Jm@M-kErYUFzF^!8D4ukQb6%iePy(;`pJg
zc5#X>vwnp8jQ%K07tm2vVw-;XTZQs7+3WC5uhTTqlkKptCYDePlI+LKz0Y#!Kk~X7
z*GLY%zHwtYV#CC4e4n}u2h{@yW-IeYqj3+d4ei_oz+7YK4~w7Pbkwm=tIGDv!_a0&
ztRrfaJl<~Wc>r;MvUKdYQ|*#0ny)N<`qRZX7tVfvOhu7B@V@N&b2f-l>z-3cpX$KG
zS3cSfGfRj(7>@1aCIIMJYMzrGtx?&?
zO!1;*)9u`!SFgWdI%Iz^pww-?__OI9IoO|*v)^|D0=asw@K#EyT<0JpAE2M?X{?8D
z{Rt4PPbP*n8+(z7Tn1n)<-scAPa3|c^EsU_4{3DuN4cF1GKp45C-306G^F!_>ecuB
zw2pyE5yJAUa|FndDjpgfVH;n7ranEd+;gT_c
z>R3w!Fdfs^T+f~{@a{wqOX{*7m*!<{OxTFtmq(0hS~NXi!*6pjM>zw=s4F)#mB=;G
zN3c#Dg8&fSf;bx)3h!4|S3#`O*Vn&%`SNhl=zqa0dkTEoweF>@+uAjsZZQc(%4Hq9
zds%V=%f{9D9h%zzhf@qdCNL-1#mYgr*w{KcI{OC)b0F*0E^Gi}-i_)#lyRG26QRxs;TYhfNsWjo7w0sU*;0YBH*8Q`5K&Lg
znDOzqn_H^JLf{=N$UBr0h27T1D`(O`=&i1p;pCo#`
zn3-fsS4+FjEEAZkw^oAYFhH*bC!2R?K_qCw?LiH?_5}<13JD1<_G@n6
z{JpNt{x<%7f5R!R!8D)wMc8(9`9Xxf!1Vm~9*A$`zPEqnP=l;-UZ
z>;O|2lRe8BSM)MC90>`vB5L@x3Da<_`T@V@=VJi}XZKSAaB#i7s_veHxjK(8Gtkrb
z1QmscD?NHddpuY}m@R$GY+Jor#->;HDje>^oaUw@6T@?@W
zdP|6Ei6yX6T$+!;+rS9{oD+j}`%?OO|hK$d1
zB|L{Z71T9Az$eeq%Xi&BoGlrtUwog_;J=P)LcYrslyri0Z=9+R)QVx;N-3QC?%OED
zh2o6kM4VG(>ux@7|5TtsmaQ>}Y1VB67=9rZx?nPA@mwJ2Yn7CBH^Oc}a)6bY0{(W@
z=im<|C#5_oj2Jq;Q5<$EW0f4CF6cpTAp3^tQ0Z+DCGe0aF#>XXP}xvkCUL0?07ymh
zDGm-HP9KuR72DV4#J{5jd0*{zTZlJh6I)SP>jPWHo>8v2n1vVxtz^E~4G#bx`k
zaHRX{rv1SJeL4H}c@brVH~0M_Wk|!xPxi+TOASFf+uegBBBr001iwn?`(bqVZMzlb
zNY0Rm!qY|yaUIP)!UNaEc=;;mMa0A18&IP(Pcz|=R))hVg{&NHov>ZZ=)0C=9grF1
zaDpYHMwTp)=~K9hBuO-=_CdGWc?hv^{P2*lO2wFMO_uNVWyk<}2R}}VJpMc>LN+VN
z4RO|QYKke~wZ}jEjnDeS)Ljz?tLu7@?$EU@#(e8Q
z4C=0DubgRFzidwDW+qK7`YPZ$lJl#@#u>`7{kYZjw1gG%5-=QK)n}~#^9vhzKUTy_
zoFv4$ly*&vu%Yfk;LmrX?0?+g;4LfWg;J3%%J#5H$cdb9Y{xHyAZn1vz(6d)BxVZF
zQ~#Xg3m8${&YO8A#LEbi6Vu$%zRyY3%U|oH99(n%6T}VPgA|7Dn)in(TW)>Nn6<)y
z+;0+6-*wh`;wE_o9N9BdyJNJ_x6?B}f(fA?O1$)`Jg_8gL;U3kAXS;PX8-;z;3oZ7ly(?xz2pb{$$He
zheq@j&MdMMi@GFkVbpICiW5T1B`bICU5|IBI|wCCX?%)oZq>{8hv#650@P&_4tuFM
zSMLdhQGhjRM%E-K$NH_x*gdgzh4<`&EcTmP+$by(gDc3&!g0&
zq}Pb^_$BflAWQ>9+caTyHAV_7FF?k%fbyZia_!4d*xlW|d-pCoJ3Ag;t&e<_?UX3U
z&p_llprNMD(fQud;kB3^>kU#dkUp=53vWHt(ee2cI%_~rV3%=JlyG1pnlOQ1h64#l
z$^K@oCn=Fs032rdRUuN!i?+IpmmBWgyQitCSyxvVl}bR)T3A|I%XJR~^+2L4bE~Tf
zFgG4ZOqclX9di^u0l|9Bc4wF9A?wYXsaaW!tgN|JRR;%VhJh~8tPL(p7_C!dZV_w{
zR4*60IxDPBCk0na!1n?;_U~#hRaI4;9|Y9)4tItOeU|gezx4XLYJI~z|EG6bsm03^
zLK}msWrz5aXr6!Q?y)26*%2N%}RsAdBbHMz(I8`AA28NJuS9fX^N?L!}@)H2(4#&)=_5Nt(1C9$_*>7Y$
zFk7E&K(xQ4yU}P+JuA?!RAu!#*`E&=JE`DzyE&=BZ7G|+b*X^T;wD^_<%KA(j@D0(
zxq-5M$QAv^ko(^>EieDBX&F+~1_nvOFae0CzVS9%5BXw}L6(A=6gLS^WWagm_x_gT
zLU3Ia!pY0f*`0nifZMYihKjT92KCo}O=A=Y{Yx4{%B%~GwjL`ZBPE4d8&f%Xc~S~*
zIs^_H%;;)Bh?;0e6`1)yg@7Cj?OEe6f|mjiG_RNX^agZWK0z*obn8B
zwAx~Degs;Ap0K$p(MN-!;qPGPSGEnhdmF5vVO|la&fd&2V9ae4*@g(S^6$^3)Qv4K
zJsh{8L4*1!@SeABmj~;<1jS$GXv#|LhQbR;8V2hw^&Z-D{y(gU&QvLPuN%_x%ds^f
zc?N{TX(%!`%wsv*J%0&(=7OJmUo!5$Dh1ON5T`GwssJLOq|Z(I7r>(4h`S-9FU^za
zYA4s>0}OjJe?;oR(<;*@5x!Y5rYuV=wcKInWauCBsdEQs=D3C!y4#(au$?0g8{SGR--Zd
zkyXepCJ*CjAhKpZ0SqXar)vi~5eUVKx6fO0rB_MSh)oZx8&UiOK5F^x{8Qppz@Q@V
zY!6EK@BC2o;uF*-Ixh%&@Qv;hxeK(`mMmG+BJ=Y@|V{#MI*>1aU%=^XfXyAWbGt
zxmg4_6PQRQ!*qRByJv7?^;hHKZ*YPGqU`LBrnj<|=0JI^`w9#ch|>X0(LIq`h3CQV
zYreRmDG#`^>UkldUb_&%Zlp96s0yOd>SReI2xy!b;0(60hJ8%9!q`{w!;$ZY~OuO@p5Axwp@`5Ts~0&AI!<0$W0
zRqO?z61?sk*dD+2qeLs*fP7>m53XNve_tA1AdK4`wBEcAj2RI~1h%_y?U%@%Ro@_z
zo8uhL?K7_&qC2dJU@jAUQhdFwVv512SZNd+Pn=A_i$eSJ*
z>&4o|i@7^PasGgb!i|;g5j|As7bwH|2uIK9)>w^H#fy#L~5py9y>1XHa
zU#k|{<=54nScBkFh~&1u;2t3QRui6;;Y7f7%IWJL)tTuwgOext*MwVzus9h&C9J8l
zdZj_@!p}7sWfH06?WN;D3d~$pg&L86Ze^m|K{>er;Uu44Fdhjj;gPFRI5Wz8ax^
zNQun8LCj0->H3Zwt0|b=g;Am3O4+!Cj|#6~1vT*^=d4Uqr`*ot4?>ARrZ;And>f6-
z0B+PrG*HGN-||utb^$d(IZfD*>qX+^)PN#nz_&pD_U_NIR0Nnp?XqU&Gyb+3SU9Iw
z01~B!-yLcVd__qcLTQ3D6V#&GSXI1IOAgNg?7aq-|B8l;HOuNJph7GES_SHhamirw
zoWUpjShCa8$%f6lP~|vEUE*+QSuYYGwm60X-$pT1145UH
zXV12PAho8pc6)1U;SZ<^sHv&>U&bzc|6cF6#?H>pkXUp}we0Orsd8&2>ZP%B>T$hiQ<1Nl`=<{f(gj6$jr;r6l$k9N_J5vvAeNIQIary1s;Mlo9TI%ZEVsE0NZi&wP{{4G;`Xg6JaCEeWxVSIBjE|n49)S;_bNv1NLG0R{
zZuBF$El(-roGX7DC^5QkDriN=#yaFhd~>%kxVriQmxpB-6ViTpo{*B&`T_p*uV0qO
zt|I@kEOKW_OTWr7XpRsM&xrQV#>T~}r$Hr#^nL2~l?KB+S9FDdVQ2^Y-^(;tN~${0
z)?;Ajjk@ZGyD=^H!t^z*$Y@(zpj-D=r>{$r@=lMdNuNDPAIQ@_AAJ)#ur6?<#LSiCV+VSH@
z>eP(MLpCHe^V}Pz@qi(6
zBfqpOW#dkke&EF8ECJzxaB#FE1WxDas5j=H`bdOxyrK_>gXbKcV~20ZAXFn*O)Ow;
z=a8g%ntY}6FMzNX=KHt}o+)+to@DTeI?@s3rr0*YWpRJDH{G!$P
zAvaVJtQ$a#rKs`)2kQ9)k92x6)^or6AgPY*p{_OBT!&=>$JS$4Hq_|$Pc
zrT2_aRxWF$=)G{);}8$LD#GQU>9@+#=ZGUUxe@9(He^}8BT-@0yCx3gNNBWGh}J|(
zuaS(@qx5)Qn{Eo^8L3aX9_^1c4BJv*wjBCccmMawD#J?x`cc8^5$?F_XAjBSkgaOL
zY&M50^pOLQVu2#5*?D`c{E>OjsSvfmJx{l3)>U=6T{BRd0No+}rMpXGJ6HL=-sU~8
zMDpZ&@;Y9xce8E^XaM9f?QR+VA#-jf+>n+%+uFnur3rs|u1^U?&)OY#d)E-csb#1E
z*2{vTe=Ez;vI}03jehCV*0aaN=%c&q?l@TDgZ*CUNN?Q*y34EyZSRYGojk_yb`I9V
zduX6V<+ZnN5xo?Sl^PT@Wb>)0?L3Z3$!f+>1stgX?QUpAF5*!;dB8=mMEkR4_uHWBja0P@w7
zv0erYT7N*U$g{+}=YKsRQm)85y_?Jf6d8~>0T8FA-Tg8t!kS$ellR;l8AMb=-zHN8
z#?L>jfwl}17fQl5!=HLGQ2KS4`iwkV?z7oZ9`hM<1e_uCG%v6CFupXWtJl<4ELr&@
zU(g7`B#3q#3nc8u6ZPzaSNTdmGKp-yJmAYdHWLDdL9o!W0`N(S6CN{YI2(X`BYiBG
zN*O?GHcOG=B0*N}eMyA-s2;68pgg1>K%7s3h&Qo*NwSndi^h5h6cADF$R6-n-stMd
z-vVN4V16HAY7gWbs?A$Q4NT*@4QLqSY1@K~_UY=!SqlT?-H`;7w{{mt9j{=RFTX
ztil_p%2xXS*yq`-0e}bjZAj{0FJ%L2S5UAi=H&!_3~Xk2_v>e)l8^T8g3=g72>6c=
zzg=GF#+ePZmbyHykvL=cjJs-d`_&N%!NCW&JEgRVI}IYjFY6&QAr=eJeMw*E1Y7hNS=Ul#;a9)!-+%in8@
zxn*Gu3WviZq0L^A2x(!bto$ZAI=Z017BpeJz$UP=w*K*b_+<0jw{Oh>MBBT&op9`F
z4kT7mP#E}Nbb4|OB6J0)inCqwJ+0Fzl{=SKbP9TS4m9j5y?{)(e>ztU1J
zZ~e^-Qbc|N6(ul_22Nq`Z)Ivzd%F@MVTTH@K&9b-QKahb0iEz6ENQU$G^hprt=;F_
zS5ZJxcrwOul~J!V<8-kMo*od{3*HkbF&U}SijSE
zk#h#H7NbNnT`+3_`DIs(i`u6zeV+!U%r9|`x0|LhgujM=WU_&aT1HwumbJfQ^cVwL
z1Yn&-t0Ro+p|*5@>MKNuAWDq~I?a?jagCxC*l%Fg``;-y0NYhnOubh7XOg3CZ)UvG
zVJOiJ3&7iqyv-bslFmu*7vMNhP69=~{{@}vnVf{}-N2pvuLwq;jxMRDM)<&M*5u)f
zI?)${@I283Hlaykx>K|l{a3v!3+CIhMgv^xw_Q&RjLtSCF{{(G!){rdHp*>~-N
zzIrATB|Uu^hU$GygP=D|B3T*MA@E9(Xwg=>Rwl6dT>>T=reiRzAmRchVsSfr$?Ko^kOP;`nI8hQbvf6K7OW@}QUU>%
zy6)@5XR^aJIv)OO+Vl>@iH7?$I9fENQ`FGih;Ix7tzPDmbhZ3fwHmQ}gU$Jb(r{4G6g>+nl!AuRSwi%>BpS
zY}Vt%9uH$?{|`Uv|1f8!8jgDk1hg%TtraocS?9Z?J$#W-3^#@1Dj>Q
zRY4hFScGFoSwN17AkvSv@NDMF4yCB9yZ;I;ag>!3`(XzWp&-JuToD@PUO?FLN;tBV
z&7DP-2J{^lN4%+bfJ%7m!SH~c?;7kkw-+N=TG>L<6sVHsA>Yah<*D5_Rch$KlT5Ha
zcF24f^3oFT&Vbzd+viJIzWe+tp6P06SiV7oM#PJrFn__eC-$5Qb_gY)@1~bYaHXBT
z4Cesp(GrD`>ihLGq>9?oq*a}0Zm>6((saA2=XVd@B+fusWpSiE@w)RRg!2A{52Kl^
zOYc|63ih;>W*Ba;aFu8Xlb{BnyU6Tetiy%Z#D1-7E
z#OWi0T`G~e5FPmJO`{;Zx*H8o9X!9V1l;WaY8(I`8g86RH&`UC#iYt*flYvzwV+65
zf$t~IImE(BdsM^95Jjz$do1)u2=2tqY%$4Jc+!Oz!=3#4#wXK@BOusIEQX3B{OM>uHNz7*eNcCU-bB1ndyJEG
z59bf8viz^TF1HFzO1%zu1%!k^9^SX!-L1T7PfScKE!{R|Q@zSpzu&>rB^>$gUCw6O
zqmBHCI!DKEKp@bK6YKG9`1B6sn~m4ONP**j_?V8n!Ua)H{3~U~YZ`jK|EA1%+>LYY
zoRqXQ1sNF#{ax`q^F#F~P6`S>f3)We=)GjV79=aI0@=UgDHe1MAXrlFy=~MJI}U_V
z0n&^u5S&_MKrP{4iA62?3obsgk(Vb#a_@tpE#(W=W5%a
zFu}XTT$06WIXG_LMgwa5@-nw0c?~okJ?bo#J=j_qacKd7g)e3J~kFfuwi>ABqj;FPVd?z_E;ug%S~8O!GNvhMbqz>_{IdHa_m?RDlX
zp267JYdp0j=H8?Jubsopqz%uXKc7jfudfdc4dn;J>H=IB5wX=Nv_3pGrlqA7Lf`l%
z2?ICB*8iX!?(zU#-ERHyEL2BRBA|f+(+{_hR_p~i%zTWEz4ubbT4!Et!L9B!Zyvpv
z&;{_LINM5lIt$g+mxP)CM*q(s0`vsmNPQ5xwXuPs8vx9Q-w?glYdvxqy@5r7SireJ
z`ecbt^nMgK{Py9VZ*q3Yt?lg}py9j1E*Ug@o0~IKvL`iy*SMrqB$j0YPjBRb2`v~(4W_Fs0YXwpa>n}h-@(NH&U^qg
z(FC`&_yEBSz(J4P*F^q|Z*TqJ_%n(p2YOV9CYQbnkH(#lz`8tW8Yi$z6Q4@h$K^q}
zeq#*1AAHDA#xn-#jfWtCETLbO;{b#Y)cyYJ3Rr6UkB&TXFvF1h&Xmep60J5d4m9E*
zDNq@SPNk3&^mBt_D87J(;+1ZKsaN@Ij2VxAEe&Pi8Bk;a6!-pbC!+00i9B44j@AKROVUh^vGeDRA6qke7SX)+4d=r
zU_cE8sd9CUcjPL{wt@Jc{Hm08w~l?p)m`(ptzk$?D7u9{2lVF8RQU43L(6
ztNIQ40|}^(Km15ll0y*e)0Z;iRUKQ2bXCNME>p=@~S4XhDN17=O
zH$iS_eX+Pa2|nQ;wLpVrhL4XvKiSC=>TUv6u|pgK^3KiHGSX4fvNI_A(bnLZzl&@B
zn?`^gv~`ERcNzsCp{|#CncU`)?ioC7;0&A7z;ijJsb<~kj>j*vvO%7?oS=4JuOadi
zsHMpQ&kx}^WPd&ecL_9QZ-7qi;x&H+szs+CcJ+Gv1x+EWp6Hp4
zv)USDoNcpq@u0c43v|AX33lWvz-DLH6*k*~HtiaJ*8~i|p#lJ%a<2U*>wUmYxqy@-
zq4^~YQ7YyQ$CSE-YU%TVv<_-ee#n$VVxzE7c#XWPR!?Ot
zVG(HB?!IRbvSKYZFmma*eJa6--0;#?Msz7W
zw&~?`+W8E+3PuFvfjC%3#1}NNCYSH_)yBhiAusN3-aKpgiCvwlnEx^8mSjp|1<=1A
zYi|U!&(-NR&}c&t>i}q)1_>2tnuae`0O@((Yop1M2uH}UW?a#B9(2l}*Ff#z!L38i%9%%dR
ztMs4;6BP6n8;(}A{TZ;}SqAQ{$jJHIy#VT|{N3T8Ic%!+7hlPRW8C-}bNUHr4aT)<
zMEG6@O(w7vWB~TXqiyqXvJd$&Kby$Prra)&96O8N1jld@X9HZR;NAu36BCQ68{i&~
z7{FnH+Z_OTaqrbnQ1cr{pj-m%)-N2`kG&}rn|&Os-`Hse
zT8im=pnUTfJX<9v2V8(SAqUOL?%jb#E6}_}uzECWNC&D!HoQk2tYFDfFs?q!`7jwx
zQKx{+;1WA>MF7WZ@P5!)1`As{>xTEyph|j+x?$VJto!iuk{vBTG7v#;4?Xciw!a5o+X-S9i72aCd!J6VV?hmX)R?1z@CGzgkvxw3>z)AFHwzd{TR-o;ZkXWBeTN^9y8W{m$_MLZNKmb0mvy&4nJ(z6`1-ij8_}U6^iG_wb
zA?HO*5Hl9Qy$=UInay$3{abek69>HnejUM^qwEJT=l%0
z`KqVXV^wP|S~Ocxe1yMgegIo1UD!0GsWES%X#k=}M++A3H2J^Zs-phjHRWfU2|@da
z6Iy|1X%CKr7Zw)Y+tyW{#)AHdV*(+Epx!@c91MiI_-{0<|LC3pFbo7`5(Y&8miXe^
zg5+;oR^`;xe6Wl@_@iTY8++rgkbt`QnzR8>S2H&g6B9u-12iiuL%29N!I38?CqQLH
z?WAFwyV#d0hVGWEtNd|BrsEeRY*M0x|yWT(1T92s2ygdyJ
z*0pA+*9hZEzE;^|Tn1q|_J00Y0#T611R8THPgp3A$6oji`sW|+
zY+|b__N4s%kG*;Ay&n`1)pDr4AXwpQ;HIx=pv7YSryu&z6koA1a&g@aGtN;A8~d8b
z+C3M+(f;Qj7CGJ!U}_Vi>)@BRAushfo54SR=rg=V@eR9B{`U69#>Xiuql^E+)9}4Z
zOyUsmnuHkeB}@$$|LKP}%1~QdTm9Zl$e(eCE(N}c5}#A=vi7dAF%QtKRt^Mlpgzaz
zb%1u5@BHkrF;?TYopgfEJXP^P_ekHoabsA(@qR&!rf1^|Y^yfq`ub)ZV9uQ^gJ&
z)vlW};7Gd7C%Bv}Y1O~uG?D<4EFoM5piu9ikP!uq?9R7oCnhH3VpuaEwG
zp8Wn1Yw1(`zU~emU*8xPXr0rY12xE)n3&d1t~Yge&C^w7Y)M2E;C)~I+@QF8ZD6@`yxVLw!N16^?Sm0H{3+_r#A<5iK!$6m|
z=j1RSB&D@wx7ay3b#!$NxvC}|n143m9%-ZVk$=x;Erw3g$=|8Hg)1Y0A#e1+nz8g|
zyGJ&U`>e|f=+@UWHJvj&JWPcH8x
z*V=O%!=H^=3UacujpnEOySv5R_6tDA*_R*^&_l_^#RY2OEAk%}G=d5^1qI(tZVnG?
zgDnAmt;8X~-@k{2@WO=*FIPa)gQvhH51VabCvV@yy?B9nV3>{*bwO%4{$OC}_Ui1c
zB(5EL4_+vrV<(GD#ucyvNp&%mmP71>!SQDHwg
zG-N59E#}4c#+h93+3`wo7dJQe(+P8uy!`w@*GEYQx*zvOzn*nunCj-Xg6$r{g!@!g
zQXlkOczD=3n5PG%na@+vJ
zcO_3ll|*((YiFNT9vzrndHE7kyMNi{?!%H#eUG!bObGn8$EHqp!QMTg`z&4XYvmM=2k3bk~(*x*RrUj5q%AG*_AO(kSx0UEP@YgD_e19d^?w{J$4
zA0~m8)}Mf)m+k-`_0w2VFg+u|@JH&bH@}cj-;YCJO7=qrE+Tu=eq&|U#`BafJ3AEx
z9d@?19us`11J`g^yfE*|tY6&NKRsDL#l*rIiFfU0Vv4-zsH|k$T`J)E)TS35)_sfS
zYsf&Xp?7>4hP|p>?b>Y|e3YPI8V}oK^`)IsarZvBb2Sj=Rjl$1+cC-Tu@eaK=$sa~
zd>8$DG8EgsI9g==9?7}gf!)he13Geoiw*ALR|e%cYSRk(_(y9%HEa?6%DI1I!>+77NELQg0EdXU
z<4?bO^$JjDJt3ru**?If4h;-^=GiP%(JOg+$pj}M3vRmxx3@eo>m5iG@mTOtJzS?a
zl(79=T`lB)j#mZ^3Q6x1{HB2Od~o_4J-UEeo=@`0Xc6S+FXS
zVRI&!0uQ26?E@Nv@knlWk6d3DiMvCgt)n9o=_pq12e+u*3$=#
ze{~AsE8tR?K6ezBYV$!~TNQGA@EQ1uP`A?b%J%$xc6Xsi9;>{;^1A$ayc(5db{Q=h
zeuVBu?|8ZoPfiZkEkz$+JC73C+h`^>_v;?|vs)t5(=!6@=*w<7omltgi{qIlpkVK=
zq)zyZ7wL|NQUJ}-qz#rs`6dLy001)vfBANLG%x1Jo~1%ZI?4vl=lxw3Z^x_KfnrC`
zz5CwcmR6RqkOgz~|mx+&FGoJ!d8aNDs4mGzf4ql45#6-ohGS{oA1<;_>
z`X%clKnYnT8@?k*ifA;pFM8Uzkxzcl79&ZXXmlxO?C@DR9u#UU?gzMXbiPJASb1hVx
zi*71G@@*8IOiiBo>^+oB87&hC_URx+2wf+HxiCytwG-z|3_T!lL@WxSKl;k>xvR33JoE^yLbF1(8Zs**_Yq
zV6fRt;D+&iaUv(*Hn_fW8qceDd%oGyjtsEj=_bd9NHwTRlGX&n{{$q>&H#dxSg&SU
zdNGcT>eYE)d;V9pqw>nnM^@L3yJ5aX6BL76^~y_oDoSnTw$2o!$e!+=5`
z*b;bAxsm7;r2BVmf6GKHB`PisTN@7UGx3u70+g&8yw?J8*~HiPJ;MfYSEgb`%LKZp
zVsv}v+WzwJ5US2YKFbi6GUo{+|UP%TsS^yry8A+G}RPNS6kj&TdQ9NJK
z)`C1~MAyxM`@*lesf?@&J<=GrVn-Z7VEfXirJmv?00Z1-eAf3CpJjAROiYUGVmQLb
zEr*1*TXr09w;Tk~<`pB=I=&0W%v#IpZz;Vcw=M&vrq3*?&$|6GddxOc(+))3tKb}a
z`qL2+neBvM-#&Alp9Vx*RRc4UZBpk+^`-8j1M#T+u{0X(f{@`l7D0a3MCtp^oP8R)>AW
zgoFg=SZ`nt@i(W*pSm=teR3qVbUbn+8|dHrG$Ps*&h@n7eonyoO|$-0R%%x0!1#5M8_xg7Kc60q+nWk;+?8SD?rBcQD?L(oD#cKekrlbIk+63tl{MDdc_QRv
z7tBuML__cSj96H+RelR`aaZsd3W(R_BH
zNcytOU`QEU6LD@Uex6ZSUM(X!6n0$Tx{z+|VGyDBwm0J1iI2vVI!_ltzbZ+bdol-(
znF_MWeSoMdE{ZnS#D6If6ePjl{dQXY8si(v_|W%zfJ7}pHA^=m=(beCzmk!*40hM<
z9_RVWg{tKM6)MdnsNv2d-%$u`@QrGmSYP+-n4~gTvnXDZ8@FX>2whXOUrO$}m|A@K
z_3>5fkb?Iq?s21ezR-UXYX^>d^42b5%RepZzFjdWi9*5FzvqwUij@vu!?ee@jLU~U
zmiLmT%Al2(yjLF6wQOJndrg)JF^054fEDbub$Vl1Lp?(=-Vg@J$`7qT2e1nm$~x)!
gO4zmk^<(R%5Ys)4P;yWvAb$|Nx1U%2(XfpF0MM}o%m4rY
literal 0
HcmV?d00001
From 4a1658ab25063b56fec4c7acdcb2bf4a8e8376dd Mon Sep 17 00:00:00 2001
From: zhujian <505169307@qq.com>
Date: Tue, 5 May 2020 00:12:04 +0800
Subject: [PATCH 4/4] perf(run): before train should run step() once
---
warmup_scheduler/run.py | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/warmup_scheduler/run.py b/warmup_scheduler/run.py
index 5e1a384..6d11432 100644
--- a/warmup_scheduler/run.py
+++ b/warmup_scheduler/run.py
@@ -31,13 +31,16 @@ def plot(lr_list):
# this zero gradient update is needed to avoid a warning message, issue #8.
optim.zero_grad()
+ optim.step()
+ scheduler_warmup.step()
lr_list = list()
for epoch in range(epochs):
+ current_lr = optim.param_groups[0]['lr']
+
optim.step()
scheduler_warmup.step()
- current_lr = optim.param_groups[0]['lr']
print(epoch + 1, current_lr)
lr_list.append(current_lr)