mirror of
https://github.com/pytorch/pytorch.git
synced 2025-10-20 21:14:14 +08:00
30fb2c4aba
Let's have some fun. Pull Request resolved: https://github.com/pytorch/pytorch/pull/78828 Approved by: https://github.com/ezyang
3061 lines
106 KiB
C++
3061 lines
106 KiB
C++
// @generated from optim_baseline.py
|
|
|
|
#include <torch/types.h>
|
|
|
|
#include <vector>
|
|
|
|
namespace expected_parameters {
|
|
|
|
inline std::vector<std::vector<torch::Tensor>> LBFGS() {
|
|
return {
|
|
{
|
|
torch::tensor(
|
|
{-0.20959197386869663,
|
|
-0.49580870398532073,
|
|
-0.1313442585372408,
|
|
-0.3287331939506787,
|
|
-0.24613947168465267,
|
|
0.705889510763571}),
|
|
torch::tensor(
|
|
{-0.10412662274500666, -0.2644705062031845, 0.7102859961803084}),
|
|
torch::tensor(
|
|
{-0.19787984636009417, -0.5320223708266223, -0.5396083236337847}),
|
|
torch::tensor({-0.43108206822505857}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{0.4377600774755075,
|
|
0.3828823919505896,
|
|
0.5308031277873992,
|
|
0.5752746453369446,
|
|
0.23943592910168343,
|
|
1.3739197373644627}),
|
|
torch::tensor(
|
|
{2.209263823172053, 2.154134023426646, 2.534834254325867}),
|
|
torch::tensor(
|
|
{-4.091952315741579, -4.67916063385269, -4.781279234594454}),
|
|
torch::tensor({-4.776742087865583}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{0.4377600774755075,
|
|
0.3828823919505896,
|
|
0.5308031277873992,
|
|
0.5752746453369446,
|
|
0.23943592910168343,
|
|
1.3739197373644627}),
|
|
torch::tensor(
|
|
{2.209263823172053, 2.154134023426646, 2.534834254325867}),
|
|
torch::tensor(
|
|
{-4.091952315741579, -4.67916063385269, -4.781279234594454}),
|
|
torch::tensor({-4.776742087865583}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{0.4377600774755075,
|
|
0.3828823919505896,
|
|
0.5308031277873992,
|
|
0.5752746453369446,
|
|
0.23943592910168343,
|
|
1.3739197373644627}),
|
|
torch::tensor(
|
|
{2.209263823172053, 2.154134023426646, 2.534834254325867}),
|
|
torch::tensor(
|
|
{-4.091952315741579, -4.67916063385269, -4.781279234594454}),
|
|
torch::tensor({-4.776742087865583}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{0.4377600774755075,
|
|
0.3828823919505896,
|
|
0.5308031277873992,
|
|
0.5752746453369446,
|
|
0.23943592910168343,
|
|
1.3739197373644627}),
|
|
torch::tensor(
|
|
{2.209263823172053, 2.154134023426646, 2.534834254325867}),
|
|
torch::tensor(
|
|
{-4.091952315741579, -4.67916063385269, -4.781279234594454}),
|
|
torch::tensor({-4.776742087865583}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{0.4377600774755075,
|
|
0.3828823919505896,
|
|
0.5308031277873992,
|
|
0.5752746453369446,
|
|
0.23943592910168343,
|
|
1.3739197373644627}),
|
|
torch::tensor(
|
|
{2.209263823172053, 2.154134023426646, 2.534834254325867}),
|
|
torch::tensor(
|
|
{-4.091952315741579, -4.67916063385269, -4.781279234594454}),
|
|
torch::tensor({-4.776742087865583}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{0.4377600774755075,
|
|
0.3828823919505896,
|
|
0.5308031277873992,
|
|
0.5752746453369446,
|
|
0.23943592910168343,
|
|
1.3739197373644627}),
|
|
torch::tensor(
|
|
{2.209263823172053, 2.154134023426646, 2.534834254325867}),
|
|
torch::tensor(
|
|
{-4.091952315741579, -4.67916063385269, -4.781279234594454}),
|
|
torch::tensor({-4.776742087865583}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{0.4377600774755075,
|
|
0.3828823919505896,
|
|
0.5308031277873992,
|
|
0.5752746453369446,
|
|
0.23943592910168343,
|
|
1.3739197373644627}),
|
|
torch::tensor(
|
|
{2.209263823172053, 2.154134023426646, 2.534834254325867}),
|
|
torch::tensor(
|
|
{-4.091952315741579, -4.67916063385269, -4.781279234594454}),
|
|
torch::tensor({-4.776742087865583}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{0.4377600774755075,
|
|
0.3828823919505896,
|
|
0.5308031277873992,
|
|
0.5752746453369446,
|
|
0.23943592910168343,
|
|
1.3739197373644627}),
|
|
torch::tensor(
|
|
{2.209263823172053, 2.154134023426646, 2.534834254325867}),
|
|
torch::tensor(
|
|
{-4.091952315741579, -4.67916063385269, -4.781279234594454}),
|
|
torch::tensor({-4.776742087865583}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{0.4377600774755075,
|
|
0.3828823919505896,
|
|
0.5308031277873992,
|
|
0.5752746453369446,
|
|
0.23943592910168343,
|
|
1.3739197373644627}),
|
|
torch::tensor(
|
|
{2.209263823172053, 2.154134023426646, 2.534834254325867}),
|
|
torch::tensor(
|
|
{-4.091952315741579, -4.67916063385269, -4.781279234594454}),
|
|
torch::tensor({-4.776742087865583}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{0.4377600774755075,
|
|
0.3828823919505896,
|
|
0.5308031277873992,
|
|
0.5752746453369446,
|
|
0.23943592910168343,
|
|
1.3739197373644627}),
|
|
torch::tensor(
|
|
{2.209263823172053, 2.154134023426646, 2.534834254325867}),
|
|
torch::tensor(
|
|
{-4.091952315741579, -4.67916063385269, -4.781279234594454}),
|
|
torch::tensor({-4.776742087865583}),
|
|
},
|
|
};
|
|
}
|
|
|
|
inline std::vector<std::vector<torch::Tensor>> LBFGS_with_line_search() {
|
|
return {
|
|
{
|
|
torch::tensor(
|
|
{-0.2108988568338871,
|
|
-0.4975560466422629,
|
|
-0.14129216202471762,
|
|
-0.3420288967865903,
|
|
-0.2523635082803723,
|
|
0.6975570255493777}),
|
|
torch::tensor(
|
|
{-0.10853121966252377, -0.297948499687533, 0.6892015099955717}),
|
|
torch::tensor(
|
|
{-0.05080313011597659,
|
|
-0.39413518751058996,
|
|
-0.28433759745928844}),
|
|
torch::tensor({-0.07113812430174116}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{-0.2108988568338871,
|
|
-0.4975560466422629,
|
|
-0.14129216202471762,
|
|
-0.3420288967865903,
|
|
-0.2523635082803723,
|
|
0.6975570255493777}),
|
|
torch::tensor(
|
|
{-0.10853121966252377, -0.297948499687533, 0.6892015099955717}),
|
|
torch::tensor(
|
|
{-0.05080313011597659,
|
|
-0.39413518751058996,
|
|
-0.28433759745928844}),
|
|
torch::tensor({-0.07113812430174116}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{-0.2108988568338871,
|
|
-0.4975560466422629,
|
|
-0.14129216202471762,
|
|
-0.3420288967865903,
|
|
-0.2523635082803723,
|
|
0.6975570255493777}),
|
|
torch::tensor(
|
|
{-0.10853121966252377, -0.297948499687533, 0.6892015099955717}),
|
|
torch::tensor(
|
|
{-0.05080313011597659,
|
|
-0.39413518751058996,
|
|
-0.28433759745928844}),
|
|
torch::tensor({-0.07113812430174116}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{-0.2108988568338871,
|
|
-0.4975560466422629,
|
|
-0.14129216202471762,
|
|
-0.3420288967865903,
|
|
-0.2523635082803723,
|
|
0.6975570255493777}),
|
|
torch::tensor(
|
|
{-0.10853121966252377, -0.297948499687533, 0.6892015099955717}),
|
|
torch::tensor(
|
|
{-0.05080313011597659,
|
|
-0.39413518751058996,
|
|
-0.28433759745928844}),
|
|
torch::tensor({-0.07113812430174116}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{-0.2108988568338871,
|
|
-0.4975560466422629,
|
|
-0.14129216202471762,
|
|
-0.3420288967865903,
|
|
-0.2523635082803723,
|
|
0.6975570255493777}),
|
|
torch::tensor(
|
|
{-0.10853121966252377, -0.297948499687533, 0.6892015099955717}),
|
|
torch::tensor(
|
|
{-0.05080313011597659,
|
|
-0.39413518751058996,
|
|
-0.28433759745928844}),
|
|
torch::tensor({-0.07113812430174116}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{-0.2108988568338871,
|
|
-0.4975560466422629,
|
|
-0.14129216202471762,
|
|
-0.3420288967865903,
|
|
-0.2523635082803723,
|
|
0.6975570255493777}),
|
|
torch::tensor(
|
|
{-0.10853121966252377, -0.297948499687533, 0.6892015099955717}),
|
|
torch::tensor(
|
|
{-0.05080313011597659,
|
|
-0.39413518751058996,
|
|
-0.28433759745928844}),
|
|
torch::tensor({-0.07113812430174116}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{-0.2108988568338871,
|
|
-0.4975560466422629,
|
|
-0.14129216202471762,
|
|
-0.3420288967865903,
|
|
-0.2523635082803723,
|
|
0.6975570255493777}),
|
|
torch::tensor(
|
|
{-0.10853121966252377, -0.297948499687533, 0.6892015099955717}),
|
|
torch::tensor(
|
|
{-0.05080313011597659,
|
|
-0.39413518751058996,
|
|
-0.28433759745928844}),
|
|
torch::tensor({-0.07113812430174116}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{-0.2108988568338871,
|
|
-0.4975560466422629,
|
|
-0.14129216202471762,
|
|
-0.3420288967865903,
|
|
-0.2523635082803723,
|
|
0.6975570255493777}),
|
|
torch::tensor(
|
|
{-0.10853121966252377, -0.297948499687533, 0.6892015099955717}),
|
|
torch::tensor(
|
|
{-0.05080313011597659,
|
|
-0.39413518751058996,
|
|
-0.28433759745928844}),
|
|
torch::tensor({-0.07113812430174116}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{-0.2108988568338871,
|
|
-0.4975560466422629,
|
|
-0.14129216202471762,
|
|
-0.3420288967865903,
|
|
-0.2523635082803723,
|
|
0.6975570255493777}),
|
|
torch::tensor(
|
|
{-0.10853121966252377, -0.297948499687533, 0.6892015099955717}),
|
|
torch::tensor(
|
|
{-0.05080313011597659,
|
|
-0.39413518751058996,
|
|
-0.28433759745928844}),
|
|
torch::tensor({-0.07113812430174116}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{-0.2108988568338871,
|
|
-0.4975560466422629,
|
|
-0.14129216202471762,
|
|
-0.3420288967865903,
|
|
-0.2523635082803723,
|
|
0.6975570255493777}),
|
|
torch::tensor(
|
|
{-0.10853121966252377, -0.297948499687533, 0.6892015099955717}),
|
|
torch::tensor(
|
|
{-0.05080313011597659,
|
|
-0.39413518751058996,
|
|
-0.28433759745928844}),
|
|
torch::tensor({-0.07113812430174116}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{-0.2108988568338871,
|
|
-0.4975560466422629,
|
|
-0.14129216202471762,
|
|
-0.3420288967865903,
|
|
-0.2523635082803723,
|
|
0.6975570255493777}),
|
|
torch::tensor(
|
|
{-0.10853121966252377, -0.297948499687533, 0.6892015099955717}),
|
|
torch::tensor(
|
|
{-0.05080313011597659,
|
|
-0.39413518751058996,
|
|
-0.28433759745928844}),
|
|
torch::tensor({-0.07113812430174116}),
|
|
},
|
|
};
|
|
}
|
|
|
|
inline std::vector<std::vector<torch::Tensor>> Adam() {
|
|
return {
|
|
{
|
|
torch::tensor(
|
|
{0.7890972864438472,
|
|
0.5024410688121617,
|
|
0.8587073313055582,
|
|
0.6579707241208395,
|
|
0.7476356819075531,
|
|
1.697556420651692}),
|
|
torch::tensor(
|
|
{0.891467636010675, 0.7020513497567501, 1.6892012709428947}),
|
|
torch::tensor(
|
|
{-1.0508030958460797, -1.3941351509567657, -1.284337577714353}),
|
|
torch::tensor({-1.071138110298716}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{8.233039313231828,
|
|
7.971150747377481,
|
|
6.6436209506776,
|
|
6.470977407900541,
|
|
6.170125488259256,
|
|
7.1507391033435015}),
|
|
torch::tensor(
|
|
{8.417695070103735, 6.597188212844593, 7.23175710827678}),
|
|
torch::tensor(
|
|
{-6.729624357635757, -7.09743493108154, -6.753301896575352}),
|
|
torch::tensor({-6.435639096011218}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{8.233424596059296,
|
|
7.971537360032308,
|
|
6.643920150720394,
|
|
6.47127807553724,
|
|
6.170405874224489,
|
|
7.151021086137982}),
|
|
torch::tensor(
|
|
{8.418084791214294, 6.597493171180545, 7.232043740621598}),
|
|
torch::tensor(
|
|
{-6.729918250724671, -7.097730102046093, -6.753584809755359}),
|
|
torch::tensor({-6.4359165566974985}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{8.233424610557648,
|
|
7.971537374586563,
|
|
6.643920161995285,
|
|
6.471278086877829,
|
|
6.170405884785074,
|
|
7.151021096766405}),
|
|
torch::tensor(
|
|
{8.418084805901902, 6.597493182713584, 7.2320437514477875}),
|
|
torch::tensor(
|
|
{-6.72991829363266, -7.097730147102975, -6.753584838821182}),
|
|
torch::tensor({-6.435916580217771}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{8.233424610575101,
|
|
7.971537374611125,
|
|
6.643920162027962,
|
|
6.471278086923278,
|
|
6.170405884809245,
|
|
7.15102109680004}),
|
|
torch::tensor(
|
|
{8.418084805946389, 6.597493182796847, 7.232043751509309}),
|
|
torch::tensor(
|
|
{-6.729918332327653, -7.097730188349552, -6.753584861205486}),
|
|
torch::tensor({-6.435916596115672}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{8.233424610594858,
|
|
7.971537374639166,
|
|
6.643920162065571,
|
|
6.471278086975759,
|
|
6.170405884836981,
|
|
7.1510210968387975}),
|
|
torch::tensor(
|
|
{8.418084805997614, 6.59749318289335, 7.232043751580523}),
|
|
torch::tensor(
|
|
{-6.72991837738045, -7.097730236373201, -6.753584887267492}),
|
|
torch::tensor({-6.43591661462546}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{8.233424610617288,
|
|
7.971537374671012,
|
|
6.643920162108285,
|
|
6.471278087035362,
|
|
6.170405884868481,
|
|
7.151021096882811}),
|
|
torch::tensor(
|
|
{8.418084806055795, 6.59749318300295, 7.232043751661401}),
|
|
torch::tensor(
|
|
{-6.729918428547273, -7.09773029091405, -6.753584916866329}),
|
|
torch::tensor({-6.4359166356471755}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{8.233424610642352,
|
|
7.9715373747065925,
|
|
6.6439201621560064,
|
|
6.471278087101955,
|
|
6.1704058849036745,
|
|
7.151021096931989}),
|
|
torch::tensor(
|
|
{8.418084806120799, 6.597493183125404, 7.232043751751764}),
|
|
torch::tensor(
|
|
{-6.729918485714688, -7.0977303518511805, -6.753584949936365}),
|
|
torch::tensor({-6.43591665913422}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{8.233424610670035,
|
|
7.97153737474589,
|
|
6.6439201622087145,
|
|
6.471278087175502,
|
|
6.170405884942545,
|
|
7.151021096986302}),
|
|
torch::tensor(
|
|
{8.418084806192592, 6.597493183260647, 7.232043751851564}),
|
|
torch::tensor(
|
|
{-6.729918548853505, -7.097730419153473, -6.753584986460725}),
|
|
torch::tensor({-6.435916685074594}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{8.233424610700348,
|
|
7.971537374788922,
|
|
6.643920162266433,
|
|
6.4712780872560405,
|
|
6.17040588498511,
|
|
7.151021097045779}),
|
|
torch::tensor(
|
|
{8.418084806271214, 6.597493183408747, 7.232043751960854}),
|
|
torch::tensor(
|
|
{-6.7299186179943, -7.097730492853521, -6.753585026457088}),
|
|
torch::tensor({-6.435916713480863}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{8.233424610733326,
|
|
7.971537374835737,
|
|
6.643920162329225,
|
|
6.471278087343659,
|
|
6.170405885031416,
|
|
7.151021097110483}),
|
|
torch::tensor(
|
|
{8.418084806356743, 6.597493183569867, 7.232043752079749}),
|
|
torch::tensor(
|
|
{-6.729918693213275, -7.097730573032567, -6.753585069969552}),
|
|
torch::tensor({-6.43591674438434}),
|
|
},
|
|
};
|
|
}
|
|
|
|
inline std::vector<std::vector<torch::Tensor>> Adam_with_weight_decay() {
|
|
return {
|
|
{
|
|
torch::tensor(
|
|
{0.7890990163499767,
|
|
0.5024427688479549,
|
|
0.858707365154099,
|
|
0.65797076763247,
|
|
0.7476358193232038,
|
|
1.6975559791029715}),
|
|
torch::tensor(
|
|
{0.8914677624298939, 0.7020513562204098, 1.6892012237887575}),
|
|
torch::tensor(
|
|
{-1.050803095786311, -1.3941351504224309, -1.2843375776028747}),
|
|
torch::tensor({-1.0711381102847533}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{0.17835734655765323,
|
|
0.2542117171890537,
|
|
0.19681971909229715,
|
|
0.23522651199260597,
|
|
0.17806083719648957,
|
|
0.22943655675307303}),
|
|
torch::tensor(
|
|
{0.6227676931552837, 0.6058596954431213, 0.6077176546857177}),
|
|
torch::tensor(
|
|
{-1.4259755901844118, -1.4333355461952704, -1.408545526635006}),
|
|
torch::tensor({-2.0710783081666215}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{0.17965695035191162,
|
|
0.24254352340441693,
|
|
0.17964663531482672,
|
|
0.24250834976541322,
|
|
0.17962893833698693,
|
|
0.24249920074277215}),
|
|
torch::tensor(
|
|
{0.6287144967638043, 0.6286955805603279, 0.6286563093833837}),
|
|
torch::tensor(
|
|
{-1.4123887230853596, -1.4124007126659273, -1.4122701589749163}),
|
|
torch::tensor({-2.063357041247863}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{0.1796366651819,
|
|
0.24250861931831874,
|
|
0.17963731759793083,
|
|
0.24250861142436989,
|
|
0.1796372002681969,
|
|
0.24250890248031373}),
|
|
torch::tensor(
|
|
{0.6287221269294724, 0.6287225821354421, 0.6287220274975922}),
|
|
torch::tensor(
|
|
{-1.4123466103044011, -1.4123465669572683, -1.4123462614739388}),
|
|
torch::tensor({-2.063368365143669}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{0.17963666103165563,
|
|
0.24250882317446784,
|
|
0.17963665831217887,
|
|
0.24250882481082656,
|
|
0.17963666029066117,
|
|
0.24250882426223175}),
|
|
torch::tensor(
|
|
{0.6287216329900817, 0.6287216340515608, 0.6287216326960158}),
|
|
torch::tensor(
|
|
{-1.4123467542623926, -1.4123467542350234, -1.4123467478191443}),
|
|
torch::tensor({-2.0633690432440437}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{0.17963666098500394,
|
|
0.24250882442377164,
|
|
0.17963666099348902,
|
|
0.2425088244120223,
|
|
0.1796366609725109,
|
|
0.24250882441058697}),
|
|
torch::tensor(
|
|
{0.6287216343798432, 0.6287216343800675, 0.6287216343742645}),
|
|
torch::tensor(
|
|
{-1.4123467490742723, -1.412346749072554, -1.4123467490678536}),
|
|
torch::tensor({-2.0633690434425396}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{0.17963666098407144,
|
|
0.24250882442250174,
|
|
0.17963666098407347,
|
|
0.2425088244224233,
|
|
0.17963666098409325,
|
|
0.2425088244225157}),
|
|
torch::tensor(
|
|
{0.6287216343836609, 0.6287216343836147, 0.6287216343836255}),
|
|
torch::tensor(
|
|
{-1.412346749067226, -1.412346749067243, -1.412346749067169}),
|
|
torch::tensor({-2.063369043434909}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{0.17963666098406988,
|
|
0.2425088244224408,
|
|
0.17963666098407077,
|
|
0.24250882442244073,
|
|
0.17963666098407008,
|
|
0.2425088244224409}),
|
|
torch::tensor(
|
|
{0.6287216343837067, 0.6287216343837065, 0.6287216343837069}),
|
|
torch::tensor(
|
|
{-1.4123467490671706, -1.412346749067171, -1.4123467490671713}),
|
|
torch::tensor({-2.0633690434349057}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{0.17963666098407038,
|
|
0.24250882442244104,
|
|
0.17963666098407027,
|
|
0.24250882442244104,
|
|
0.17963666098407025,
|
|
0.24250882442244098}),
|
|
torch::tensor(
|
|
{0.6287216343837067, 0.628721634383707, 0.6287216343837067}),
|
|
torch::tensor(
|
|
{-1.4123467490671706, -1.4123467490671708, -1.4123467490671706}),
|
|
torch::tensor({-2.0633690434349052}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{0.1796366609840706,
|
|
0.24250882442244143,
|
|
0.17963666098407047,
|
|
0.24250882442244096,
|
|
0.17963666098407025,
|
|
0.24250882442244098}),
|
|
torch::tensor(
|
|
{0.6287216343837069, 0.6287216343837067, 0.6287216343837067}),
|
|
torch::tensor(
|
|
{-1.4123467490671706, -1.4123467490671706, -1.4123467490671708}),
|
|
torch::tensor({-2.0633690434349052}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{0.1796366609840692,
|
|
0.24250882442244046,
|
|
0.17963666098407022,
|
|
0.24250882442244082,
|
|
0.17963666098407,
|
|
0.24250882442244104}),
|
|
torch::tensor(
|
|
{0.6287216343837063, 0.6287216343837068, 0.6287216343837067}),
|
|
torch::tensor(
|
|
{-1.4123467490671708, -1.4123467490671706, -1.4123467490671708}),
|
|
torch::tensor({-2.0633690434349052}),
|
|
},
|
|
};
|
|
}
|
|
|
|
inline std::vector<std::vector<torch::Tensor>>
|
|
Adam_with_weight_decay_and_amsgrad() {
|
|
return {
|
|
{
|
|
torch::tensor(
|
|
{0.7890972867575196,
|
|
0.5024410692260988,
|
|
0.8587073313091852,
|
|
0.6579707241257546,
|
|
0.7476356819241026,
|
|
1.6975564206261673}),
|
|
torch::tensor(
|
|
{0.8914676360248869, 0.7020513497574256, 1.6892012709389561}),
|
|
torch::tensor(
|
|
{-1.050803095846074, -1.3941351509567128, -1.284337577714342}),
|
|
torch::tensor({-1.0711381102987145}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{6.790598887618061,
|
|
6.914995398136696,
|
|
6.41533478566264,
|
|
6.297644005485053,
|
|
5.845162499872375,
|
|
6.862229173597117}),
|
|
torch::tensor(
|
|
{7.958707058914726, 6.511338624975532, 7.100969502256063}),
|
|
torch::tensor(
|
|
{-6.690689640539306, -7.056584601121166, -6.72114879738572}),
|
|
torch::tensor({-6.406608295022552}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{4.707506618354547,
|
|
5.291519064582759,
|
|
6.0451502264500006,
|
|
6.024403702678936,
|
|
5.309533822430375,
|
|
6.388110918107735}),
|
|
torch::tensor(
|
|
{7.200495189000188, 6.398387074819269, 6.904125817198589}),
|
|
torch::tensor(
|
|
{-6.664150387053514, -7.026716194929788, -6.705821732490459}),
|
|
torch::tensor({-6.396310969025695}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{2.9508632109188633,
|
|
3.7657755643994775,
|
|
5.60741774331852,
|
|
5.6957903028180565,
|
|
4.70145185833677,
|
|
5.835064148607041}),
|
|
torch::tensor(
|
|
{6.343524400109462, 6.258242740866945, 6.663022973860484}),
|
|
torch::tensor(
|
|
{-6.630461605133603, -6.988854886932907, -6.686194352796841}),
|
|
torch::tensor({-6.383010489575922}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{1.7128944635692829,
|
|
2.536365345915568,
|
|
5.140416924817106,
|
|
5.33803266121343,
|
|
4.083921806116116,
|
|
5.254596369238127}),
|
|
torch::tensor(
|
|
{5.477917349690043, 6.100068192681452, 6.394747035239918}),
|
|
torch::tensor(
|
|
{-6.591742325353548, -6.945355504749947, -6.663584873152447}),
|
|
torch::tensor({-6.367675348676994}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{0.9341502247285258,
|
|
1.6339620765410685,
|
|
4.6679910940755835,
|
|
4.967688979298023,
|
|
3.4933141073198866,
|
|
4.678195347615295}),
|
|
torch::tensor(
|
|
{4.655117321178743, 5.9294836450698245, 6.110061909503652}),
|
|
torch::tensor(
|
|
{-6.549116242899458, -6.897486328511809, -6.638629863638681}),
|
|
torch::tensor({-6.350731975696792}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{0.483518223008081,
|
|
1.014261673266094,
|
|
4.205928052060015,
|
|
4.596195204751035,
|
|
2.9502123780175378,
|
|
4.125826973031755}),
|
|
torch::tensor(
|
|
{3.90359317709392, 5.750505227817309, 5.816617309738603}),
|
|
torch::tensor(
|
|
{-6.503371263778851, -6.846137347295816, -6.611773185243846}),
|
|
torch::tensor({-6.3324769650576656}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{0.2393576446248765,
|
|
0.6100241101779533,
|
|
3.764601561942264,
|
|
4.231602962540335,
|
|
2.4647709193637635,
|
|
3.6096961114614476}),
|
|
torch::tensor(
|
|
{3.236721556734532, 5.566168160977344, 5.520085344708356}),
|
|
torch::tensor(
|
|
{-6.455100840665729, -6.791979259673106, -6.583347815648856}),
|
|
torch::tensor({-6.3131323905801136}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{0.11401265024593016,
|
|
0.3570521972760832,
|
|
3.350517925954931,
|
|
3.8795333009419823,
|
|
2.0402068661130683,
|
|
3.136550602110189}),
|
|
torch::tensor(
|
|
{2.6579570162250215, 5.378834741966309, 5.224742933241745}),
|
|
torch::tensor(
|
|
{-6.4047717084756375, -6.7355397098532155, -6.55361495837462}),
|
|
torch::tensor({-6.2928722155069945}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{0.05251515193791458,
|
|
0.20410212473600725,
|
|
2.9673680881961273,
|
|
3.543794405777883,
|
|
1.6752677855061209,
|
|
2.709287985107431}),
|
|
torch::tensor(
|
|
{2.164479166686583, 5.190372657839918, 4.9338240234040756}),
|
|
torch::tensor(
|
|
{-6.352761531270841, -6.6772456859648175, -6.52278570167088}),
|
|
torch::tensor({-6.271836898876738}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{0.023489947480426834,
|
|
0.11428338573638941,
|
|
2.616797245623764,
|
|
3.226821571853439,
|
|
1.3659994589608537,
|
|
2.328136084816453}),
|
|
torch::tensor(
|
|
{1.74978620664146, 5.002269811977871, 4.649756802441968}),
|
|
torch::tensor(
|
|
{-6.299382007948917, -6.617449564196286, -6.491034254081261}),
|
|
torch::tensor({-6.250142306631259}),
|
|
},
|
|
};
|
|
}
|
|
|
|
inline std::vector<std::vector<torch::Tensor>> AdamW() {
|
|
return {
|
|
{
|
|
torch::tensor(
|
|
{0.7912062750121864,
|
|
0.5074166292785842,
|
|
0.8601202529258052,
|
|
0.6613910130887053,
|
|
0.7501593169903569,
|
|
1.6905808503961983}),
|
|
torch::tensor(
|
|
{0.8925529482073002, 0.7050308347536254, 1.682309255842939}),
|
|
torch::tensor(
|
|
{-1.05029506454492, -1.3901937990816595, -1.2814942017397601}),
|
|
torch::tensor({-1.0704267290556988}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{3.3165329599188507,
|
|
3.223120441823618,
|
|
2.665544565239194,
|
|
2.6044341406663225,
|
|
2.479859063483047,
|
|
2.836831717112226}),
|
|
torch::tensor(
|
|
{3.3885192024669744, 2.6544147219174556, 2.8709245656887328}),
|
|
torch::tensor(
|
|
{-2.70172647102137, -2.836731459490802, -2.69652471546253}),
|
|
torch::tensor({-2.575239255076019}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{2.231471944853865,
|
|
2.3549328325971755,
|
|
1.5699078054795328,
|
|
1.6160272935884685,
|
|
1.5339085081403547,
|
|
1.7397405105941612}),
|
|
torch::tensor(
|
|
{2.8552579170807926, 1.8369866847839356, 1.9735168512425862}),
|
|
torch::tensor(
|
|
{-2.6042083360293855, -2.6996673713262336, -1.8976087706977893}),
|
|
torch::tensor({-1.6180915942867784}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{2.084688381515552,
|
|
2.3141612674892946,
|
|
1.4850714710140511,
|
|
1.5961047256668386,
|
|
1.440300645879787,
|
|
1.6065354941586025}),
|
|
torch::tensor(
|
|
{3.0111385685659444, 1.955556497153507, 1.9596562467797627}),
|
|
torch::tensor(
|
|
{-2.889337305884852, -2.965249100126337, -1.7721676671605975}),
|
|
torch::tensor({-1.4001341655590005}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{2.0465343456006604,
|
|
2.311613891239368,
|
|
1.4666717526896398,
|
|
1.601383980913499,
|
|
1.4223660595993763,
|
|
1.5711552625612757}),
|
|
torch::tensor(
|
|
{3.07151984580744, 2.0112690538174802, 1.9592484602763875}),
|
|
torch::tensor(
|
|
{-3.0186469726426863, -3.093855445542849, -1.7367953899738784}),
|
|
torch::tensor({-1.3299011560804312}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{2.039659777412556,
|
|
2.3178034179536273,
|
|
1.4654302718412722,
|
|
1.6094701969162322,
|
|
1.4230510816446773,
|
|
1.565168902852383}),
|
|
torch::tensor(
|
|
{3.1007583934270064, 2.039757113618415, 1.9652096140698696}),
|
|
torch::tensor(
|
|
{-3.0880626664330832, -3.166705422245348, -1.73538367534238}),
|
|
torch::tensor({-1.3130428735015893}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{2.0413773043991963,
|
|
2.3251469369586366,
|
|
1.4690808101517236,
|
|
1.6174065798291044,
|
|
1.4280274009117935,
|
|
1.5682418226469732}),
|
|
torch::tensor(
|
|
{3.118843540209399, 2.057729936485249, 1.9742319629710936}),
|
|
torch::tensor(
|
|
{-3.1331019663177013, -3.2154332694373107, -1.7459831639793468}),
|
|
torch::tensor({-1.3148644134154366}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{2.0452604138357113,
|
|
2.332074253989847,
|
|
1.4738845773449165,
|
|
1.6246403004735728,
|
|
1.4335712611625357,
|
|
1.573826630920094}),
|
|
torch::tensor(
|
|
{3.1324088069784093, 2.0711763619826575, 1.9841582498316732}),
|
|
torch::tensor(
|
|
{-3.16737058959847, -3.2529206463859146, -1.7602788393925501}),
|
|
torch::tensor({-1.32281766461531}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{2.0495243704493262,
|
|
2.338413341249581,
|
|
1.4787599440132637,
|
|
1.631210274009555,
|
|
1.438849155552895,
|
|
1.5798736919537595}),
|
|
torch::tensor(
|
|
{3.1438209015414227, 2.0823943437659658, 1.9940075805973108}),
|
|
torch::tensor(
|
|
{-3.19628690363529, -3.2845941643030367, -1.7752333900055153}),
|
|
torch::tensor({-1.332456718314933}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{2.0536979895206295,
|
|
2.3442520601250334,
|
|
1.4834272584224222,
|
|
1.6372462654983486,
|
|
1.4437517398490174,
|
|
1.585780877834892}),
|
|
torch::tensor(
|
|
{3.1540081461072447, 2.0923381262560454, 2.0034284957296107}),
|
|
torch::tensor(
|
|
{-3.222142968201519, -3.312867602521477, -1.7898220261118043}),
|
|
torch::tensor({-1.3422692037690986}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{2.0576784836825315,
|
|
2.3496934395759377,
|
|
1.4878413407927933,
|
|
1.6428612479757005,
|
|
1.4483225979568104,
|
|
1.5914034339763325}),
|
|
torch::tensor(
|
|
{3.163383747232199, 2.101446878895216, 2.012344413569353}),
|
|
torch::tensor(
|
|
{-3.246000281299229, -3.338904166978488, -1.8037666936489785}),
|
|
torch::tensor({-1.3517884775416527}),
|
|
},
|
|
};
|
|
}
|
|
|
|
inline std::vector<std::vector<torch::Tensor>> AdamW_without_weight_decay() {
|
|
return {
|
|
{
|
|
torch::tensor(
|
|
{0.7890972864438476,
|
|
0.5024410688121617,
|
|
0.858707331305558,
|
|
0.6579707241208395,
|
|
0.7476356819075531,
|
|
1.6975564206516922}),
|
|
torch::tensor(
|
|
{0.891467636010675, 0.70205134975675, 1.689201270942895}),
|
|
torch::tensor(
|
|
{-1.0508030958460797, -1.3941351509567654, -1.284337577714353}),
|
|
torch::tensor({-1.071138110298716}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{8.233039313231831,
|
|
7.971150747377481,
|
|
6.643620950677599,
|
|
6.47097740790054,
|
|
6.170125488259256,
|
|
7.150739103343502}),
|
|
torch::tensor(
|
|
{8.417695070103738, 6.597188212844593, 7.23175710827678}),
|
|
torch::tensor(
|
|
{-6.729624357635757, -7.09743493108154, -6.753301896575352}),
|
|
torch::tensor({-6.435639096011218}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{8.233424596059299,
|
|
7.971537360032308,
|
|
6.643920150720393,
|
|
6.471278075537239,
|
|
6.170405874224489,
|
|
7.151021086137983}),
|
|
torch::tensor(
|
|
{8.418084791214298, 6.597493171180545, 7.232043740621598}),
|
|
torch::tensor(
|
|
{-6.729918250724671, -7.097730102046093, -6.753584809755359}),
|
|
torch::tensor({-6.4359165566974985}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{8.233424610557652,
|
|
7.971537374586563,
|
|
6.643920161995284,
|
|
6.471278086877828,
|
|
6.170405884785074,
|
|
7.151021096766406}),
|
|
torch::tensor(
|
|
{8.418084805901906, 6.597493182713584, 7.2320437514477875}),
|
|
torch::tensor(
|
|
{-6.72991829363266, -7.097730147102975, -6.753584838821182}),
|
|
torch::tensor({-6.435916580217771}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{8.233424610575105,
|
|
7.971537374611125,
|
|
6.643920162027961,
|
|
6.471278086923277,
|
|
6.170405884809245,
|
|
7.151021096800041}),
|
|
torch::tensor(
|
|
{8.418084805946393, 6.597493182796847, 7.232043751509309}),
|
|
torch::tensor(
|
|
{-6.729918332327653, -7.097730188349552, -6.753584861205486}),
|
|
torch::tensor({-6.435916596115672}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{8.233424610594861,
|
|
7.971537374639166,
|
|
6.64392016206557,
|
|
6.471278086975758,
|
|
6.170405884836981,
|
|
7.151021096838798}),
|
|
torch::tensor(
|
|
{8.418084805997617, 6.59749318289335, 7.232043751580523}),
|
|
torch::tensor(
|
|
{-6.72991837738045, -7.097730236373201, -6.753584887267492}),
|
|
torch::tensor({-6.43591661462546}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{8.233424610617291,
|
|
7.971537374671012,
|
|
6.643920162108284,
|
|
6.471278087035361,
|
|
6.170405884868481,
|
|
7.151021096882812}),
|
|
torch::tensor(
|
|
{8.418084806055798, 6.59749318300295, 7.232043751661401}),
|
|
torch::tensor(
|
|
{-6.729918428547273, -7.09773029091405, -6.753584916866329}),
|
|
torch::tensor({-6.4359166356471755}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{8.233424610642356,
|
|
7.9715373747065925,
|
|
6.643920162156006,
|
|
6.471278087101954,
|
|
6.1704058849036745,
|
|
7.15102109693199}),
|
|
torch::tensor(
|
|
{8.418084806120802, 6.597493183125404, 7.232043751751764}),
|
|
torch::tensor(
|
|
{-6.729918485714688, -7.0977303518511805, -6.753584949936365}),
|
|
torch::tensor({-6.43591665913422}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{8.233424610670038,
|
|
7.97153737474589,
|
|
6.643920162208714,
|
|
6.471278087175501,
|
|
6.170405884942545,
|
|
7.151021096986303}),
|
|
torch::tensor(
|
|
{8.418084806192596, 6.597493183260647, 7.232043751851564}),
|
|
torch::tensor(
|
|
{-6.729918548853505, -7.097730419153473, -6.753584986460725}),
|
|
torch::tensor({-6.435916685074594}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{8.233424610700352,
|
|
7.971537374788922,
|
|
6.643920162266432,
|
|
6.47127808725604,
|
|
6.17040588498511,
|
|
7.1510210970457795}),
|
|
torch::tensor(
|
|
{8.418084806271217, 6.597493183408747, 7.232043751960854}),
|
|
torch::tensor(
|
|
{-6.7299186179943, -7.097730492853521, -6.753585026457088}),
|
|
torch::tensor({-6.435916713480863}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{8.23342461073333,
|
|
7.971537374835737,
|
|
6.643920162329224,
|
|
6.471278087343658,
|
|
6.170405885031416,
|
|
7.151021097110484}),
|
|
torch::tensor(
|
|
{8.418084806356747, 6.597493183569867, 7.232043752079749}),
|
|
torch::tensor(
|
|
{-6.729918693213275, -7.097730573032567, -6.753585069969552}),
|
|
torch::tensor({-6.43591674438434}),
|
|
},
|
|
};
|
|
}
|
|
|
|
inline std::vector<std::vector<torch::Tensor>> AdamW_with_amsgrad() {
|
|
return {
|
|
{
|
|
torch::tensor(
|
|
{0.7912062750121864,
|
|
0.5074166292785842,
|
|
0.8601202529258052,
|
|
0.6613910130887053,
|
|
0.7501593169903569,
|
|
1.6905808503961983}),
|
|
torch::tensor(
|
|
{0.8925529482073002, 0.7050308347536254, 1.682309255842939}),
|
|
torch::tensor(
|
|
{-1.05029506454492, -1.3901937990816595, -1.2814942017397601}),
|
|
torch::tensor({-1.0704267290556988}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{3.3017259270507915,
|
|
3.2082991753694565,
|
|
2.653930978510442,
|
|
2.5927674339810585,
|
|
2.4689608790182933,
|
|
2.825873703467739}),
|
|
torch::tensor(
|
|
{3.373698198112671, 2.6425942964586664, 2.8597930424244304}),
|
|
torch::tensor(
|
|
{-2.690360632302962, -2.8253191596069525, -2.6855499873057473}),
|
|
torch::tensor({-2.5644658591929406}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{2.222607725541013,
|
|
2.3447188854637004,
|
|
1.5614270655258826,
|
|
1.606610018462357,
|
|
1.5260497191448619,
|
|
1.7309643622674138}),
|
|
torch::tensor(
|
|
{2.84137462783552, 1.824806600633721, 1.9620493659996037}),
|
|
torch::tensor(
|
|
{-2.576642773625787, -2.6706153846815766, -1.8799876863754623}),
|
|
torch::tensor({-1.6044722984810953}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{2.0739558768648205,
|
|
2.3008338863863496,
|
|
1.4738888208638767,
|
|
1.5829485271829449,
|
|
1.4296176764284294,
|
|
1.5939984909850073}),
|
|
torch::tensor(
|
|
{2.9908013612792415, 1.936590940953305, 1.941691630199464}),
|
|
torch::tensor(
|
|
{-2.846562884997548, -2.9195962101501203, -1.746484716887341}),
|
|
torch::tensor({-1.381525131003179}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{2.0333926094256953,
|
|
2.294977109171754,
|
|
1.452870514716895,
|
|
1.584853677999522,
|
|
1.4086299433181402,
|
|
1.5548201727855224}),
|
|
torch::tensor(
|
|
{3.0454817801193976, 1.9867169062383696, 1.935312753106444}),
|
|
torch::tensor(
|
|
{-2.9612116762746394, -3.0322275992001084, -1.7026114905180725}),
|
|
torch::tensor({-1.30563541393247}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{2.02392417168201,
|
|
2.2977279587859,
|
|
1.4488511120131309,
|
|
1.5894646930743725,
|
|
1.406253686759073,
|
|
1.5450647949022756}),
|
|
torch::tensor(
|
|
{3.069025602955343, 2.0096872138967488, 1.935438546309299}),
|
|
torch::tensor(
|
|
{-3.016103148166836, -3.0893062953033583, -1.6925290685615872}),
|
|
torch::tensor({-1.282870120405012}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{2.0230257817348316,
|
|
2.3016167065040647,
|
|
1.4496901978629444,
|
|
1.5939034289777392,
|
|
1.4082421794430946,
|
|
1.5444538756003756}),
|
|
torch::tensor(
|
|
{3.0814016132787954, 2.022150201844143, 1.9387429991308658}),
|
|
torch::tensor(
|
|
{-3.0466485946438406, -3.1223144611322446, -1.6944083009127773}),
|
|
torch::tensor({-1.2786980736911064}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{2.024305922065404,
|
|
2.305101549510461,
|
|
1.4516863420588493,
|
|
1.5976447954882376,
|
|
1.4108596097183552,
|
|
1.5464236284303425}),
|
|
torch::tensor(
|
|
{3.0892574233545065, 2.0300944858242236, 1.943040321845021}),
|
|
torch::tensor(
|
|
{-3.0664189567897306, -3.1440820888166425, -1.6999750448893618}),
|
|
torch::tensor({-1.2806281811826203}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{2.0259854116237364,
|
|
2.3080169152218293,
|
|
1.4537680296813915,
|
|
1.6007369432392426,
|
|
1.4132529823064277,
|
|
1.5489035046525346}),
|
|
torch::tensor(
|
|
{3.0949717180851337, 2.0358251379915764, 1.9473249654893}),
|
|
torch::tensor(
|
|
{-3.0808231377905426, -3.160021699873689, -1.7062031001273494}),
|
|
torch::tensor({-1.28423401369705}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{2.0275923948348638,
|
|
2.3104512601389637,
|
|
1.455657715721078,
|
|
1.6033123357613526,
|
|
1.4153003204463288,
|
|
1.5512775896116622}),
|
|
torch::tensor(
|
|
{3.099479021299846, 2.0403012223048775, 1.9512285931847464}),
|
|
torch::tensor(
|
|
{-3.092151979336299, -3.1725453680885267, -1.7120689614428697}),
|
|
torch::tensor({-1.2880095517062655}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{2.029022468328371,
|
|
2.3125066985045892,
|
|
1.4573100228823295,
|
|
1.605484259933419,
|
|
1.417038245960655,
|
|
1.5533932056240227}),
|
|
torch::tensor(
|
|
{3.103195011518616, 2.043964003458376, 1.9546640840748621}),
|
|
torch::tensor(
|
|
{-3.1014680843131184, -3.1828179298513968, -1.7172933346797972}),
|
|
torch::tensor({-1.2914899987134136}),
|
|
},
|
|
};
|
|
}
|
|
|
|
inline std::vector<std::vector<torch::Tensor>> Adagrad() {
|
|
return {
|
|
{
|
|
torch::tensor(
|
|
{0.7891011045987429,
|
|
0.502443924512199,
|
|
0.8587078329085825,
|
|
0.6579710994224826,
|
|
0.7476364836215006,
|
|
1.697557019500397}),
|
|
torch::tensor(
|
|
{0.8914687688941954, 0.7020514988069096, 1.6892015076050444}),
|
|
torch::tensor(
|
|
{-1.0508031297732776, -1.3941351871450518, -1.284337597261839}),
|
|
torch::tensor({-1.071138124161711}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{2.4079229696892583,
|
|
2.2346803754764286,
|
|
1.6967885588547365,
|
|
1.552279695827649,
|
|
1.2259044248443602,
|
|
2.221279696180243}),
|
|
torch::tensor(
|
|
{2.9334079162217193, 1.7619824934767887, 2.3464577179091473}),
|
|
torch::tensor(
|
|
{-2.221396083069719, -2.549950976011168, -1.9709315957317095}),
|
|
torch::tensor({-1.5858816837541876}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{2.510404433941812,
|
|
2.3522584510262887,
|
|
1.7921695110761213,
|
|
1.657755825836846,
|
|
1.2891186618593045,
|
|
2.291878516133922}),
|
|
torch::tensor(
|
|
{3.092171180776419, 1.8971624370952997, 2.438734251283465}),
|
|
torch::tensor(
|
|
{-2.437641633486504, -2.7704264590526573, -2.0949471699460225}),
|
|
torch::tensor({-1.6769121890401757}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{2.5652648968109415,
|
|
2.4155313947260972,
|
|
1.844241233613541,
|
|
1.7156513351246399,
|
|
1.3245206506797171,
|
|
2.3315409972138825}),
|
|
torch::tensor(
|
|
{3.178399916514377, 1.9721945764936502, 2.4909037706250428}),
|
|
torch::tensor(
|
|
{-2.5658710403147933, -2.901921821645266, -2.168560672193225}),
|
|
torch::tensor({-1.7307903926154131}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{2.6021584494332592,
|
|
2.4582101324909065,
|
|
1.8796060082750778,
|
|
1.7550965207414717,
|
|
1.3489253597999988,
|
|
2.3589345190118247}),
|
|
torch::tensor(
|
|
{3.2368674310041516, 2.0236468833666894, 2.52707132741292}),
|
|
torch::tensor(
|
|
{-2.6573969292994164, -2.9960731060650505, -2.2211375717304076}),
|
|
torch::tensor({-1.7692090167089707}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{2.629700772579208,
|
|
2.4901377017698683,
|
|
1.906173477530586,
|
|
1.7847957161833832,
|
|
1.3674517119505822,
|
|
2.3797578857769905}),
|
|
torch::tensor(
|
|
{3.2807643102638546, 2.062561811940094, 2.5546379424362775}),
|
|
torch::tensor(
|
|
{-2.7286379977755035, -3.0695109399636236, -2.262081199960513}),
|
|
torch::tensor({-1.7990936323432214}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{2.6515471766995247,
|
|
2.51550257362603,
|
|
1.927341363452414,
|
|
1.8084994719811576,
|
|
1.3823309942932445,
|
|
2.3964995243914373}),
|
|
torch::tensor(
|
|
{3.3157334001309473, 2.093728023484945, 2.5768468697402924}),
|
|
torch::tensor(
|
|
{-2.786981763434855, -3.129746439571402, -2.29562487034177}),
|
|
torch::tensor({-1.8235564908139104}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{2.6695780544837886,
|
|
2.53646401614724,
|
|
1.9448721033433505,
|
|
1.828157582353901,
|
|
1.3947329882074622,
|
|
2.4104657178934947}),
|
|
torch::tensor(
|
|
{3.344694775590452, 2.1196465761628516, 2.5954050923596252}),
|
|
torch::tensor(
|
|
{-2.8363936812536537, -3.1808219609745194, -2.32404190866147}),
|
|
torch::tensor({-1.8442667636913117}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{2.684883801533072,
|
|
2.5542762192735515,
|
|
1.9597939532350015,
|
|
1.844909608012419,
|
|
1.4053459079217485,
|
|
2.4224257790968386}),
|
|
torch::tensor(
|
|
{3.369349515259956, 2.1417845308976795, 2.611319989214332}),
|
|
torch::tensor(
|
|
{-2.879251075341889, -3.225165734647855, -2.3486956737228057}),
|
|
torch::tensor({-1.86222449978646}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{2.698151012423769,
|
|
2.56972998600169,
|
|
1.972757472697587,
|
|
1.8594775691681182,
|
|
1.4146081751022495,
|
|
2.43287021079559}),
|
|
torch::tensor(
|
|
{3.390772758897601, 2.1610741754331757, 2.6252349489549824}),
|
|
torch::tensor(
|
|
{-2.917092322961074, -3.264351563375218, -2.370468664387175}),
|
|
torch::tensor({-1.8780765115117757}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{2.7098389356033783,
|
|
2.5833548721723747,
|
|
1.9841994925173085,
|
|
1.8723468731726323,
|
|
1.4228158926355312,
|
|
2.4421305315945085}),
|
|
torch::tensor(
|
|
{3.4096859099156673, 2.178143852041279, 2.6375854547611364}),
|
|
torch::tensor(
|
|
{-2.9509704554208467, -3.2994581338995044, -2.3899651139415874}),
|
|
torch::tensor({-1.8922653655195538}),
|
|
},
|
|
};
|
|
}
|
|
|
|
inline std::vector<std::vector<torch::Tensor>> Adagrad_with_weight_decay() {
|
|
return {
|
|
{
|
|
torch::tensor(
|
|
{0.7891011218979068,
|
|
0.5024439415126254,
|
|
0.8587078332470682,
|
|
0.6579710998575992,
|
|
0.7476364849956589,
|
|
1.6975570150849029}),
|
|
torch::tensor(
|
|
{0.8914687701583902, 0.7020514988715463, 1.6892015071335027}),
|
|
torch::tensor(
|
|
{-1.0508031297726799, -1.3941351871397083, -1.2843375972607243}),
|
|
torch::tensor({-1.0711381241615712}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{0.1846116678522213,
|
|
0.24944077103107917,
|
|
0.18651745437755768,
|
|
0.25219093533041764,
|
|
0.18712037968446713,
|
|
0.25289206444055234}),
|
|
torch::tensor(
|
|
{0.6482869597891656, 0.6580215784646755, 0.6581256007663537}),
|
|
torch::tensor(
|
|
{-1.454709711443681, -1.4748063405174818, -1.4811625946604765}),
|
|
torch::tensor({-1.905292836544363}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{0.18059895999281475,
|
|
0.2438515539257779,
|
|
0.18067177884778182,
|
|
0.24397186395008694,
|
|
0.18168388351830797,
|
|
0.24533853846052017}),
|
|
torch::tensor(
|
|
{0.6325250261983028, 0.6331827793513023, 0.6366659383355598}),
|
|
torch::tensor(
|
|
{-1.420803333750877, -1.4215627240541653, -1.4320264544533396}),
|
|
torch::tensor({-2.030135641848322}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{0.17981392697398363,
|
|
0.2427571544305695,
|
|
0.17981150414451733,
|
|
0.24275725992310523,
|
|
0.18014798619115763,
|
|
0.2432144956227816}),
|
|
torch::tensor(
|
|
{0.6294321320817985, 0.6294873737410742, 0.6306958589251878}),
|
|
torch::tensor(
|
|
{-1.4139253354785764, -1.413902680470981, -1.4173628530293867}),
|
|
torch::tensor({-2.056210117690093}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{0.17967006242163747,
|
|
0.24255582734557277,
|
|
0.17966873677301953,
|
|
0.24255462870545766,
|
|
0.1797588230898851,
|
|
0.24267729072765756}),
|
|
torch::tensor(
|
|
{0.6288576295241085, 0.6288643132826753, 0.6291921485342001}),
|
|
torch::tensor(
|
|
{-1.4126465879787569, -1.4126335126907266, -1.4135586793353685}),
|
|
torch::tensor({-2.0618018405404825}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{0.17964321284685653,
|
|
0.24251808241139364,
|
|
0.17964291377171066,
|
|
0.24251779104198598,
|
|
0.1796651574178102,
|
|
0.2425481059085456}),
|
|
torch::tensor(
|
|
{0.628748693136779, 0.6287498167193976, 0.6288312441271762}),
|
|
torch::tensor(
|
|
{-1.412405895385289, -1.4124029484481164, -1.4126313051315378}),
|
|
torch::tensor({-2.0630223163099304}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{0.1796379973927849,
|
|
0.2425107191246215,
|
|
0.1796379363134342,
|
|
0.2425106592536174,
|
|
0.17964321802205502,
|
|
0.24251786094585864}),
|
|
torch::tensor(
|
|
{0.6287272170354161, 0.6287274414587727, 0.6287468362309863}),
|
|
torch::tensor(
|
|
{-1.4123588626342634, -1.412358263650784, -1.412412480569672}),
|
|
torch::tensor({-2.0632918101480255}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{0.17963694231402444,
|
|
0.24250922426893615,
|
|
0.1796369298071621,
|
|
0.2425092121007451,
|
|
0.17963815759528073,
|
|
0.24251088666939838}),
|
|
torch::tensor(
|
|
{0.6287228195255881, 0.6287228675172439, 0.628727383936762}),
|
|
torch::tensor(
|
|
{-1.4123493065102781, -1.412349184462438, -1.4123617872243597}),
|
|
torch::tensor({-2.063351765096138}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{0.1796367215904632,
|
|
0.24250891070978667,
|
|
0.17963671897003045,
|
|
0.24250890818318074,
|
|
0.17963700091084855,
|
|
0.24250929278196054}),
|
|
torch::tensor(
|
|
{0.6287218911936107, 0.6287219017313679, 0.6287229399204574}),
|
|
torch::tensor(
|
|
{-1.4123473011084142, -1.412347275640343, -1.41235016959507}),
|
|
torch::tensor({-2.0633651674043505}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{0.17963667424978783,
|
|
0.24250884333120687,
|
|
0.17963667368829764,
|
|
0.24250884279379462,
|
|
0.17963673796131557,
|
|
0.24250893047794023}),
|
|
torch::tensor(
|
|
{0.6287216908150736, 0.6287216931558691, 0.6287219299749583}),
|
|
torch::tensor(
|
|
{-1.4123468700596724, -1.4123468646187736, -1.4123475243360133}),
|
|
torch::tensor({-2.0633681724342527}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{0.1796366639185348,
|
|
0.24250882860835257,
|
|
0.17963666379614568,
|
|
0.24250882849182892,
|
|
0.17963667838367053,
|
|
0.2425088483939741}),
|
|
torch::tensor(
|
|
{0.6287216468984888, 0.6287216474215305, 0.6287217011907862}),
|
|
torch::tensor(
|
|
{-1.4123467758545658, -1.412346774671038, -1.4123469244007658}),
|
|
torch::tensor({-2.0633688474977467}),
|
|
},
|
|
};
|
|
}
|
|
|
|
inline std::vector<std::vector<torch::Tensor>>
|
|
Adagrad_with_weight_decay_and_lr_decay() {
|
|
return {
|
|
{
|
|
torch::tensor(
|
|
{0.7891011046018798,
|
|
0.5024439245163383,
|
|
0.8587078329086189,
|
|
0.6579710994225316,
|
|
0.747636483621666,
|
|
1.697557019500142}),
|
|
torch::tensor(
|
|
{0.8914687688943375, 0.7020514988069164, 1.6892015076050049}),
|
|
torch::tensor(
|
|
{-1.0508031297732776, -1.3941351871450511, -1.284337597261839}),
|
|
torch::tensor({-1.0711381241617108}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{2.346218944110103,
|
|
2.191939439502003,
|
|
1.683355201740813,
|
|
1.5405520021635604,
|
|
1.2137800230828062,
|
|
2.205283463717303}),
|
|
torch::tensor(
|
|
{2.9090564593404, 1.7509657336815554, 2.336166413186925}),
|
|
torch::tensor(
|
|
{-2.206159683368316, -2.5344318233445415, -1.9622783535807609}),
|
|
torch::tensor({-1.5796101463783623}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{2.3889328781057233,
|
|
2.2678221038007296,
|
|
1.7667624725138267,
|
|
1.6358015176639822,
|
|
1.2655767687152566,
|
|
2.261088056711282}),
|
|
torch::tensor(
|
|
{3.045569451994985, 1.8770196253823253, 2.4192707519566765}),
|
|
torch::tensor(
|
|
{-2.4079300017528613, -2.7399112002234305, -2.0780613510632375}),
|
|
torch::tensor({-1.664722108226537}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{2.3886137557806384,
|
|
2.2922158071009178,
|
|
1.8078384116424007,
|
|
1.684352474440932,
|
|
1.290353948335789,
|
|
2.2870715509706496}),
|
|
torch::tensor(
|
|
{3.111110355394278, 1.9438501730282314, 2.4630249355872826}),
|
|
torch::tensor(
|
|
{-2.5226122034499263, -2.857315093916292, -2.143964860243905}),
|
|
torch::tensor({-1.7130685809905042}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{2.374703352203156,
|
|
2.298804499257456,
|
|
1.8330249458212446,
|
|
1.7151661013307244,
|
|
1.3048586226945842,
|
|
2.3017650590464274}),
|
|
torch::tensor(
|
|
{3.150318222034133, 1.9877926185369321, 2.491399976401679}),
|
|
torch::tensor(
|
|
{-2.601415913361488, -2.938203895113964, -2.1892988334550028}),
|
|
torch::tensor({-1.7462964261966805}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{2.3553658567303812,
|
|
2.297191758042688,
|
|
1.8501154749072124,
|
|
1.736836058688188,
|
|
1.3141313000193942,
|
|
2.3107452592153854}),
|
|
torch::tensor(
|
|
{3.1762315339155434, 2.0197585204578647, 2.5117041377790197}),
|
|
torch::tensor(
|
|
{-2.6606644002288697, -2.9991216074293856, -2.223413376189609}),
|
|
torch::tensor({-1.7712905233118807}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{2.3338052201696207,
|
|
2.2913023710914993,
|
|
1.8624163948044772,
|
|
1.7530300731725454,
|
|
1.3203313209234842,
|
|
2.3163969478854747}),
|
|
torch::tensor(
|
|
{3.1943525925688934, 2.044447386769377, 2.527109724607397}),
|
|
torch::tensor(
|
|
{-2.7076634717294894, -3.047500808469036, -2.250495807208967}),
|
|
torch::tensor({-1.7911288238757486}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{2.3114979154644892,
|
|
2.2830501835377808,
|
|
1.871616142999356,
|
|
1.765632597660841,
|
|
1.324565631636651,
|
|
2.319939234205203}),
|
|
torch::tensor(
|
|
{3.2074779925809085, 2.0642940833670544, 2.5392671301471235}),
|
|
torch::tensor(
|
|
{-2.7463093287485925, -3.087315541134716, -2.272780318857348}),
|
|
torch::tensor({-1.8074516661537263}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{2.2891841627387346,
|
|
2.2734716995793693,
|
|
1.8786818999895825,
|
|
1.7757301317117602,
|
|
1.3274682997719436,
|
|
2.3220679353993825}),
|
|
torch::tensor(
|
|
{3.2172019619454075, 2.0807140893178175, 2.5491374815141876}),
|
|
torch::tensor(
|
|
{-2.7789204504423823, -3.1209351402429175, -2.2915969523376867}),
|
|
torch::tensor({-1.821234722948421}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{2.2672498238343066,
|
|
2.2631678037928893,
|
|
1.8842131287032622,
|
|
1.7840007705383882,
|
|
1.3294311820750493,
|
|
2.323211243034543}),
|
|
torch::tensor(
|
|
{3.224507488068445, 2.094598223519413, 2.5573257155791715}),
|
|
torch::tensor(
|
|
{-2.8069849199086647, -3.1498826045022925, -2.3077996970997727}),
|
|
torch::tensor({-1.8331040438272388}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{2.2458961718688957,
|
|
2.2525031725114775,
|
|
1.8886034384961112,
|
|
1.7908930341267952,
|
|
1.3307102435291205,
|
|
2.323647497600462}),
|
|
torch::tensor(
|
|
{3.230036385413078, 2.1065407459636134, 2.5642349249609664}),
|
|
torch::tensor(
|
|
{-2.83151249424399, -3.1751926295316566, -2.3219682378974036}),
|
|
torch::tensor({-1.8434843744626483}),
|
|
},
|
|
};
|
|
}
|
|
|
|
inline std::vector<std::vector<torch::Tensor>> RMSprop() {
|
|
return {
|
|
{
|
|
torch::tensor(
|
|
{0.7890625772821005,
|
|
0.502415108650816,
|
|
0.8587027713011453,
|
|
0.657967312300643,
|
|
0.7476283936579036,
|
|
1.6975509766054537}),
|
|
torch::tensor(
|
|
{0.8914573371873159, 0.7020499947573374, 1.6891991194739453}),
|
|
torch::tensor(
|
|
{-1.0508027874171133, -1.3941348219724659, -1.2843374000099703}),
|
|
torch::tensor({-1.0711379842715099}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{2.448571858277443,
|
|
2.2809152044417678,
|
|
1.7346424449151965,
|
|
1.5940004770230667,
|
|
1.250761131839982,
|
|
2.248993270255382}),
|
|
torch::tensor(
|
|
{2.994661478530102, 1.8150485290864256, 2.382542610897819}),
|
|
torch::tensor(
|
|
{-2.3036981738757825, -2.6337299521275646, -2.018370122358821}),
|
|
torch::tensor({-1.620787559800898}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{2.5837582475607785,
|
|
2.4365737242301537,
|
|
1.8622886519354538,
|
|
1.7357065282848232,
|
|
1.3369695670141974,
|
|
2.3454934716983695}),
|
|
torch::tensor(
|
|
{3.2061266499381618, 1.9981112525417788, 2.5092495986614}),
|
|
torch::tensor(
|
|
{-2.6110809365525958, -2.9484807193016787, -2.194898560798439}),
|
|
torch::tensor({-1.7501043480625826}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{2.669969051134511,
|
|
2.536559412710799,
|
|
1.9456091681389671,
|
|
1.828914948091767,
|
|
1.3952956766999587,
|
|
2.4110816686341923}),
|
|
torch::tensor(
|
|
{3.343672975593657, 2.1204057198913002, 2.5961524902119497}),
|
|
torch::tensor(
|
|
{-2.8372329851331006, -3.1817729538857207, -2.3249971853996954}),
|
|
torch::tensor({-1.8450422173907486}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{2.7375365004059122,
|
|
2.6153071545358633,
|
|
2.0117493624534313,
|
|
1.9033001982031035,
|
|
1.4427501882445097,
|
|
2.4646213743186127}),
|
|
torch::tensor(
|
|
{3.452912454199796, 2.2190451524127535, 2.667552790123282}),
|
|
torch::tensor(
|
|
{-3.0329479456731505, -3.384582488936652, -2.4377299824997136}),
|
|
torch::tensor({-1.9271014784118226}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{2.7952372917068753,
|
|
2.682820220375722,
|
|
2.0687223272686994,
|
|
1.967654548778711,
|
|
1.4844410726622166,
|
|
2.5117888904510117}),
|
|
torch::tensor(
|
|
{3.5471904628565745, 2.305113548262141, 2.7307948248967304}),
|
|
torch::tensor(
|
|
{-3.2141190290332537, -3.572944633614449, -2.5421970206546827}),
|
|
torch::tensor({-2.0029976985219666}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{2.8467333937519483,
|
|
2.7432785177110395,
|
|
2.119898810135385,
|
|
2.0256805416741255,
|
|
1.5225256464280221,
|
|
2.554983108087885}),
|
|
torch::tensor(
|
|
{3.632098323876194, 2.383289304179778, 2.7889864719999222}),
|
|
torch::tensor(
|
|
{-3.387579944167926, -3.7537658010839294, -2.6423123266260427}),
|
|
torch::tensor({-2.075616951445725}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{2.8938600498060203,
|
|
2.798776984183615,
|
|
2.16697381475156,
|
|
2.079238538430203,
|
|
1.5580820887115123,
|
|
2.5954023023969692}),
|
|
torch::tensor(
|
|
{3.71043881435304, 2.455919099321953, 2.8436784441941008}),
|
|
torch::tensor(
|
|
{-3.5567368287146417, -3.930484868709691, -2.740026479434574}),
|
|
torch::tensor({-2.146398256871758}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{2.937649394399939,
|
|
2.850492965312311,
|
|
2.210902777232446,
|
|
2.1293746183147633,
|
|
1.5917084661873866,
|
|
2.6337100421079533}),
|
|
torch::tensor(
|
|
{3.7837853328443516, 2.5243155701130604, 2.8957265009949373}),
|
|
torch::tensor(
|
|
{-3.7234268485210475, -4.104949193318518, -2.836390693799751}),
|
|
torch::tensor({-2.216118773360611}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{2.9787316798887558,
|
|
2.8991451078473207,
|
|
2.252272487010975,
|
|
2.1767288729202767,
|
|
1.6237627746697592,
|
|
2.670302507579268}),
|
|
torch::tensor(
|
|
{3.8530980655045086, 2.589275531553025, 2.9456388178450936}),
|
|
torch::tensor(
|
|
{-3.8886856459619636, -4.278191888396593, -2.9319964928350313}),
|
|
torch::tensor({-2.285217699505124}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{3.017515620579049,
|
|
2.9452004251453268,
|
|
2.291469925522962,
|
|
2.2217217025782916,
|
|
1.6544730927621272,
|
|
2.705431441204422}),
|
|
torch::tensor(
|
|
{3.9190041004420166, 2.651317624465938, 2.993736489599992}),
|
|
torch::tensor(
|
|
{-4.053111559341913, -4.450801801162238, -3.0271845519131957}),
|
|
torch::tensor({-2.3539498905973444}),
|
|
},
|
|
};
|
|
}
|
|
|
|
inline std::vector<std::vector<torch::Tensor>> RMSprop_with_weight_decay() {
|
|
return {
|
|
{
|
|
torch::tensor(
|
|
{0.7890798754118442,
|
|
0.5024321083861885,
|
|
0.8587031097835685,
|
|
0.6579677474141494,
|
|
0.7476297677960806,
|
|
1.6975465611838714}),
|
|
torch::tensor(
|
|
{0.891458601354904, 0.7020500593937647, 1.6891986479348047}),
|
|
torch::tensor(
|
|
{-1.0508027868194278, -1.3941348166291232, -1.2843373988951865}),
|
|
torch::tensor({-1.0711379841318796}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{0.2139892652405453,
|
|
0.2779011713353896,
|
|
0.18684802794665187,
|
|
0.2507569370785562,
|
|
0.19145335235130007,
|
|
0.2557687813140708}),
|
|
torch::tensor(
|
|
{0.6720959116689083, 0.6480734848064099, 0.654263007004671}),
|
|
torch::tensor(
|
|
{-1.4357633640899097, -1.4493557950073235, -1.4619011018357073}),
|
|
torch::tensor({-1.9673083558727926}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{0.23961935744660673,
|
|
0.30354236865888945,
|
|
0.19567694278583514,
|
|
0.2544696440133763,
|
|
0.21982879020261814,
|
|
0.27495711471979495}),
|
|
torch::tensor(
|
|
{0.6927895724658635, 0.638015535479105, 0.6523245375960234}),
|
|
torch::tensor(
|
|
{-1.413722583500382, -1.4170291001633526, -1.4166977298480703}),
|
|
torch::tensor({-2.0626651115437147}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{0.2506635865272117,
|
|
0.314639511428635,
|
|
0.25116892910818034,
|
|
0.30431399579592144,
|
|
0.25219625048710015,
|
|
0.3160110008170742}),
|
|
torch::tensor(
|
|
{0.7051419232960522, 0.6699011906397543, 0.699097284678438}),
|
|
torch::tensor(
|
|
{-1.4206083241624232, -1.4257037444100107, -1.4171061826065132}),
|
|
torch::tensor({-2.075537874763694}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{0.23285924743063605,
|
|
0.29652494304777544,
|
|
0.2335322002738168,
|
|
0.2969991261380461,
|
|
0.23358272245229555,
|
|
0.2973997498166104}),
|
|
torch::tensor(
|
|
{0.6855589594925036, 0.6796983775695974, 0.6864174803983276}),
|
|
torch::tensor(
|
|
{-1.43110762794651, -1.4334934742818164, -1.422739552145125}),
|
|
torch::tensor({-2.0842642493046184}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{0.23356397699389828,
|
|
0.29737142391985355,
|
|
0.23367622061822368,
|
|
0.29749447597160267,
|
|
0.23418481357395918,
|
|
0.29818122925156104}),
|
|
torch::tensor(
|
|
{0.6866530583001205, 0.6858933385102559, 0.6883944045412603}),
|
|
torch::tensor(
|
|
{-1.4564955509607018, -1.4583548131500643, -1.4418225445708595}),
|
|
torch::tensor({-2.1064103749186183}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{0.2318717301174723,
|
|
0.2952904159872858,
|
|
0.23194024439476665,
|
|
0.29537019824987687,
|
|
0.2316421336904657,
|
|
0.2951041425983894}),
|
|
torch::tensor(
|
|
{0.6834813130194509, 0.6834401711464199, 0.6837275457100463}),
|
|
torch::tensor(
|
|
{-1.4647835805276763, -1.4653452408179053, -1.4571142112777709}),
|
|
torch::tensor({-2.1209598505912086}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{0.2308683396504178,
|
|
0.2940474629750448,
|
|
0.23089067678260966,
|
|
0.29407615110959306,
|
|
0.23064069314214175,
|
|
0.29379043611390243}),
|
|
torch::tensor(
|
|
{0.6815062281792611, 0.6815233687209215, 0.6812759203026146}),
|
|
torch::tensor(
|
|
{-1.4643013018530682, -1.4644523635284246, -1.4617493939684878}),
|
|
torch::tensor({-2.1247293635678854}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{0.23066464201462678,
|
|
0.29376059273730426,
|
|
0.23067069245857366,
|
|
0.2937690399784267,
|
|
0.23057551211606675,
|
|
0.2936477517373107}),
|
|
torch::tensor(
|
|
{0.6809028781780304, 0.6809134105028244, 0.6807404613096301}),
|
|
torch::tensor(
|
|
{-1.4637927352177986, -1.4638374228010727, -1.46299287102643}),
|
|
torch::tensor({-2.1258082720638107}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{0.23062625079199173,
|
|
0.2936990787425707,
|
|
0.2306278924729115,
|
|
0.2937014834661651,
|
|
0.23059813368157003,
|
|
0.29366073890476396}),
|
|
torch::tensor(
|
|
{0.6807251804689082, 0.6807295616357246, 0.6806523640328994}),
|
|
torch::tensor(
|
|
{-1.4635790398985618, -1.463592926902286, -1.4633272688236565}),
|
|
torch::tensor({-2.1261396358141798}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{0.23061701122700193,
|
|
0.293683983817782,
|
|
0.23061747865501653,
|
|
0.29368467998690806,
|
|
0.23060855595638208,
|
|
0.2936719507340021}),
|
|
torch::tensor(
|
|
{0.6806714673830832, 0.6806730903175793, 0.6806434720800856}),
|
|
torch::tensor(
|
|
{-1.4635008778278134, -1.4635052859178375, -1.4634208375068285}),
|
|
torch::tensor({-2.1262432969587723}),
|
|
},
|
|
};
|
|
}
|
|
|
|
inline std::vector<std::vector<torch::Tensor>>
|
|
RMSprop_with_weight_decay_and_centered() {
|
|
return {
|
|
{
|
|
torch::tensor(
|
|
{0.7941000061626792,
|
|
0.507452636734552,
|
|
0.8637405354185987,
|
|
0.663005089317529,
|
|
0.7526661272860107,
|
|
1.7025887305065852}),
|
|
torch::tensor(
|
|
{0.8964950370033696, 0.7070877948157552, 1.6942369105467197}),
|
|
torch::tensor(
|
|
{-1.055840599214661, -1.3991726335388424, -1.2893752132746332}),
|
|
torch::tensor({-1.0761757981162612}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{2.3762999876885833,
|
|
2.239095829416783,
|
|
1.726175067071914,
|
|
1.5891569459230444,
|
|
1.2410074108588462,
|
|
2.2345431036725723}),
|
|
torch::tensor(
|
|
{2.990896455635836, 1.8152108764849464, 2.377985429759037}),
|
|
torch::tensor(
|
|
{-2.3071822180635286, -2.636859516619699, -2.0198181394256642}),
|
|
torch::tensor({-1.622583045791722}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{2.372800588647971,
|
|
2.3022753207224254,
|
|
1.836028714221617,
|
|
1.7190937269287105,
|
|
1.3068955839895078,
|
|
2.3035835673200364}),
|
|
torch::tensor(
|
|
{3.1656599892042343, 1.9942937608209466, 2.4947143457182657}),
|
|
torch::tensor(
|
|
{-2.6139790332516775, -2.9507738987695404, -2.1954425128779516}),
|
|
torch::tensor({-1.7513053380188806}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{2.2398453700818455,
|
|
2.2513384246965904,
|
|
1.8892176431436287,
|
|
1.7921873754661686,
|
|
1.3310951408713538,
|
|
2.3236392222350397}),
|
|
torch::tensor(
|
|
{3.240166119454613, 2.109742813600189, 2.5651614461576973}),
|
|
torch::tensor(
|
|
{-2.8388734382997454, -3.1824200770676123, -2.324831397600949}),
|
|
torch::tensor({-1.8460315737386976}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{1.9829606312242465,
|
|
2.097356567850692,
|
|
1.9050263843525033,
|
|
1.8325835415812346,
|
|
1.3222762370713104,
|
|
2.3024963133870147}),
|
|
torch::tensor(
|
|
{3.2465360572089974, 2.1967266045869915, 2.6091992649970672}),
|
|
torch::tensor(
|
|
{-3.0326878099587207, -3.3827004807595005, -2.436989182250496}),
|
|
torch::tensor({-1.928273216206344}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{1.6051175329080525,
|
|
1.8332107491649114,
|
|
1.8794767349053179,
|
|
1.8403588051948856,
|
|
1.273824111314107,
|
|
2.2296571379436823}),
|
|
torch::tensor(
|
|
{3.1814362940910437, 2.263019214072847, 2.6273016977574013}),
|
|
torch::tensor(
|
|
{-3.210932646440219, -3.567153254014387, -2.541016943923914}),
|
|
torch::tensor({-2.0049155134617154}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{1.1588059349082709,
|
|
1.477861379523226,
|
|
1.7992410089026634,
|
|
1.806460009198667,
|
|
1.1739931551629919,
|
|
2.08647960875392}),
|
|
torch::tensor(
|
|
{3.03843703712275, 2.308203068375877, 2.6125393914734083}),
|
|
torch::tensor(
|
|
{-3.379830678608588, -3.741970414470626, -2.6410082400846546}),
|
|
torch::tensor({-2.079294995910487}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{0.7701433312419088,
|
|
1.1105026677424745,
|
|
1.646507516936639,
|
|
1.71625269098179,
|
|
1.013748545414221,
|
|
1.8532966501655352}),
|
|
torch::tensor(
|
|
{2.827176875885245, 2.327401948159928, 2.5535309398603405}),
|
|
torch::tensor(
|
|
{-3.54193329850986, -3.9096652952123145, -2.739408870192437}),
|
|
torch::tensor({-2.1537939241668997}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{0.5598923129351211,
|
|
0.8460500042788701,
|
|
1.4084175549165017,
|
|
1.5547314210944563,
|
|
0.8019580519338424,
|
|
1.5258384663629627}),
|
|
torch::tensor(
|
|
{2.5774950379490265, 2.313101306699127, 2.4388695757441745}),
|
|
torch::tensor(
|
|
{-3.6974974230160087, -4.070190514312716, -2.8378932675718405}),
|
|
torch::tensor({-2.2307225014430423}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{0.5016784472836648,
|
|
0.7258690889265433,
|
|
1.0976902935953956,
|
|
1.319949187972513,
|
|
0.5853930356154851,
|
|
1.1446978015944624}),
|
|
torch::tensor(
|
|
{2.3235249877284945, 2.2592840970420176, 2.2681461698609375}),
|
|
torch::tensor(
|
|
{-3.8444921272569115, -4.22021051361099, -2.9373192115434263}),
|
|
torch::tensor({-2.312733063937045}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{0.4875468895095056,
|
|
0.6878747871467128,
|
|
0.7787871237567606,
|
|
1.0462592546102176,
|
|
0.4416468896022397,
|
|
0.8122992916762792}),
|
|
torch::tensor(
|
|
{2.1078734515587483, 2.17034337037527, 2.0666325968568535}),
|
|
torch::tensor(
|
|
{-3.9782695475825216, -4.352093055115415, -3.0377809502927033}),
|
|
torch::tensor({-2.403496388200805}),
|
|
},
|
|
};
|
|
}
|
|
|
|
inline std::vector<std::vector<torch::Tensor>>
|
|
RMSprop_with_weight_decay_and_centered_and_momentum() {
|
|
return {
|
|
{
|
|
torch::tensor(
|
|
{0.7941000061626794,
|
|
0.507452636734552,
|
|
0.8637405354185985,
|
|
0.663005089317529,
|
|
0.7526661272860107,
|
|
1.7025887305065852}),
|
|
torch::tensor(
|
|
{0.8964950370033699, 0.7070877948157552, 1.6942369105467197}),
|
|
torch::tensor(
|
|
{-1.055840599214661, -1.3991726335388424, -1.2893752132746332}),
|
|
torch::tensor({-1.0761757981162612}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{11.587263945492355,
|
|
12.552112516667206,
|
|
10.773002960161074,
|
|
10.782117868337808,
|
|
9.675467654064093,
|
|
10.830689360054789}),
|
|
torch::tensor(
|
|
{15.298238342006444, 11.252244653209866, 11.423905295074075}),
|
|
torch::tensor(
|
|
{-11.287147147258441, -11.673871066494183, -11.143068139029769}),
|
|
torch::tensor({-10.744790465364126}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{5.993130757784388,
|
|
7.778269455146452,
|
|
9.705741295559012,
|
|
9.974952848613889,
|
|
8.171307305871647,
|
|
9.551498426643077}),
|
|
torch::tensor(
|
|
{12.811268477045155, 10.912201832960703, 10.87477550647832}),
|
|
torch::tensor(
|
|
{-11.20842921856976, -11.58706973895515, -11.098172235374586}),
|
|
torch::tensor({-10.714110383698559}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{1.917316794757853,
|
|
3.442098373003915,
|
|
8.160846071267297,
|
|
8.76673426856121,
|
|
6.163892823252042,
|
|
7.748894752821816}),
|
|
torch::tensor(
|
|
{9.52929937981379, 10.371703621802425, 10.02242566317017}),
|
|
torch::tensor(
|
|
{-11.07914626767133, -11.444639737948599, -11.02397978065452}),
|
|
torch::tensor({-10.663204622623406}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{0.24211162925745067,
|
|
0.8235150923738451,
|
|
6.109652191353378,
|
|
7.070860554523037,
|
|
3.8366635637770212,
|
|
5.46037058418296}),
|
|
torch::tensor(
|
|
{5.7908039507441, 9.534309069066389, 8.752252906881251}),
|
|
torch::tensor(
|
|
{-10.868651889371552, -11.212965695734527, -10.90242744782103}),
|
|
torch::tensor({-10.579596899816439}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{0.0024206009020476234,
|
|
0.05521740497689468,
|
|
3.753606156332189,
|
|
4.9331546064599685,
|
|
1.7094621184709604,
|
|
3.022224882400484}),
|
|
torch::tensor(
|
|
{2.4729429920325234, 8.290211439306459, 6.983317870704776}),
|
|
torch::tensor(
|
|
{-10.529133489023623, -10.839885990130032, -10.704345435808353}),
|
|
torch::tensor({-10.44279235413811}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{8.523664833406631e-06,
|
|
-0.00018498015809617104,
|
|
1.6343074841140277,
|
|
2.683608480982546,
|
|
0.41425107807132744,
|
|
1.092111816609512}),
|
|
torch::tensor(
|
|
{0.553119873538318, 6.566845593450314, 4.783317472190566}),
|
|
torch::tensor(
|
|
{-9.990101114696575, -10.24914448933998, -10.38447825909146}),
|
|
torch::tensor({-10.220382375374728}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{5.3669182339397725e-08,
|
|
-2.899704029399283e-07,
|
|
0.37916783268568177,
|
|
0.9399553431452395,
|
|
0.02859528129337607,
|
|
0.17650614337704745}),
|
|
torch::tensor(
|
|
{0.03166973497545419, 4.442846994093523, 2.5203464928754724}),
|
|
torch::tensor(
|
|
{-9.15653357178671, -9.339631853060773, -9.875729313751442}),
|
|
torch::tensor({-9.862669711962374}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{2.1133356499004335e-06,
|
|
2.4524630407768025e-06,
|
|
0.023655729923601883,
|
|
0.14273709578291396,
|
|
-8.950192389690758e-05,
|
|
0.004237697008964042}),
|
|
torch::tensor(
|
|
{-0.00012364097582548376, 2.291191859107928, 0.8331414409602524}),
|
|
torch::tensor(
|
|
{-7.922566174765117, -8.003055545094796, -9.086673634672907}),
|
|
torch::tensor({-9.297519364373224}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{0.0023497430294992434,
|
|
0.0028611316714725037,
|
|
0.0006998739627296072,
|
|
0.003657156536057531,
|
|
0.001654303471369622,
|
|
0.0018171459470053366}),
|
|
torch::tensor(
|
|
{0.004569191565477355, 0.7292466599711233, 0.11475431260766135}),
|
|
torch::tensor(
|
|
{-6.223834483308681, -6.185383631607397, -7.912955414853613}),
|
|
torch::tensor({-8.430731662958186}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{0.10393820340367545,
|
|
0.13982074666181732,
|
|
0.0831407198272949,
|
|
0.10183584198629944,
|
|
0.13949594516972202,
|
|
0.17822672100147108}),
|
|
torch::tensor(
|
|
{0.340394645020639, 0.24860888862359687, 0.3191404515531066}),
|
|
torch::tensor(
|
|
{-4.174294597914298, -4.037528929635062, -6.297198700024484}),
|
|
torch::tensor({-7.182093090194918}),
|
|
},
|
|
};
|
|
}
|
|
|
|
inline std::vector<std::vector<torch::Tensor>> SGD() {
|
|
return {
|
|
{
|
|
torch::tensor(
|
|
{-0.21063957030131192,
|
|
-0.4972093725858961,
|
|
-0.13931849072410168,
|
|
-0.33939101965581686,
|
|
-0.25112865488453673,
|
|
0.6992101966874735}),
|
|
torch::tensor(
|
|
{-0.1076573444246077, -0.2913064413859577, 0.6933846874181748}),
|
|
torch::tensor(
|
|
{-0.07998325778863398,
|
|
-0.42149210515421365,
|
|
-0.33498349553944556}),
|
|
torch::tensor({-0.14255126505509488}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{-0.15543131540224012,
|
|
-0.42351103963720343,
|
|
-0.04196796248622072,
|
|
-0.2095223178068499,
|
|
-0.16031407286541022,
|
|
0.8209742464453325}),
|
|
torch::tensor(
|
|
{0.07724343607160136, 0.03387529472490231, 1.0028793648054941}),
|
|
torch::tensor(
|
|
{-0.8213382425894498, -1.1570800333254736, -1.615476033165743}),
|
|
torch::tensor({-1.8734090731084845}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{-0.13342791770744886,
|
|
-0.3941509709488104,
|
|
-0.011470356542661934,
|
|
-0.16885142516066962,
|
|
-0.13306680693528108,
|
|
0.8576491729785701}),
|
|
torch::tensor(
|
|
{0.15081014600761677, 0.13560816175111742, 1.0971559708365837}),
|
|
torch::tensor(
|
|
{-0.9780975407869251, -1.3215153697157924, -1.876021387605152}),
|
|
torch::tensor({-2.2024413056528886}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{-0.11963097684681223,
|
|
-0.37573675130134543,
|
|
0.0069987166413883715,
|
|
-0.14420855651125972,
|
|
-0.11733423659038758,
|
|
0.8788673419128562}),
|
|
torch::tensor(
|
|
{0.1969829338759005, 0.1973461164047132, 1.1520119567305152}),
|
|
torch::tensor(
|
|
{-1.0677802792431819, -1.4166561260631119, -2.022033753216991}),
|
|
torch::tensor({-2.383452427292781}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{-0.10950806441156272,
|
|
-0.3622226699218595,
|
|
0.02028489243523426,
|
|
-0.1264725422838007,
|
|
-0.10635775660996463,
|
|
0.8936912722040982}),
|
|
torch::tensor(
|
|
{0.23089462331826793, 0.24184450074084418, 1.1904864598387046}),
|
|
torch::tensor(
|
|
{-1.1306213044009719, -1.4837186483578142, -2.122884602514208}),
|
|
torch::tensor({-2.5071352505158395}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{-0.10149090356585248,
|
|
-0.3515172115812867,
|
|
0.030662536099764083,
|
|
-0.11261325211798616,
|
|
-0.09797248308626623,
|
|
0.905027632401109}),
|
|
torch::tensor(
|
|
{0.25777759826689434, 0.2766609657536915, 1.2199973265718322}),
|
|
torch::tensor(
|
|
{-1.1789655573653979, -1.5355073692636774, -2.199612583884608}),
|
|
torch::tensor({-2.600529541471662}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{-0.09484472748389533,
|
|
-0.3426405023243085,
|
|
0.03917399284640637,
|
|
-0.10124188994381228,
|
|
-0.09121264836307835,
|
|
0.9141743475340721}),
|
|
torch::tensor(
|
|
{0.2800829300171032, 0.3052600200290069, 1.2438661306695873}),
|
|
torch::tensor(
|
|
{-1.2182324765944266, -1.5776851394085492, -2.2613704866316295}),
|
|
torch::tensor({-2.6752743361973184}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{-0.08916446117741175,
|
|
-0.33505233521798666,
|
|
0.04638527943959316,
|
|
-0.09160422984057517,
|
|
-0.08556486270584644,
|
|
0.9218219103015535}),
|
|
torch::tensor(
|
|
{0.2991619380154852, 0.3295237551295101, 1.2638639017720827}),
|
|
torch::tensor(
|
|
{-1.251282493526328, -1.6132564639504312, -2.3129529937213853}),
|
|
torch::tensor({-2.73741957239466}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{-0.08420245801272856,
|
|
-0.3284224385121882,
|
|
0.05263847708646642,
|
|
-0.08324438788845245,
|
|
-0.08072424164719598,
|
|
0.9283806476306355}),
|
|
torch::tensor(
|
|
{0.31584087342663564, 0.35059019818200393, 1.2810450644764015}),
|
|
torch::tensor(
|
|
{-1.2798091496372141, -1.6440072538210193, -2.357180462961105}),
|
|
torch::tensor({-2.7905023459395872}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{-0.07979600214534928,
|
|
-0.3225337978155753,
|
|
0.0581562720006689,
|
|
-0.07586555700667826,
|
|
-0.07649523955108037,
|
|
0.9341138824526719}),
|
|
torch::tensor(
|
|
{0.3306627217189733, 0.3692005578577212, 1.2960873917356066}),
|
|
torch::tensor(
|
|
{-1.3048976883823566, -1.6710855742501123, -2.395849898454614}),
|
|
torch::tensor({-2.836765085555123}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{-0.07583232846497832,
|
|
-0.3172360102461862,
|
|
0.06309179259248046,
|
|
-0.06926361352067158,
|
|
-0.07274510848082802,
|
|
0.9392004636935606}),
|
|
torch::tensor(
|
|
{0.3440038606091545, 0.3858647867996722, 1.3094518934419668}),
|
|
torch::tensor(
|
|
{-1.3272851146877218, -1.6952731308502653, -2.4301754289421598}),
|
|
torch::tensor({-2.8777164728823017}),
|
|
},
|
|
};
|
|
}
|
|
|
|
inline std::vector<std::vector<torch::Tensor>> SGD_with_weight_decay() {
|
|
return {
|
|
{
|
|
torch::tensor(
|
|
{-0.21042867144447805,
|
|
-0.49671181653925384,
|
|
-0.13917719856207697,
|
|
-0.3390489907590303,
|
|
-0.2508762913762564,
|
|
0.6985126396619242}),
|
|
torch::tensor(
|
|
{-0.10754881320494518, -0.2910084928862701, 0.6926954859081793}),
|
|
torch::tensor(
|
|
{-0.079932454658518, -0.42109796996670307, -0.33469915794198624}),
|
|
torch::tensor({-0.14248012693079315}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{-0.13579982290274883,
|
|
-0.3765456284475787,
|
|
-0.03166970700350034,
|
|
-0.18102559254681197,
|
|
-0.1373234786735746,
|
|
0.7522156177001302}),
|
|
torch::tensor(
|
|
{0.08550003826014418, 0.051563225553454196, 0.9321399061276381}),
|
|
torch::tensor(
|
|
{-0.796312238882584, -1.1010063686038731, -1.5363716774172782}),
|
|
torch::tensor({-1.8045854907382846}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{-0.09659168723529124,
|
|
-0.30562076936588267,
|
|
0.0067128671455129185,
|
|
-0.1166002367977548,
|
|
-0.09012083166238948,
|
|
0.7264953102453368}),
|
|
torch::tensor(
|
|
{0.16531808496504802, 0.16488328577596398, 0.9610743966573317}),
|
|
torch::tensor(
|
|
{-0.9202466399245914, -1.2052829272891832, -1.7049756710541348}),
|
|
torch::tensor({-2.0415977924493043}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{-0.06728100597713035,
|
|
-0.2496589601654196,
|
|
0.03186158526394668,
|
|
-0.07105441484407878,
|
|
-0.056478595544178806,
|
|
0.6910758436366733}),
|
|
torch::tensor(
|
|
{0.21707768347081777, 0.23575238192099465, 0.9564382346520686}),
|
|
torch::tensor(
|
|
{-0.9788195039029999, -1.2447191597975946, -1.762020156061963}),
|
|
torch::tensor({-2.131504419683077}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{-0.04304955053155505,
|
|
-0.20206572730420902,
|
|
0.050959513946324475,
|
|
-0.034700093557440984,
|
|
-0.02922465201167018,
|
|
0.6547611705604361}),
|
|
torch::tensor(
|
|
{0.2563898231537708, 0.2878867158887637, 0.9414221685252802}),
|
|
torch::tensor(
|
|
{-1.0143969472996655, -1.2623288365082088, -1.7800471460065668}),
|
|
torch::tensor({-2.170255083720924}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{-0.022154717038262738,
|
|
-0.16036518660639862,
|
|
0.06644401410758827,
|
|
-0.004183373274651896,
|
|
-0.005965877978527781,
|
|
0.6200298215101535}),
|
|
torch::tensor(
|
|
{0.2886406829874717, 0.32924516791460257, 0.9230983700837223}),
|
|
torch::tensor(
|
|
{-1.0397895250773481, -1.2710914166240181, -1.7807758009603087}),
|
|
torch::tensor({-2.1862978976514738}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{-0.0037439139848317077,
|
|
-0.12328293308251938,
|
|
0.07944696186805641,
|
|
0.022100305718442022,
|
|
0.014399113804332037,
|
|
0.587697912745227}),
|
|
torch::tensor(
|
|
{0.3162871074692008, 0.36346293565421134, 0.9042402154310412}),
|
|
torch::tensor(
|
|
{-1.060234961430088, -1.2762264965487675, -1.7731268727630662}),
|
|
torch::tensor({-2.191253945056341}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{0.012675985938854726,
|
|
-0.09003711893222131,
|
|
0.09059095692632844,
|
|
0.04506778924310349,
|
|
0.03247299240601001,
|
|
0.5579755127260052}),
|
|
torch::tensor(
|
|
{0.3406226998933173, 0.3924947745885882, 0.8860121369119325}),
|
|
torch::tensor(
|
|
{-1.0781407849705034, -1.2800528898634018, -1.7613120374342215}),
|
|
torch::tensor({-2.190575043873577}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{0.027425440985777993,
|
|
-0.06008809958617219,
|
|
0.10026092920861808,
|
|
0.06531092947039244,
|
|
0.048628754907931976,
|
|
0.5308215072596255}),
|
|
torch::tensor(
|
|
{0.36239744520280553, 0.4175162387638887, 0.8688788105023479}),
|
|
torch::tensor(
|
|
{-1.0946579691370502, -1.283610342226948, -1.7474706191775764}),
|
|
torch::tensor({-2.1870021744944763}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{0.04073250980147411,
|
|
-0.03303024103555013,
|
|
0.1087177047593139,
|
|
0.08324870459183518,
|
|
0.0631222868881554,
|
|
0.5060892094042873}),
|
|
torch::tensor(
|
|
{0.38208249693950175, 0.4393002654989596, 0.8529817924677643}),
|
|
torch::tensor(
|
|
{-1.1103326127955466, -1.287332405916359, -1.73273866274852}),
|
|
torch::tensor({-2.1819672316721337}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{0.05277160918732605,
|
|
-0.008539186625351441,
|
|
0.1161515444487197,
|
|
0.09919929206676087,
|
|
0.07614530177703588,
|
|
0.48359250162323586}),
|
|
torch::tensor(
|
|
{0.3999968617221315, 0.45839442009256354, 0.8383132966805791}),
|
|
torch::tensor(
|
|
{-1.1254107858333455, -1.2913604197768889, -1.717739109221235}),
|
|
torch::tensor({-2.1762368071604308}),
|
|
},
|
|
};
|
|
}
|
|
|
|
inline std::vector<std::vector<torch::Tensor>>
|
|
SGD_with_weight_decay_and_momentum() {
|
|
return {
|
|
{
|
|
torch::tensor(
|
|
{-0.21042867144447805,
|
|
-0.49671181653925384,
|
|
-0.13917719856207697,
|
|
-0.3390489907590303,
|
|
-0.2508762913762564,
|
|
0.6985126396619242}),
|
|
torch::tensor(
|
|
{-0.10754881320494518, -0.2910084928862701, 0.6926954859081793}),
|
|
torch::tensor(
|
|
{-0.079932454658518, -0.42109796996670307, -0.33469915794198624}),
|
|
torch::tensor({-0.14248012693079315}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{0.0056118487251954775,
|
|
-0.0710915563059199,
|
|
0.07701400891926036,
|
|
0.047067327035013866,
|
|
0.0428654052972598,
|
|
0.4352977220593751}),
|
|
torch::tensor(
|
|
{0.23834837300214828, 0.32366382503704183, 0.7128321016634689}),
|
|
torch::tensor(
|
|
{-1.041947788394885, -1.1730950187020548, -1.7648205873351157}),
|
|
torch::tensor({-2.3359277661920594}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{0.11520007183759418,
|
|
0.1289453768763286,
|
|
0.14586845555951963,
|
|
0.1775341535876219,
|
|
0.15614155642578995,
|
|
0.33379126147460536}),
|
|
torch::tensor(
|
|
{0.465853656413685, 0.520197917876909, 0.7274876508280723}),
|
|
torch::tensor(
|
|
{-1.2034746444882527, -1.286126969233868, -1.604528340632377}),
|
|
torch::tensor({-2.203215909196624}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{0.15331258730374997,
|
|
0.197909036233604,
|
|
0.16663814647374195,
|
|
0.2183320498727895,
|
|
0.1803274550482287,
|
|
0.28362745794417826}),
|
|
torch::tensor(
|
|
{0.5532312776994917, 0.5834224152126115, 0.6903579410976886}),
|
|
torch::tensor(
|
|
{-1.3052171323471546, -1.3514190497186434, -1.5153574535010634}),
|
|
torch::tensor({-2.123181139806548}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{0.16814113185552507,
|
|
0.22386572201448868,
|
|
0.17413795101952864,
|
|
0.23280515326261633,
|
|
0.1839142207976228,
|
|
0.2614499495870909}),
|
|
torch::tensor(
|
|
{0.592282876576759, 0.6083877519652824, 0.663438748699906}),
|
|
torch::tensor(
|
|
{-1.3591143274292896, -1.383673065830997, -1.467157893517277}),
|
|
torch::tensor({-2.087859547998447}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{0.1743742243877178,
|
|
0.2343126153059798,
|
|
0.17716942927642254,
|
|
0.23838669643330088,
|
|
0.18308461132092924,
|
|
0.25149544624452974}),
|
|
torch::tensor(
|
|
{0.6108281747800746, 0.6192657661217672, 0.6475519545045926}),
|
|
torch::tensor(
|
|
{-1.3860527054444407, -1.398816664238087, -1.4412527948055516}),
|
|
torch::tensor({-2.0731939075659627}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{0.1771465478751462,
|
|
0.23875859951719522,
|
|
0.1784868271584857,
|
|
0.2406786372566496,
|
|
0.18181103291606765,
|
|
0.24687877342069478}),
|
|
torch::tensor(
|
|
{0.6198586021174767, 0.6242349464856269, 0.638736845373371}),
|
|
torch::tensor(
|
|
{-1.3993307716862977, -1.4058965193851591, -1.42747775986796}),
|
|
torch::tensor({-2.0672675843404598}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{0.17843093585357683,
|
|
0.24073954802700465,
|
|
0.17908697027440873,
|
|
0.2416675839909268,
|
|
0.18088350526559058,
|
|
0.24467193314356378}),
|
|
torch::tensor(
|
|
{0.6243071074374693, 0.6265628975677455, 0.6339840865876518}),
|
|
torch::tensor(
|
|
{-1.4058750036106915, -1.4092362337714568, -1.4202202926903085}),
|
|
torch::tensor({-2.0649062340635584}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{0.17904350645021613,
|
|
0.24165496946247034,
|
|
0.17936920658487726,
|
|
0.24211164489776849,
|
|
0.18031858582735988,
|
|
0.2435923992630521}),
|
|
torch::tensor(
|
|
{0.626513445507806, 0.6276715667697311, 0.6314641991686346}),
|
|
torch::tensor(
|
|
{-1.409113940967948, -1.410830795235453, -1.4164247285253404}),
|
|
torch::tensor({-2.0639728292802046}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{0.17934167113683835,
|
|
0.242089962404631,
|
|
0.17950490408309286,
|
|
0.24231745350706005,
|
|
0.17999989292556767,
|
|
0.2430557755257577}),
|
|
torch::tensor(
|
|
{0.6276131793232345, 0.6282062328090801, 0.6301427155170752}),
|
|
torch::tensor(
|
|
{-1.4107251789010826, -1.4116011824171857, -1.4144511767962422}),
|
|
torch::tensor({-2.0636056316673934}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{0.17948886155124505,
|
|
0.24230096332204806,
|
|
0.17957117450689372,
|
|
0.242415213133214,
|
|
0.17982712042628357,
|
|
0.2427862039224869}),
|
|
torch::tensor(
|
|
{0.6281635672171683, 0.6284667582211864, 0.6294549191500093}),
|
|
torch::tensor(
|
|
{-1.4115305541843781, -1.4119772978756444, -1.4134296522818641}),
|
|
torch::tensor({-2.0634616066978615}),
|
|
},
|
|
};
|
|
}
|
|
|
|
inline std::vector<std::vector<torch::Tensor>>
|
|
SGD_with_weight_decay_and_nesterov_momentum() {
|
|
return {
|
|
{
|
|
torch::tensor(
|
|
{-0.21040617235121148,
|
|
-0.49689727139951717,
|
|
-0.13754215970803657,
|
|
-0.33701686525263036,
|
|
-0.2500172388792182,
|
|
0.700697918175925}),
|
|
torch::tensor(
|
|
{-0.1068708360895515, -0.2853285323043249, 0.6971494161502307}),
|
|
torch::tensor(
|
|
{-0.10624536304143092, -0.4461132561477894, -0.3805647497874434}),
|
|
torch::tensor({-0.2068230782168696}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{-0.1262387113548655,
|
|
-0.3844658218758334,
|
|
0.03124406856508884,
|
|
-0.11170532152425781,
|
|
-0.09823268522398332,
|
|
0.9040698525178972}),
|
|
torch::tensor(
|
|
{0.17551336074135096, 0.27976614792027166, 1.2138399680985128}),
|
|
torch::tensor(
|
|
{-1.592840413595591, -1.8986806244521564, -2.966181914454827}),
|
|
torch::tensor({-3.7728444542017687}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{-0.11614716303292183,
|
|
-0.3709539909720773,
|
|
0.04307078045512772,
|
|
-0.09588329367245825,
|
|
-0.08795603365024904,
|
|
0.9178771227283019}),
|
|
torch::tensor(
|
|
{0.20944042006388683, 0.3195483889401668, 1.2500270348310718}),
|
|
torch::tensor(
|
|
{-1.635011052494502, -1.9463243375558272, -3.035708036973984}),
|
|
torch::tensor({-3.8570351018212796}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{-0.10793942832760066,
|
|
-0.35995697973682966,
|
|
0.05260329955808716,
|
|
-0.08312010825923577,
|
|
-0.07986326997915319,
|
|
0.9287409473303162}),
|
|
torch::tensor(
|
|
{0.2370574459090396, 0.35168415020524857, 1.278618438127574}),
|
|
torch::tensor(
|
|
{-1.669141810658011, -1.984894370767313, -3.091259532917102}),
|
|
torch::tensor({-3.923827025320545}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{-0.1010142826857921,
|
|
-0.35067247612415425,
|
|
0.06058642765135953,
|
|
-0.07242353828264116,
|
|
-0.07320722520220559,
|
|
0.9376663294528951}),
|
|
torch::tensor(
|
|
{0.26037531373638517, 0.37864768429039036, 1.3021925174954938}),
|
|
torch::tensor(
|
|
{-1.6978623668013235, -2.017346013780729, -3.137511248751908}),
|
|
torch::tensor({-3.9791368472670334}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{-0.0950223925827384,
|
|
-0.34263425874631004,
|
|
0.06744912149060932,
|
|
-0.0632219668955612,
|
|
-0.06756850374320933,
|
|
0.9452179348012486}),
|
|
torch::tensor(
|
|
{0.2805629021730173, 0.40186559210837897, 1.322201974233735}),
|
|
torch::tensor(
|
|
{-1.7226667375672964, -2.0453651314263936, -3.177094625235675}),
|
|
torch::tensor({-4.02626353351958}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{-0.08974074058929343,
|
|
-0.3355446553621404,
|
|
0.07346375443579244,
|
|
-0.055152336279101065,
|
|
-0.06268648871001672,
|
|
0.9517469338705254}),
|
|
torch::tensor(
|
|
{0.2983667593362755, 0.42224471824689497, 1.3395523443811077}),
|
|
torch::tensor(
|
|
{-1.7445022249557358, -2.070022023204061, -3.2116640112699977}),
|
|
torch::tensor({-4.0672678681014025}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{-0.08501805567029425,
|
|
-0.32920173535901165,
|
|
0.07881418855733362,
|
|
-0.04796950604202488,
|
|
-0.058388411064112675,
|
|
0.9574862878804136}),
|
|
torch::tensor(
|
|
{0.314293480998118, 0.44039784591234016, 1.3548455497581404}),
|
|
torch::tensor(
|
|
{-1.7640079833055848, -2.092039516337239, -3.2423272727017984}),
|
|
torch::tensor({-4.1035226231344275}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{-0.08074691916762683,
|
|
-0.32346209125355385,
|
|
0.08363041954091821,
|
|
-0.04150011326440421,
|
|
-0.05455400195824525,
|
|
0.9625982669377223}),
|
|
torch::tensor(
|
|
{0.3287029554083447, 0.456758647445438, 1.3685016029692183}),
|
|
torch::tensor(
|
|
{-1.781635969965323, -2.1119291060463254, -3.2698625668054278}),
|
|
torch::tensor({-4.135987715622241}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{-0.07684825926741544,
|
|
-0.3182201487496606,
|
|
0.08800775942949753,
|
|
-0.03561702050647377,
|
|
-0.05109626940874274,
|
|
0.967200308063431}),
|
|
torch::tensor(
|
|
{0.3418602073865105, 0.47164537681262036, 1.3808249543962092}),
|
|
torch::tensor(
|
|
{-1.7977177360253929, -2.1300662313234127, -3.2948372910743537}),
|
|
torch::tensor({-4.165360368453936}),
|
|
},
|
|
{
|
|
torch::tensor(
|
|
{-0.07326215742204903,
|
|
-0.31339589848358795,
|
|
0.09201816976416921,
|
|
-0.030224217178854797,
|
|
-0.0479503174605994,
|
|
0.9713800632469923}),
|
|
torch::tensor(
|
|
{0.35396614509118257, 0.485298524494989, 1.3920431643924076}),
|
|
torch::tensor(
|
|
{-1.8125038126190638, -2.146734711618823, -3.3176778240157505}),
|
|
torch::tensor({-4.192162739857097}),
|
|
},
|
|
};
|
|
}
|
|
|
|
} // namespace expected_parameters
|