#?coding=utf-8
import?torch
from?torch.optim?import?Optimizer

class?AdamWeightDecayOptimizer（Optimizer）:
????“““A?basic?Adam?optimizer?that?includes?“correct“?L2?weight?decay.
????https://github.com/google-research/bert/blob/master/optimization.py
????https://raw.githubusercontent.com/pytorch/pytorch/v1.0.0/torch/optim/adam.py“““
????def?__init__（self?params?lr=1e-3?betas=（0.9?0.999）?eps=1e-8
?????????????????weight_decay=0?amsgrad=False）:
????????if?not?0.0?<=?lr:
????????????raise?ValueError（“Invalid?learning?rate:?{}“.format（lr））
????????if?not?0.0?<=?eps:
????????????raise?ValueError（“Invalid?epsilon?value:?{}“.format（eps））
????????if?not?0.0?<=?betas[0]?????????????raise?ValueError（“Invalid?beta?parameter?at?index?0:?{}“.format（betas[0]））
????????if?not?0.0?<=?betas[1]?????????????raise?ValueError（“Invalid?beta?parameter?at?index?1:?{}“.format（betas[1]））
????????defaults?=?dict（lr=lr?betas=betas?eps=eps
????????????????????????weight_decay=weight_decay?amsgrad=amsgrad）
????????super（AdamWeightDecayOptimizer?self）.__init__（params?defaults）

????def?__setstate__（self?state）:
????????super（AdamWeightDecayOptimizer?self）.__setstate__（state）
????????for?group?in?self.param_groups:
????????????group.setdefault（‘amsgrad‘?False）

????def?step（self?closure=None）:
????????“““Performs?a?single?optimization?step.

????????Arguments:
????????????closure?（callable?optional）:?A?closure?that?reevaluates?the?model
????????????????and?returns?the?loss.
????????“““
????????loss?=?None
????????if?closure?is?not?None:
????????????loss?=?closure（）

????????for?group?in?self.param_groups:
????????????for?p?in?group[‘params‘]:
????????????????if?p.grad?is?None:
????????????????????continue
????????????????grad?=?p.grad.data
????????????????if?grad.is_sparse:
????????????????????raise?RuntimeError（‘Adam?does?not?support?sparse?gradients?please?consider?SparseAdam?instead‘）
????????????????amsgrad?=?group[‘amsgrad‘]

????????????????state?=?self.state[p]

????????????????#?State?initialization
????????????????if?len（state）?==?0:
????????????????????state[‘step‘]?=?0
????????????????????#?Exponential?moving?average?of?gradient?values
????????????????????state[‘exp_avg‘]?=?torch.zeros_like（p.data）
????????????????????#?Exponential?moving?average?of?squared?gradient?values
????????????????????state[‘exp_avg_sq‘]?=?torch.zeros_like（p.data）
????????????????????if?amsgrad:
????????????????????????#?Maintains?max?of?all?exp.?moving?avg.?of?sq.?grad.?values
????????????????????????state[‘max_exp_avg_sq‘]?=?torch.zeros_like（p.data）

????????????????exp_avg?exp_avg_sq?=?state[‘exp_avg‘]?state[‘exp_avg_sq‘]
????????????????if?amsgrad:
????????????????????max_exp_avg_sq?=?state[‘max_exp_avg_sq‘]
????????????????beta1?beta2?=?group[‘betas‘]

????????????????state[‘step‘]?+=?1

????????????????#?Decay?the?first?and?second?moment?running?average?coefficient
????????????????exp_avg.mul_（beta1）.add_（1?-?beta1?grad）
????????????????exp_avg_sq.mul_

?屬性????????????大小?????日期????時間???名稱
-----------?---------??----------?-----??----
?????目錄???????????0??2019-03-15?10:01??BERT-master\
?????文件??????????66??2019-03-15?10:01??BERT-master\.gitattributes
?????文件????????1316??2019-03-15?10:01??BERT-master\.gitignore
?????文件????????1430??2019-03-15?10:01??BERT-master\README.md
?????文件????????4134??2019-03-15?10:01??BERT-master\adam.py
?????文件????????5522??2019-03-15?10:01??BERT-master\bert.py
?????文件????????4687??2019-03-15?10:01??BERT-master\data.py
?????文件????????7028??2019-03-15?10:01??BERT-master\example_use.py
?????文件?????????292??2019-03-15?10:01??BERT-master\example_use.sh
?????文件????????9862??2019-03-15?10:01??BERT-master\google_bert.py
?????文件????????2754??2019-03-15?10:01??BERT-master\preprocess.py
?????目錄???????????0??2019-03-15?10:01??BERT-master\toy\
?????文件?????????323??2019-03-15?10:01??BERT-master\toy\gen.py
?????文件??????193086??2019-03-15?10:01??BERT-master\toy\sample_from_zhwiki
?????文件????????1883??2019-03-15?10:01??BERT-master\toy\train
?????文件?????????110??2019-03-15?10:01??BERT-master\toy\vocab
?????文件????????6682??2019-03-15?10:01??BERT-master\train.py
?????文件?????????831??2019-03-15?10:01??BERT-master\train.sh
?????文件???????10717??2019-03-15?10:01??BERT-master\transformer.py
?????文件????????1047??2019-03-15?10:01??BERT-master\utils.py