U yh@sjUddlmZmZmZmZddlZddlmmZ ddlm Z gZ ee e d<ejjGdddZdS))DictListOptionalTupleN)Tensor__all__c @sdeZdZdeeeeeefeeeeeeed ddZee ed d d Z ee ed d dZ dS)_FunctionalAdamWMbP?g?g+?:0yE>{Gz?F) paramslrbetaseps weight_decayamsgradmaximizeforeachfused_allow_empty_param_listc Csd|kstd|d|ks,td|d|dkrDdksXntd|dd|dkrpdksntd|dd|kstd ||||d|d|d |_||_||_||_| |_tjt tj t t tj ffi|_ t |dkr | s td d |i|_dS) NzInvalid learning rate: zInvalid epsilon value: rg?z#Invalid beta parameter at index 0: z#Invalid beta parameter at index 1: zInvalid weight_decay value: )rrbeta1beta2rz%optimizer got an empty parameter listr ) ValueErrordefaultsrrrrtorchjitZannotaterrstrstatelen param_group) selfr rrrrrrrrrr$Z/var/www/html/venv/lib/python3.8/site-packages/torch/distributed/optim/functional_adamw.py__init__s0 $z_FunctionalAdamW.__init__)paramgradc Cs`g}g}g}g}g}g}t|} |dk r>||||||jkri|j|<|j|} td| d<tj|tjd| d<tj|tjd| d<|jrtj|tjd| d<|j|} || d|| d|jr|| d|| dtZt j |||||||j|j |j d|j d|j d |j d |j d |j |jdd| d W5QRXdS) NrstepZ memory_formatexp_avg exp_avg_sqmax_exp_avg_sqrrrrr rrrrrrrrrZ grad_scaleZ found_inf has_complex)r is_complexappendr tensor zeros_likepreserve_formatrno_gradFadamwrrrr) r#r'r(params_with_gradgradsexp_avgs exp_avg_sqsmax_exp_avg_sqs state_stepsr/r r$r$r% step_paramBsl           z_FunctionalAdamW.step_param) gradientsc Cs|jd}g}g}g}g}g}g}t|t|krXtddt|ddt|d} t|jd|D]\} } | dk rl| t| O} || || | |jkri|j| <|j| } td| d<tj | tj d | d <tj | tj d | d <|j rtj | tj d | d <|j| } || d || d |j rL|| d || dqlt Zt j|||||||j |j|jd |jd|jd|jd|jd|j|jdd| dW5QRXdS)Nr zEthe gradients passed in does not equal to the size of the parameters!zParams length: z. zGradients length: Frr)r*r+r,r-rrrrrr.)r"r!rziprr0r1r r2r3r4rr5r6r7rrrr) r#r?r r8r9r:r;r<r=r/r'Zgradientr r$r$r%r)s            z_FunctionalAdamW.stepN) r r r r FFFFF) __name__ __module__ __qualname__rrfloatrboolr&rr>r)r$r$r$r%rs.  ,=r)typingrrrrrZtorch.optim._functionalZoptimZ _functionalr6rrr__annotations__rscriptrr$r$r$r%s