U Mh w@sHddlmZmZmZmZmZddlZddlmZddlm Z ddl m Z m Z m Z mZmZmZmZmZmZmZmZmZmZmZmZmZmZddgZGd ddeZd d ed ed e d e d ed e_eeeeeeeeeeeeeeeeee e eee fe e eeeedddZ!eeeeeeeeeeeeeeeeee e eee fe e eeeedddZ"eeeeeeeeeeeeeeeeee e eee fe e eeeeddddZ#ee!ddeeeeeeeeeeeeeeeeeeeeeeeee e ee efe e edddZ$dS))castListOptionalTupleUnionN)Tensor)$_get_fused_kernels_supported_devices)_capturable_doc_default_to_fused_or_foreach_differentiable_doc_disable_dynamo_if_unsupported_dispatch_sqrt _foreach_doc _fused_doc!_get_capturable_supported_devices_get_scalar_dtype _get_value _maximize_doc_stack_if_compiling_use_grad_for_differentiable _view_as_real DeviceDict OptimizerParamsTAdamWadamwcseZdZdddddddeeeefeeefeeeee eeee ed fd d Z fd d Z d dZ e dddZZS)rMbP?g?g+?:0yE>{Gz?FN)maximizeforeach capturabledifferentiablefused) paramslrbetaseps weight_decayamsgradr!r"r#r$r%c s8d|kstd|t|tr0|r0| s0tdd|ksFtd|d|dkr^dksrntd|dd|dkrdksntd |dd|kstd |t|||||||| | | d } t|| | r4| rtd d |_tt fdd|j Ds&tdd|r4tddS)NzInvalid learning rate: Elr as a Tensor is not supported for capturable=False and foreach=TruezInvalid epsilon value: r?z#Invalid beta parameter at index 0: r z#Invalid beta parameter at index 1: zInvalid weight_decay value: ) r'r(r)r*r+r"r!r#r$r%z)`fused` does not support `differentiable`Tc3s2|]*}|dD]}|jjko&t|VqqdS)r&N)devicetypetorchZis_floating_point).0ZpgpZfused_supported_devicesC/var/www/html/venv/lib/python3.8/site-packages/torch/optim/adamw.py Ss z!AdamW.__init__..zX`fused=True` requires all the params to be floating point Tensors of supported devices: .z0`fused` and `foreach` cannot be `True` together.) ValueError isinstancerdictsuper__init__ RuntimeErrorZ_step_supports_amp_scalingrall param_groups) selfr&r'r(r)r*r+r!r"r#r$r%defaults __class__r4r6r=sP    zAdamW.__init__cst||jD]}|dd|dd|dd|dd|dd|dd}|dD]t}|j|g}t|d krft|d sft |d }|ds|drtj |t |d |j d ntj |t d |d <qfqdS)Nr+Fr!r"r#r$r%r&rstepZis_fuseddtyper/rH) r< __setstate__r@ setdefaultstategetlenr1Z is_tensorfloattensorrr/)rArLgroupr%r3Zp_stateZstep_valrCr5r6rJ_s,          zAdamW.__setstate__c Cszd} |dD]f} | jdkrq | t| O} || | jjrFtd|| j|j| } t| dkr|dsx|drtjdt |dd| j d ntj d t d | d <tj | tj d | d<tj | tj d | d<|rtj | tj d | d<|| d|| d|dr|| d|dr:| d jr:td|drft|dtrf|dsftd|| d q | S)NFr&z'AdamW does not support sparse gradientsrr#r%r5rFrGr,rIrE)Z memory_formatexp_avg exp_avg_sqmax_exp_avg_sqr+r$zB`requires_grad` is not supported for `step` in differentiable moder"r'r-)gradr1 is_complexappendZ is_sparser>rLrNZzerosrr/rPZ zeros_likeZpreserve_formatZ requires_gradr:r) rArQparams_with_gradgradsr+exp_avgs exp_avg_sqsmax_exp_avg_sqs state_steps has_complexr3rLr5r5r6 _init_groupvsj             zAdamW._init_groupcCs|d}|dk r.t |}W5QRX|jD]}g}g}g}g}g}g} |d} ttttf|d\} } ||||| |||| } t|||||| | | | |d|d|d|d|d|d |d |d t |d dt |d d| dq4|S)zPerform a single optimization step. Args: closure (Callable, optional): A closure that reevaluates the model and returns the loss. Nr+r(r'r*r)r!r"r#r$r% grad_scale found_inf)r+beta1beta2r'r*r)r!r"r#r$r%r`rar^) Z _cuda_graph_capture_health_checkr1Z enable_gradr@rrrOr_rgetattr)rAclosureZlossrQrXrYrZr[r\r]r+rbrcr^r5r5r6rEs^     z AdamW.step)rrrr F)N)__name__ __module__ __qualname__rrrOrrboolrr=rJr_rrE __classcell__r5r5rCr6rs8   @ Ia Implements AdamW algorithm. .. math:: \begin{aligned} &\rule{110mm}{0.4pt} \\ &\textbf{input} : \gamma \text{(lr)}, \: \beta_1, \beta_2 \text{(betas)}, \: \theta_0 \text{(params)}, \: f(\theta) \text{(objective)}, \: \epsilon \text{ (epsilon)} \\ &\hspace{13mm} \lambda \text{(weight decay)}, \: \textit{amsgrad}, \: \textit{maximize} \\ &\textbf{initialize} : m_0 \leftarrow 0 \text{ (first moment)}, v_0 \leftarrow 0 \text{ ( second moment)}, \: \widehat{v_0}^{max}\leftarrow 0 \\[-1.ex] &\rule{110mm}{0.4pt} \\ &\textbf{for} \: t=1 \: \textbf{to} \: \ldots \: \textbf{do} \\ &\hspace{5mm}\textbf{if} \: \textit{maximize}: \\ &\hspace{10mm}g_t \leftarrow -\nabla_{\theta} f_t (\theta_{t-1}) \\ &\hspace{5mm}\textbf{else} \\ &\hspace{10mm}g_t \leftarrow \nabla_{\theta} f_t (\theta_{t-1}) \\ &\hspace{5mm} \theta_t \leftarrow \theta_{t-1} - \gamma \lambda \theta_{t-1} \\ &\hspace{5mm}m_t \leftarrow \beta_1 m_{t-1} + (1 - \beta_1) g_t \\ &\hspace{5mm}v_t \leftarrow \beta_2 v_{t-1} + (1-\beta_2) g^2_t \\ &\hspace{5mm}\widehat{m_t} \leftarrow m_t/\big(1-\beta_1^t \big) \\ &\hspace{5mm}\widehat{v_t} \leftarrow v_t/\big(1-\beta_2^t \big) \\ &\hspace{5mm}\textbf{if} \: amsgrad \\ &\hspace{10mm}\widehat{v_t}^{max} \leftarrow \mathrm{max}(\widehat{v_t}^{max}, \widehat{v_t}) \\ &\hspace{10mm}\theta_t \leftarrow \theta_t - \gamma \widehat{m_t}/ \big(\sqrt{\widehat{v_t}^{max}} + \epsilon \big) \\ &\hspace{5mm}\textbf{else} \\ &\hspace{10mm}\theta_t \leftarrow \theta_t - \gamma \widehat{m_t}/ \big(\sqrt{\widehat{v_t}} + \epsilon \big) \\ &\rule{110mm}{0.4pt} \\[-1.ex] &\bf{return} \: \theta_t \\[-1.ex] &\rule{110mm}{0.4pt} \\[-1.ex] \end{aligned} For further details regarding the algorithm we refer to `Decoupled Weight Decay Regularization`_. a Args: params (iterable): iterable of parameters to optimize or dicts defining parameter groups lr (float, Tensor, optional): learning rate (default: 1e-3). A tensor LR is not yet supported for all our implementations. Please use a float LR if you are not also specifying fused=True or capturable=True. betas (Tuple[float, float], optional): coefficients used for computing running averages of gradient and its square (default: (0.9, 0.999)) eps (float, optional): term added to the denominator to improve numerical stability (default: 1e-8) weight_decay (float, optional): weight decay coefficient (default: 1e-2) amsgrad (bool, optional): whether to use the AMSGrad variant of this algorithm from the paper `On the Convergence of Adam and Beyond`_ (default: False) z z .. _Decoupled Weight Decay Regularization: https://arxiv.org/abs/1711.05101 .. _On the Convergence of Adam and Beyond: https://openreview.net/forum?id=ryQu7f-RZ )r&rYrZr[r\r]r`rar+rbrcr'r*r)r!r#r$r^c !Cs|dkr|dksttjr,t| ts,tt|D]p\}}|sJ||n|| }||}||}||}tjs|rt }|j j |j j kr|j j |kstd|dt |rt |}t |}t |}|rt ||||<t |}|d7}|d| | ||d| || j||d| d|sD|r|}d| |}d| |}| |}|}|}|r|r||}n||}||t||||||| |} n|||| |} ||| nt|}d| |}d| |}| |}t|}|r^tj|||||d|||| } n||| } |j|| | d|r4t ||r4t||||<q4dS)NIIf capturable=True, params and state_steps must be on supported devices: r8r )value)out)AssertionErrorr1jit is_scriptingr:rO enumerate_utils is_compilingrr/r0rVZ view_as_realZmul_Zlerp_Zaddcmul_negsqrtcloneZcopy_maximumZadd_Zaddcdiv_rrZview_as_complex)!r&rYrZr[r\r]r`rar+rbrcr'r*r)r!r#r$r^iparamrUrRrSZstep_tcapturable_supported_devicesrEbias_correction1bias_correction2 step_sizeZ step_size_negbias_correction2_sqrtrTdenomr5r5r6_single_tensor_adamwBsx             rc st|dkrdSttr&|s&tdtjsj|rjtddtfddt ||Dsjt dd|rvt d |dkr|dkst t ||||||g}| D],\\}}}}}}}|r|rt|||||nt|||||rt|}|djrtj|tjd d d d d n t|d| dkrFt|d| t||dt|t|||d~|r>t|}t|}t|dt|dt|t|t|t||}|}|rt||t|}n t|}t||t|| t||t|||qfdd|D}fdd|D}tfdd|D}dd|D}|rt||t|}n t|}t||t|| t||||qdS)Nrr-F)Z supports_xlac3s.|]&\}}|jj|jjko$|jjkVqdSN)r/r0)r2r3rE)rzr5r6r7sz&_multi_tensor_adamw..rkr8z#_foreach ops don't support autogradr.cpu)r/)alphar csg|]}dt|qSr rr2rE)rbr5r6 Dsz'_multi_tensor_adamw..csg|]}dt|qSrrr)rcr5r6rGscsg|]}|dqS)r5r2bc)r'r5r6rKscSsg|] }t|qSr5)rrr5r5r6rMs) rNr:rr>r1rrrsrr?ziprnr"_group_tensors_by_device_and_dtypevaluesrZ _foreach_negZis_cpu _foreach_add_rPZ _foreach_mul_Z_foreach_lerp_Z_foreach_addcmul_Z _foreach_pow _foreach_sub_Z _foreach_neg_Z _foreach_div_Z_foreach_reciprocal_Z_foreach_sqrt_Z_foreach_maximum_Z _foreach_sqrtZ_foreach_addcdiv_r)r&rYrZr[r\r]r`rar+rbrcr'r*r)r!r#r$r^grouped_tensors device_params device_gradsdevice_exp_avgsdevice_exp_avg_sqsdevice_max_exp_avg_sqsdevice_state_steps_r{r|r}r~Zexp_avg_sq_sqrtr5)rbrcrzr'r6_multi_tensor_adamws                                  r)r&rYrZr[r\r]r`rar+rbrcr'r*r)r!r#r$r^returnc Csf|sdS|rtd|dk r&|j|ini}|dk r<|j|ini}t| trbt| jdkrb| j| ind}t||||||g}|D]\\}}\\}}}}}}}d\}}|dk r|||j |dd}|dk r|||j |dd}|dk r||kr||| j |dd} t |dt j |||||||| | | | | |||d|dk rt ||gt|qdS) Nz9Adam with fused=True does not support differentiable=Truer)NNT) non_blocking)r/rr ) r+r'rbrcr*r)r!r`ra)r>r/r:rstrrritemsrKtor1rZ _fused_adamw_rrN) r&rYrZr[r\r]r`rar+rbrcr'r*r)r!r#r$r^Zgrad_scale_dictZfound_inf_dictZlr_dictrr/rrrrrrrZdevice_grad_scaleZdevice_found_infr5r5r6 _fused_adamwds|$      r)Zsingle_tensor_fnF)r&rYrZr[r\r]r"r#r$r%r`rar^r+rbrcr'r*r)r!c Cstjs$tdd|Ds$td| dkr\|dkr\t||dd\}}|r\t|tr\|s\d}| dkrhd} |dkrtd}|rtj rtd| rtj rtd| rtj st }n|rtj st }nt }|||||||| ||||||||| | | d dS) zpFunctional API that performs AdamW algorithm computation. See :class:`~torch.optim.AdamW` for details. css|]}t|tjVqdSr)r:r1r)r2tr5r5r6r7szadamw..zPAPI has changed, `state_steps` argument must contain a list of singleton tensorsNF)Z use_fusedz6torch.jit.script not supported with foreach optimizersz4torch.jit.script not supported with fused optimizers) r+rbrcr'r*r)r!r#r$r`rar^) r1rrrsr?r>r r:rrorprrr)r&rYrZr[r\r]r"r#r$r%r`rar^r+rbrcr'r*r)r!rfuncr5r5r6rs^ )NFFNNNF)%typingrrrrrr1rZtorch.utils._foreach_utilsrZ optimizerr r r r rrrrrrrrrrrrr__all__r__doc__rirOrrrrr5r5r5r6s  La'F  x  -  Z