U Mh<@s|ddlmZmZmZmZddlZddlmZddlmZm Z m Z m Z m Z m Z mZmZmZmZmZmZmZddgZGdddeZd e d ed e d ed e_eeeeeeeeeeeeeeeeeeeeed d dZeeeeeeeeeeeeeeeeeeeeed ddZe eddeeeeeeeeeeeeeeeeeeeeeeedddZdS))ListOptionalTupleUnionN)Tensor) _capturable_doc_default_to_fused_or_foreach_differentiable_doc_disable_dynamo_if_unsupported _foreach_doc!_get_capturable_supported_devices_get_scalar_dtype _get_value _maximize_doc_use_grad_for_differentiable _view_as_real OptimizerParamsTASGDasgdc s\eZdZdeeeeeeeeeeed fd d Zfd d Zd dZ e dddZ Z S)r{Gz?-C6??.ArNF) paramslrlambdalphat0 weight_decayforeachmaximizedifferentiable capturablec sXd|kstd|d|ks,td|t|||||||| | d } t|| dS)NgzInvalid learning rate: zInvalid weight_decay value: ) rrrrr r!r"r#r$) ValueErrordictsuper__init__) selfrrrrrr r!r"r#r$defaults __class__B/var/www/html/venv/lib/python3.8/site-packages/torch/optim/asgd.pyr(s  z ASGD.__init__cst||jD]}|dd|dd|dd|dd|dD]}|j|g}t|dkrNt|dst |d}tj |t |j d |d<t|d stj |d t |j d |d <t|d sNtj |d t |j d |d <qNqdS) Nr!r"Fr#r$rrstep)dtypedeviceetamu) r' __setstate__ param_groups setdefaultstategetlentorchZ is_tensorfloattensorrr1)r)r7grouppZp_stateZstep_valr+r-r.r4:s8           zASGD.__setstate__c Cs d}|dD]} | jdk r |t| O}|| | jjrBtd|| j|j| } t| dkrtjd| j t d| d<tj |d| j t d | d <tjd| j t d| d <tj| tjd | d <|| d || d || d || dq |S) NFrz&ASGD does not support sparse gradientsrr-)r1r0r/rr2r3)Z memory_formatax)gradr: is_complexappendZ is_sparse RuntimeErrorr7r9Zzerosr1r as_tensorclonedetachZonesZ zeros_likeZpreserve_format) r)r=params_with_gradgradsmusaxsetas state_steps has_complexr>r7r-r-r. _init_groupRsF          zASGD._init_groupc Cs|d}|dk r.t |}W5QRX|jD]~}g}g}g}g}g}g} |||||||| } t|||||| |d|d|d|d|d|d|d|d |d | d q4|S) zPerform a single optimization step. Args: closure (Callable, optional): A closure that reevaluates the model and returns the loss. Nrrrrr r!r"r#r$) rrrrr r!r"r#r$rM)Z _cuda_graph_capture_health_checkr:Z enable_gradr5rNr) r)closureZlossr=rGrHrIrJrKrLrMr-r-r.r/vsP  z ASGD.step) rrrrrNFFF)N) __name__ __module__ __qualname__rr;rboolr(r4rNrr/ __classcell__r-r-r+r.rs2 $ahImplements Averaged Stochastic Gradient Descent. It has been proposed in `Acceleration of stochastic approximation by averaging`_. Args: params (iterable): iterable of parameters to optimize or dicts defining parameter groups lr (float, optional): learning rate (default: 1e-2) lambd (float, optional): decay term (default: 1e-4) alpha (float, optional): power for eta update (default: 0.75) t0 (float, optional): point at which to start averaging (default: 1e6) weight_decay (float, optional): weight decay (L2 penalty) (default: 0) z zx .. _Acceleration of stochastic approximation by averaging: https://dl.acm.org/citation.cfm?id=131098 )rrHrJrIrKrLrrrrr r"r#r$rMc  Cst|D]\}}||}| s"|n| }||}||}||}||}tjs| rt}|jj|jjkr|jjkr|jjkrnn |jj|kstd|dt|rt |}t |}t |}|d7}| dkr|j || d}| r| d|||j ||ddn*t |}| d|||j|| d| sX|dkrp||| |n ||| r||d|||| |dt||t|qt |}t|d|||| }||tdtd||}||qdS)NUIf capturable=True, params, mus, etas, and state_steps must be on supported devices: .rrrvalue) enumerater:_utils is_compilingr r1typeAssertionErrorrAZ view_as_realaddZmul_Zaddcmul_rZadd_itemsubZcopy_maximumZ ones_likerDmax)rrHrJrIrKrLrrrrr r"r#r$rMiparamr@r3r?r2Zstep_tcapturable_supported_devicesZ eta_valuer/Znew_etaZnew_mur-r-r._single_tensor_asgdsX        " rhc sLt|dkrdS| rtdtjsd| rdtddtfddt||||Dsdtddt ||||||g}| D]\\}\\}}}}}}}|rt |||| rt |}|dj rtj|tjd d d d d n t|d | dkr8| rtj||| d |}ntj||| d }tj||d ntj||d }tj|||dd~t||}t|||~| rt|}t|d t|t||~t|}t|t|d t|t|t|t||qfdd|D}fdd|D}t||t||qdS)Nrz#_foreach ops don't support autogradF)Z supports_xlac3sT|]L\}}}}|jj|jjko:|jjko:|jjknoJ|jjkVqdS)N)r1r^).0r>r3r2r/)rgr-r. 's 2z%_multi_tensor_asgd..rUrVg?cpur1rWrrXrYcs.g|]&}tjd|dqSrrl)r:rDrir/)rr1rrr-r. sz&_multi_tensor_asgd..cs,g|]$}tjdtdt|dqSrm)r:rDrdrrn)r1rr-r.ros)r9r_r:r\r]r allziprZ"_group_tensors_by_device_and_dtypeitemsrZ _foreach_negZis_cpuZ _foreach_add_r<Z _foreach_addZ_foreach_addcmul_Z _foreach_subZ_foreach_maximum_Z_foreach_reciprocal_Z_foreach_copy_Z _foreach_mulZ _foreach_mul_Z _foreach_pow_)rrHrJrIrKrLrrrrr r"r#r$rMZgrouped_tensors_Zgrouped_paramsZ grouped_gradsZ grouped_axsZ grouped_musZ grouped_etasZgrouped_state_stepsZ intermediateZnew_musZnew_etasr-)rrgr1rrrr._multi_tensor_asgd s                         rt)Zsingle_tensor_fnF)rrHrJrIrKrLr!r"r#r$rMrrrrr c Csr|dkrt||dd\}}|r0tjr0td|rDtjsDt}nt}|||||||| | | ||||| | ddS)znFunctional API that performs asgd algorithm computation. See :class:`~torch.optim.ASGD` for details. NF)Z use_fusedz6torch.jit.script not supported with foreach optimizers) rrrrr r"r#r$rM)r r:ZjitZ is_scriptingrCrtrh)rrHrJrIrKrLr!r"r#r$rMrrrrr rsfuncr-r-r.rs8 )NFFFF)typingrrrrr:rZ optimizerrr r r r r rrrrrrr__all__r__doc__r;rSrhrtrr-r-r-r.s <   M