U
    yh#                     @   s   d dl Z d dlmZmZmZmZmZmZmZm	Z	m
Z
 d dlZd dlmZ d dlmZ d dlmZ ddlmZmZ dZG dd	 d	eZeeed
ddZG dd dZeeddejeeejj  ejdddZejedddZdS )    N)	AnycastDictIterableListNoReturnOptionalSetTuple)_State)DistributedDataParallel   )_get_registrycontract c                       s   e Zd Zdd fddZefejeej eej e	ddddZ
dddd	Zejeej dd
ddZddddZddddZejeedf ee	ef edddZejeej ejejdddZ  ZS )_ReplicateStateNreturnc                    sN   t    t | _d| _t | _| j| _g | _d| _	d | _
i | _g | _d S )NF)super__init__nnZParameterListmodulehas_initialized_param_listZ_orig_module_param_names_no_sync
_init_args_init_kwargs_comm_hook_argsself	__class__ Y/var/www/html/venv/lib/python3.8/site-packages/torch/distributed/_composable/replicate.pyr      s    


z_ReplicateState.__init__)r   ignored_modulesignored_paramsprefixr   c           
      C   s   t |rd S ||krd S |tkr*| dnt}|jddD ]0\}}||kr:| j| | j| |  q:| D ]"\}}	| j|	||| | d qtd S )N.F)recurse)r'   )_is_fully_sharded_ROOT_MODULE_PREFIXZnamed_parametersr   appendr   Znamed_children_collect_params)
r    r   r%   r&   r'   Zrecurse_prefixnpnameZchild_moduler#   r#   r$   r-      s"    
z_ReplicateState._collect_paramsc                    s"   t jdd fdd}|  d S )NT)	recursivec                      s8    j d k	st j j  j    t  _ i  _d S N)r   AssertionErrorinitr   register_comm_hooktupler#   r   r#   r$   
_lazy_init?   s
    z-_ReplicateState.lazy_init.<locals>._lazy_init)torchZ_disable_dynamo)r    r7   r#   r   r$   	lazy_init>   s    
z_ReplicateState.lazy_initr   r%   r   c                 K   s   | j r
d S d| _ |dd }|| _dd |D }ddlm} || | ||| d|kr|d d k	r|d }t|tjr|j	dkrd |d	< q|g|d	< nd |d	< |
d t| jf|| _t| jt| j_d S )
NTdevice_meshc                 S   s   h | ]}|  D ]}|qqS r#   )
parameters).0mr/   r#   r#   r$   	<setcomp>V   s     
  z'_ReplicateState.init.<locals>.<setcomp>r   )_localize_dtensor	device_idcpuZ
device_ids)r   getr   %torch.distributed.tensor.parallel.ddpr@   r-   
isinstancer8   devicetypepopr   r   _ddpweakrefref	replicatestateZ_ddp_weakref)r    r   r%   kwargsr;   r&   r@   rA   r#   r#   r$   r4   I   s&    

z_ReplicateState.initc                 C   s,   | j D ]\}}| jj|| q| j   d S r2   )r   rI   r5   clear)r    Z	comm_argsZcomm_kwargsr#   r#   r$   r5   r   s    z"_ReplicateState.register_comm_hookc                 O   s   || _ || _d S r2   )r   r   r    argsrN   r#   r#   r$   record_init_argsw   s    z _ReplicateState.record_init_args.)r   rQ   rN   r   c                 C   s.   | j s| jr|   | j | j_| jj||S r2   )r   r   r9   r   rI   Zrequire_backward_grad_syncZ_pre_forward)r    r   rQ   rN   r#   r#   r$   forward_pre_hook{   s    z _ReplicateState.forward_pre_hook)r   inputoutputr   c                 C   s   | j |S r2   )rI   Z_post_forward)r    r   rT   rU   r#   r#   r$   forward_post_hook   s    z!_ReplicateState.forward_post_hook)__name__
__module____qualname__r   r+   r   Moduler	   	Parameterstrr-   r9   r4   r5   rR   r
   r   r   rS   r8   ZTensorrV   __classcell__r#   r#   r!   r$   r      s4    ) 
 

r   )rQ   rN   r   c                  O   s   t dd S )NzGDDP does not support deepcopy. Please use state dict for serialization.)r3   )rQ   rN   r#   r#   r$   unimplemented_deepcopy   s    r^   c                   @   s2   e Zd Zdd ZeddddZdddd	ZdS )
DDPc                 O   s   | j d }|j|f||S )z
        Override ``__new__`` to remove the DDP class and directly construct
        the original class for cases like indexing into a container module.
           )__mro____new__)clsrQ   rN   Zorig_clsr#   r#   r$   rb      s    
zDDP.__new__N)requires_gradient_syncr   c                 C   s   | t | _dS )a  
        Sets if the module should sync gradients. This can be used to implement
        gradient accumulation without communication.

        Args:
            requires_gradient_sync (bool): Whether to reduce gradients for the
                module's parameters.
        N)rL   rM   r   )r    rd   r#   r#   r$   set_requires_gradient_sync   s    	zDDP.set_requires_gradient_syncr   c                 O   s   t | j||f d S r2   )rL   rM   r   r,   rP   r#   r#   r$   r5      s    zDDP.register_comm_hook)rW   rX   rY   rb   boolre   r5   r#   r#   r#   r$   r_      s   
r_   )Z	state_clsr:   c                 K   s"  t jd d|kr>t|d tt jfs>tdt|d  t| rNtd|dkr\i }nt	|}t
tt| }| j|jdd |dd}|dk	rd	d
lm} ||dk	rd	dlm}m} | | | | | |j |j| |f| | j}dti}	td|j t|f|	}
|
| _| S )zReplicates a module

    Args:
        module (torch.nn.Module): module to replicate

    Example::
        >>> # xdoctest: +REQUIRES(module:torch._C._distributed_c10d)
        >>> module = nn.Linear(3, 3)
        >>> replicate(module)
    ztorch.distributed.replicaterA   z6Expected device_id to be int or torch.device, but got zGCannot apply `replicate()` on a Module already managed by `fully_shard`NT)Zwith_kwargsr;   r   )_mesh_resources)r@   _reconstruct_dtensor__deepcopy__r_   )r8   Z_CZ_log_api_usage_oncerE   intrF   RuntimeErrorrG   r*   setr   r   rL   rM   Zregister_forward_pre_hookrS   rC   Ztorch.distributed.device_meshrg   Zget_parent_meshrD   r@   rh   Zregister_forward_hookrV   rR   r"   r^   rW   r_   )r   r%   rN   rM   r;   rg   r@   rh   rc   dctZnew_clsr#   r#   r$   rL      s:    

rL   )r   r   c                 C   s   t | }|dkrdS d|kS )z+Check if module is marked with fully_shard.NFZfully_shard)r   )r   registryr#   r#   r$   r*      s    r*   )N)rJ   typingr   r   r   r   r   r   r   r	   r
   r8   Ztorch.nnr   Z#torch.distributed._composable_stater   Ztorch.nn.parallelr   r   r   r+   r   r^   r_   rZ   rL   rf   r*   r#   r#   r#   r$   <module>   s$   ,} E