U
    yhI                     @   s   d dl Z d dlZd dlmZ d dlmZ d dlmZ d dlm	Z	m
Z
mZmZmZ d dlZd dlmZ d dlm  m  mZ d dlmZmZmZ e eZG dd dZejjee e	e ee  f f d	d
dZ!dS )    N)defaultdict)contextmanager)Enum)DictIteratorListSetTuple)_apply_to_modules_get_module_fsdp_stateclean_tensor_namec                   @   s   e Zd ZU G dd deeZeeZe	eef e
d< e Zee e
d< eddddZeeeed d	d
dZeeddddZdS )SimpleProfilerc                   @   s$   e Zd ZdZdZdZdZdZdZdS )zSimpleProfiler.TypeallZ
all_gatherZall_gather_objectZ
reshardingH2DD2HN)	__name__
__module____qualname__ZALLZ	ALLGATHERZALLGATHER_OBJZ
RESHARDINGr   r    r   r   U/var/www/html/venv/lib/python3.8/site-packages/torch/distributed/fsdp/_debug_utils.pyType   s   r   results	profilingN)returnc                 C   s   | j   | j  d S )N)r   clearr   )clsr   r   r   reset!   s    
zSimpleProfiler.reset)profile_typer   c                 c   sh   || j kst| d| j | t }z
d V  W 5 t }| j|  || 7  < | j | X d S )Nzk is already being profiled. SimpleProfiler does not support profiling multiple instances at the same time. )r   AssertionErroraddtime	monotonicr   remove)r   r   beginendr   r   r   profile&   s    
zSimpleProfiler.profile)msgr   c                 C   s8   t  dkr,t  t jjkr,td|| j |   d S )Nr   z%s %s)	distZget_rankZget_debug_levelZ
DebugLevelINFOloggerinfor   r   )r   r&   r   r   r   dump_and_reset8   s    zSimpleProfiler.dump_and_reset)r   r   r   strr   r   r   floatr   r   __annotations__setr   r   classmethodr   r   r   r%   r+   r   r   r   r   r      s   
r   )modelr   c                 C   s:   dd }dd }dg}i }t | ||dd |  D ||S )a  
    It is used for composable fully_shard() code path, it returns
      1. sharded module tree info: each line reprents a submodule name that contats the
    submodule's FQN and its submodule class name, if the submodule is sharded by `fully_shard`,
    the submodule name will add a postfix with ' FULLY SHARDED'. Each increased tree
    level adds 4 spaces before the printed name. A printed sharded module tree info for a toy model
    is like this:
        [CompositeModel] FULLY SHARDED
            l1[Linear]
            u1[UnitModule] FULLY SHARDED
                u1.l1[Linear]
                u1.seq[Sequential]
                    u1.seq.0[ReLU]
                    u1.seq.1[Linear]
                    u1.seq.2[ReLU]
                u1.l2[Linear]
            u2[UnitModule] FULLY SHARDED
                u2.l1[Linear]
                u2.seq[Sequential]
                    u2.seq.0[ReLU]
                    u2.seq.1[Linear]
                    u2.seq.2[ReLU]
                u2.l2[Linear]
            l2[Linear]
      2. a dict mapping from the concated module FQN and class name to a list of its managed
    original parameters' FQNs. An example of the dict for the above toy sharded model is like this:
            {'[CompositeModel]': ['l1.weight', 'l1.bias', 'l2.weight', 'l2.bias'],
             'u1[UnitModule]': ['u1.l1.weight', 'u1.l1.bias', 'u1.seq.1.weight', 'u1.seq.1.bias', 'u1.l2.weight', 'u1.l2.bias'],
             'u2[UnitModule]': ['u2.l1.weight', 'u2.l1.bias', 'u2.seq.1.weight', 'u2.seq.1.bias', 'u2.l2.weight', 'u2.l2.bias']
            }
    All FQNs are prefixed starting from ``model``.

    Args:
        model (torch.nn.Module): Root module (which may or may not be passed to
                                 composable `fully_shard()`).
    c                    s  |d }t  dkr, d dkr, d d n }|d | jj d }d| | }t| }	|	d krx|d  |d 7  < d S |	j| d }
|
r|d  |d	 d 7  < n|d  |d 7  < |
r
|
j}t|tj	st
 fd
d|jD }||kr|| | n|||< d S )N   r   .[] 
z FULLY SHARDEDc                    s   g | ]}t  | qS r   )r   ).0nameprefixr   r   
<listcomp>   s    zX_get_sharded_module_tree_with_module_name_to_fqns.<locals>.module_fn.<locals>.<listcomp>)len	__class__r   r   Z_fully_sharded_module_to_handlegetZ
flat_param
isinstanceflat_param_fileZFlatParameterr   Z_fqnsextend)moduler<   Z
tree_levelsharded_tree_infosharded_module_name_to_fqns
num_spacesZtrimed_prefixZprefixed_module_nameZprinted_prefixed_module_namestatehandleparamZglobal_fqnsr   r;   r   	module_fni   s0    &


zD_get_sharded_module_tree_with_module_name_to_fqns.<locals>.module_fnc                 S   s   | d |fS )Nr   r   )rE   rF   r   r   r   	return_fn   s    zD_get_sharded_module_tree_with_module_name_to_fqns.<locals>.return_fn c                 S   s   g | ]\}}|qS r   r   )r9   key_r   r   r   r=      s     zE_get_sharded_module_tree_with_module_name_to_fqns.<locals>.<listcomp>)r
   Znamed_parameters)r1   rK   rL   rE   rF   r   r   r   1_get_sharded_module_tree_with_module_name_to_fqnsA   s    ($rP   )"loggingr    collectionsr   
contextlibr   enumr   typingr   r   r   r   r	   ZtorchZtorch.distributeddistributedr'   Z"torch.distributed.fsdp._flat_paramZfsdpZ_flat_paramrB   Z$torch.distributed.fsdp._common_utilsr
   r   r   	getLoggerr   r)   r   nnModuler,   rP   r   r   r   r   <module>   s   
-