U
    yhG                     @   s:  U d dl Z d dlZd dlZd dlZd dlmZ d dlmZ d dlm	Z	m
Z
mZmZmZmZmZ d dlZd dlmZ d dlmZ d dlmZ d dlmZmZ d d	lmZ d d
lmZ d dlmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z& d dl'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z. d dl/m0Z0m1Z1m2Z2m3Z3 d dl4m5Z5 e6e7Z8ej9e:d< ddddddgZ;G dd de,Z<G dd de)Z=G dd de=Z>d/ee?e	f e!e@e(dddZAee( ee( dddZBee?e	f e@e+dd dZCd0ee+ e@eee+ e!f d!d"dZDe#e!d#d$d%ZEe e e@d&d'd(ZFejGe e@d)d*d+ZHee+ e!e@d,d-d.ZIdS )1    N)ChainMap)reduce)AnycastDictListOptionalTupleUnion)narrow_tensor_by_index)DTensor)dedup_save_plans)FLATTEN_MAPPINGflatten_state_dict)_flatten_sharded_tensors)set_element)BytesStorageMetadataChunkStorageMetadataMetadataMetadataIndexSTATE_DICT_TYPESTORAGE_TYPESStorageMetaTensorStorageMetadata)LoadPlanLoadPlannerReadItemSavePlanSavePlanner	WriteItemWriteItemType)"_create_default_metadata_only_plan_create_read_items_create_write_items_init_state_dict)find_state_dict_objectloggerDefaultSavePlannerDefaultLoadPlannercreate_default_local_load_plancreate_default_global_load_plancreate_default_local_save_plancreate_default_global_save_planc                   @   s   e Zd ZU eed< deeee eddddZdeee	 eddd	d
Z
edddZee eee ef dddZeedddZeeejejf dddZeedddZeedddZdS )r'   mappingsTNF)r   flatten_sharded_tensorsdedup_replicated_tensorsdedup_save_to_lowest_rankreturnc                 C   s.   || _ || _i | _|| _|d k	r*td d S )NzDefaultSavePlanner's `dedup_replicated_tensors` argument is being deprecated, and no longer has any effect. Please remove this argument from your call.)r   r.   r-   r0   r&   warning)selfr   r.   r/   r0    r4   ^/var/www/html/venv/lib/python3.8/site-packages/torch/distributed/checkpoint/default_planner.py__init__B   s    zDefaultSavePlanner.__init__)
state_dictstorage_metais_coordinatorr1   c                 C   s2   | j rt |\}| _| jr"t|}|| _|| _d S N)r   r-   r.   r   r7   r9   )r3   r7   r8   r9   r4   r4   r5   set_up_plannerT   s    z!DefaultSavePlanner.set_up_plannerr1   c                 C   s0   t | j| j}| jr$tj|| jd}|| _| jS )Nplanner_data)r+   r7   r9   r   dataclassesreplacer-   plan)r3   rA   r4   r4   r5   create_local_plana   s
    z$DefaultSavePlanner.create_local_plan	all_plansr1   c                 C   sp   t || j}t|\}}| jrFdd |D }tt| }tj||d}t||sXt	d|| _
|| _| j
| jfS )Nc                 S   s   g | ]
}|j qS r4   r=   ).0pr4   r4   r5   
<listcomp>u   s     z9DefaultSavePlanner.create_global_plan.<locals>.<listcomp>r=   zFailed to validate global plan)r   r0   r,   r   dictr   r?   r@   _validate_global_plan
ValueErrorglobal_planmetadata)r3   rD   rK   rL   Zplanner_data_dictZmerged_mappingsr4   r4   r5   create_global_plani   s    
z%DefaultSavePlanner.create_global_plannew_planr1   c                 C   s
   || _ |S r:   )rA   r3   rO   r4   r4   r5   finish_plan   s    zDefaultSavePlanner.finish_plan)
write_itemr1   c                 C   s   |  |j}| ||S r:   )lookup_objectindextransform_object)r3   rR   objectr4   r4   r5   resolve_data   s    zDefaultSavePlanner.resolve_datarT   r1   c                 C   s   t | j|S zSExtension from the planner interface to make it easy to extend the default planner.r%   r7   r3   rT   r4   r4   r5   rS      s    z DefaultSavePlanner.lookup_object)rR   rV   c                 C   s(   |j tjkr$t }t|| |}|S rY   )typer    BYTE_IOioBytesIOtorchsave)r3   rR   rV   bytesr4   r4   r5   rU      s
    z#DefaultSavePlanner.transform_object)TTNF)NF)__name__
__module____qualname__r   __annotations__boolr   r6   r   r   r;   r   rB   r   r	   r   rM   rQ   r   r
   r`   Tensorr^   r_   rW   r   r   rS   rU   r4   r4   r4   r5   r'   ?   s8   
      	c                   @   s   e Zd ZU dZeed< eed< d%eeedddd	Zd&ee	e
 edd
ddZedddZee ee dddZeedddZeejddddZedddZeejddddZeejdd d!Zeejd"d#d$ZdS )'r(   ak  
    DefaultLoadPlanner that adds multiple features on top of LoadPlanner.

    In particular it adds the following:

    flatten_state_dict: Handle state_dict with nested dicts
    flatten_sharded_tensors: For FSDP in 2D parallel mode
    allow_partial_load: If False, will raise a runtime error if a key is present in state_dict, but not in the checkpoint.
    original_state_dictr-   TFN)r   r.   allow_partial_loadr1   c                 C   s"   || _ || _i | _i | _|| _d S r:   )r   r.   ri   r-   rj   )r3   r   r.   rj   r4   r4   r5   r6      s
    zDefaultLoadPlanner.__init__r7   rL   r9   r1   c                 C   sF   t | || _| jrt|}| jr0t|\}| _|| _|| _|| _d S r:   )	r$   ri   r.   r   r   r-   r7   rL   r9   )r3   r7   rL   r9   r4   r4   r5   r;      s    z!DefaultLoadPlanner.set_up_plannerr<   c                 C   s"   | j d k	stt| j| j | j S r:   )rL   AssertionErrorr)   r7   rj   r3   r4   r4   r5   rB      s      z$DefaultLoadPlanner.create_local_plan)rK   r1   c                 C   s   t |S r:   )r*   )r3   rK   r4   r4   r5   rM      s    z%DefaultLoadPlanner.create_global_planrN   c                 C   s   |S r:   r4   rP   r4   r4   r5   rQ      s    zDefaultLoadPlanner.finish_plan)	read_itemvaluer1   c                 C   sF   | j r*t| j| j|jj tj|dd ntj|dd| j|jj< d S )NF)Zweights_only)	r   r   ri   r-   
dest_indexfqnr`   loadr7   )r3   rn   ro   r4   r4   r5   
load_bytes   s     zDefaultLoadPlanner.load_bytes)rn   c                 C   s   |  |j}| ||S r:   )lookup_tensorrp   transform_tensorr3   rn   tensorr4   r4   r5   resolve_tensor   s    z!DefaultLoadPlanner.resolve_tensor)rn   rw   r1   c                 C   s   d S r:   r4   rv   r4   r4   r5   commit_tensor   s    z DefaultLoadPlanner.commit_tensorrX   c                 C   s   t | j|S rY   rZ   r[   r4   r4   r5   rt      s    z DefaultLoadPlanner.lookup_tensor)rn   rw   c                 C   s   t ||j|jS rY   )r   Zdest_offsetslengthsrv   r4   r4   r5   ru      s    z#DefaultLoadPlanner.transform_tensor)TTF)NF)rc   rd   re   __doc__r   rf   r   rg   r6   r   r   r;   r   rB   r   rM   rQ   r   r^   r_   rs   rx   r`   rh   ry   r   rt   ru   r4   r4   r4   r5   r(      s8   

     c                       sR   e Zd ZdZd fdd	ZeeedddZde	e
e edd	 fd
dZ  ZS )_EmptyStateDictLoadPlannera  
    Extension of DefaultLoadPlanner, which rebuilds state_dict from the saved metadata.
    Useful for loading in state_dict without first initializing a model, such as
    when converting a DCP checkpoint into a Torch save file.

    . N.B. `state_dict` must be an empty dictionary when used with this LoadPlanner

    .. warning::
        Because the entire state dict is initialized, It's recommended to only utilize
        this LoadPlanner on a single rank or process to avoid OOM.

    Nc                    s   || _ t j|| d S r:   )keyssuperr6   )r3   r}   argskwargs	__class__r4   r5   r6      s    z#_EmptyStateDictLoadPlanner.__init__)keyrL   r1   c                    s|    j d krdS | j krg }|j|}|D ]0}|rR|d|d t|g q,|| q,t fdd|D rxdS dS )NT.c                 3   s   | ]}| j kV  qd S r:   )r}   )rE   unflattened_keyrm   r4   r5   	<genexpr>  s     zA_EmptyStateDictLoadPlanner._should_include_key.<locals>.<genexpr>F)r}   r>   getappendjoinstrany)r3   r   rL   Zunflattened_keysr>   r   r4   rm   r5   _should_include_key   s    

z._EmptyStateDictLoadPlanner._should_include_keyFrk   c                    s   |rt |d k	st |j D ]Z\}}| ||s4qt|trRtj|j|j	j
d}||jkrpt||j| | q|||< qt ||| d S )N)dtype)rl   state_dict_metadataitemsr   
isinstancer   r`   emptysize
propertiesr   r>   r   r~   r;   )r3   r7   rL   r9   kvr   r4   r5   r;     s    


z)_EmptyStateDictLoadPlanner.set_up_planner)N)NF)rc   rd   re   r{   r6   r   r   rg   r   r   r   r;   __classcell__r4   r4   r   r5   r|      s     r|   T)r7   rL   strictr1   c                 C   s   g }|   D ]n\}}||jkr6|rtd| dnq|j| }t|trj|j d k	rz|t|||7 }q|t|||7 }qt|S )Nz&Missing key in checkpoint state_dict: r   )	r   r   RuntimeErrorr   r   device_meshget_coordinater"   r   )r7   rL   r   requestsrq   objmdr4   r4   r5   r)   ,  s    


rC   c                 C   s   | S )z
    Create global load plan used by DefaultLoadPlanner.

    The default load behavior involved no global coordination and this function
    currently doesn't change the local plans.
    r4   )rD   r4   r4   r5   r*   N  s    	)r7   r9   r1   c                 C   sT   g }|   D ]>\}}t|tr<|j dk	rJ|t||7 }q|t||7 }qt|S )a  
    Create the ``SavePlan`` used by DefaultSavePlanner.

    On non-coordinator ranks, this function ignores tensors and non-tensor objects,
    only producing writes for ShardedTensor objects.

    On the coordinator rank, produce writes for all values.
    N)r   r   r   r   r   r#   r   )r7   r9   r   rq   r   r4   r4   r5   r+   Z  s    
)rD   rewrite_index_hintsr1   c           
      C   s"  i }g }| D ]}g }|j D ]}|jtjks<|jj|ks<t|jtjkrbt ||jj< |	| q|j
dk	spttt||jjt|j
j|j
jg d}|}|rtj|jt|jd}	tj||	d}|	| |j
jdk	std|jj d|j	|j
j q|	tj||d q|t|fS )a6  
    Create the global plan and metadata used by DefaultSavePlanner.

    Metadata is produced by concatenating the metadata of all ``WriteItem`` from the supplied plans.

    The only global planning change is to update index hints in all ``MetadataIndex`` objects if
    ``rewrite_index_hints`` is True.
    N)r   r   chunks)rT   zZ
                    Cannot create MD for tensor without bounds.
                    FQN: z
                )r   )r   r\   r    ZSHARDrT   rq   rl   r]   r   r   Ztensor_datar   r   
setdefaultr   r   r?   r@   lenr   chunkr   )
rD   r   r   Z	new_plansrA   Z	new_itemsitemZ	tensor_mdZnew_itemZ	new_indexr4   r4   r5   r,   u  sN    

 

)r7   r1   c                 C   s   t | }t|g\}}|S )zTReturn the ``Metadata`` if DefaultSavePlanner was used to checkpoint ``state_dict``.)r!   r,   )r7   rA   _r   r4   r4   r5   _create_default_local_metadata  s    r   )box0box1r1   c                 C   sd   t | j}t|D ]L}| j| |j| |j|  kr: dS |j| | j| | j|  kr dS qdS )z9Check if two boxes overlap. Tuples are (offset, lengths).FT)r   offsetsrangesizes)r   r   Zndimsir4   r4   r5   _check_box_overlap  s    
r   )outer_box_size	inner_boxr1   c                 C   s`   t t| D ]N}|j| dk r$ dS |j| dk r8 dS |j| |j|  | | kr dS qdS )Nr   FT)r   r   r   r   )r   r   r   r4   r4   r5   _check_box_bounds  s    r   )rK   rL   r1   c           
   	   C   s   d}|j  D ]\}}t|tr"qt|jdkr2qd}t|jD ]t\}}t|j|sjt	
d||j| d}|ttj|jd7 }|j|d d  D ]"}t||rt	
d||| d}qq@ttj|jd}	||	krt	
d||	| d}q|S )NTr   z~
                        key:%s has out of bounds chunk:
                        tensor-size:%s chunk: %s
                    F   z$key:%s has overlapping chunks: %s %szq
                    key:%s invalid fill tensor-volume:
                    %s chunks-volume: %s
                )r   r   r   r   r   r   	enumerater   r   r&   r2   r   operatormulr   r   )
rK   rL   all_goodr   ro   Zchunks_volumeZ	chunk_idxZchunk0Zchunk1Ztensor_volumer4   r4   r5   rI     sH    
	
   	rI   )T)T)Jr?   r^   loggingr   collectionsr   	functoolsr   typingr   r   r   r   r   r	   r
   r`   Ztorch.distributed._shard._utilsr   Ztorch.distributed._tensorr   Z.torch.distributed.checkpoint._dedup_save_plansr   Z)torch.distributed.checkpoint._nested_dictr   r   Z2torch.distributed.checkpoint._sharded_tensor_utilsr   Z&torch.distributed.checkpoint._traverser   Z%torch.distributed.checkpoint.metadatar   r   r   r   r   r   r   r   Z$torch.distributed.checkpoint.plannerr   r   r   r   r   r   r    Z,torch.distributed.checkpoint.planner_helpersr!   r"   r#   r$   Z"torch.distributed.checkpoint.utilsr%   	getLoggerrc   r&   Loggerrf   __all__r'   r(   r|   r   rg   r)   r*   r+   r,   r   r   Sizer   rI   r4   r4   r4   r5   <module>   sp    $(
$	WUB 
  #
  7 