U
    yh,                     @   s  U d dl mZmZmZ d dlZd dlmZ d dlm	Z	 d dl
mZ d dlmZ d dlmZ d dlmZ d dlmZ d d	lmZ d
dlmZmZmZmZmZmZmZ d
dlmZm Z m!Z!m"Z"m#Z#m$Z$ d
dl%m&Z&m'Z' dgZ(ee) e*d< ej+edddZ,eedddZ-eee"dddZ.e)ee#dddZ/e)eee#dddZ0e)ej+e#ddd Z1e)ed!d"d#Z2d$d% Z3d&d' Z4e)eee ee  d(d)dZ5ee!d*d+d,Z6e)eee# d-d.d/Z7eedd0d1Z8ej+ee dd2d3Z9e)eeee  d4d5d6Z:edd*d7d8Z;eed9d:d;Z<dS )<    )AnycastListN)_get_device_module)ShardMetadata)ShardedTensor)DTensor)%compute_local_shape_and_global_offset)_Checkpointable)tree_map_only_   )BytesStorageMetadataChunkStorageMetadataMetadataIndexSTATE_DICT_TYPESTORAGE_TYPESTensorPropertiesTensorStorageMetadata)LoadItemTypeReadItemSavePlanTensorWriteData	WriteItemWriteItemType)"_check_shard_metadata_pair_overlap+_shards_get_overlap_region_wrt_saved_tensor create_read_items_for_chunk_list__all__)tensorreturnc                 C   s$   t tdgt|   |  dS )Nr   offsetssizes)r   torchSizelensize)r    r'   ^/var/www/html/venv/lib/python3.8/site-packages/torch/distributed/checkpoint/planner_helpers.py_create_chunk_from_tensor)   s     r)   )shard_mdr   c                 C   s   t t| jt| jdS Nr    )r   r#   r$   shard_offsetsZshard_sizes)r*   r'   r'   r(   _chunk_for_shard/   s    

r-   )sharded_tensorr*   r   c                 C   s>   |   j}t|j|j|j|j|jd}tt	|||   j
dS )N)dtypelayoutrequires_gradmemory_format
pin_memorychunk
propertiesr&   )metadataZtensor_propertiesr   r/   r0   r1   r2   r3   r   r-   r&   )r.   r*   Zshard_propertiesr6   r'   r'   r(   _sharded_tensor_metadata6   s    
r8   )fqnr   r   c              	   C   sb   t |j|j|j\}}t|t| }}tt| |tj	t
t||dt| | ddS )Nr    r4   indextypeZtensor_data)r	   shapedevice_mesh
placementsr#   r$   r   r   r   SHARDr   r   r   create_from_tensorto_localr&   )r9   r   r"   r!   r'   r'   r(   _create_write_items_for_dtensorJ   s$      rC   )r9   r.   r*   r   c                 C   s(   t |j}tt| |tjt||dS )Nr:   )r#   r$   r,   r   r   r   r@   r8   )r9   r.   r*   r!   r'   r'   r(   _create_write_item_for_shard^   s    rD   c                 C   sN   t dgt|  }tt| |tjtt	|| dt
|| ddS )Nr   r    r4   r:   )r#   r$   r%   r&   r   r   r   TENSORr   r   r   rA   )r9   r   r!   r'   r'   r(   _create_write_item_for_tensori   s    rF   r9   bytesc                 C   s   t t| tjdS )N)r;   r<   )r   r   r   BYTE_IOrG   r'   r'   r(   _create_write_item_for_bytesiov   s    rJ   c              	   C   s.   t tj| t|f|t|ft|fdS N)r<   
dest_indexdest_offsetsstorage_indexstorage_offsetslengths)r   r   rI   r#   r$   rL   Zdest_offsetrN   Zstorage_offsetlengthr'   r'   r(   _create_read_item_for_byteio}   s    


rS   c              	   C   s(   t tj| t||t|t|dS rK   )r   r   rE   r#   r$   rL   rM   rN   rO   rP   r'   r'   r(   _create_read_item_for_tensor   s    rU   )r9   checkpoint_mdlocal_chunksr   c                 C   s   g }t |D ]\}}t |jD ]\}}t||s2qg }g }	g }
t||dD ]*\}}}}|| |	| |
| qJ|tt| |j||	t| |j|||
d qq|S )aW  
    Create a list of ``ReadItem`` based on the checkpoint and local chunks.

    This applies the resharding algorithm and computes the reads needed
    to satisfy ``local_chunks`` with a checkpoint described by ``checkpoint_md``.

    Args:
        fqn (str) : The state_dict FQN to pass to ``ReadItem``.
        checkpoint_md (TensorStorageMetadata): metadata for a given tensor
            from a checkpoint.
        local_chunks (List[ChunkStorageMetadata]): Local chunks that needs to be
            loaded.

    Returns:
        A list of ``ReadItem`` that will satisfy all input chunks.
    )Zsaved_shardZcurrent_shardrT   )	enumeratechunksr   r   appendrU   r   r!   )r9   rV   rW   Z
read_itemsidxshardZstorage_idxZ
storage_mdrO   rM   rP   dimZoffset_for_saved_tensorZoffset_for_current_tensorrR   r'   r'   r(   r      s<    
 

	)
state_dictr   c                 C   s   g }|   D ]\}}t|tr0|t|| qt|tr^| jD ]}|t||| qDqt|t	j
r||t|| q|t|| qt|S N)items
isinstancer   rZ   rC   r   r7   Zshards_metadatarD   r#   TensorrF   rJ   r   )r^   requestsr9   objr*   r'   r'   r(   "_create_default_metadata_only_plan   s    

re   )r9   objectr   c                    s   t tr S t trJt  tr>  S t gS t trl fdd D S t tj	rt
 gS t gS d S )Nc                    s   g | ]}t  |jqS r'   )rD   r7   .0r\   r9   rf   r'   r(   
<listcomp>   s   z'_create_write_items.<locals>.<listcomp>)ra   r
   _create_write_itemsr   rB   rC   r   local_shardsr#   rb   rF   rJ   ri   r'   ri   r(   rk      s    


rk   c                 C   s8   t | j| j| j\}}t|t| }}t||dS r+   )r	   r=   r>   r?   r#   r$   r   )r   r"   r!   r'   r'   r(   _create_chunk_from_dtensor   s      rm   c                 C   s   t | tr| | }nzt | trHt |  tr<|  | S t| g}nHt | trfdd |  D }n*t | tj	r~t
| g}ntdt|  |S )Nc                 S   s   g | ]}t |jqS r'   )r-   r7   rg   r'   r'   r(   rj     s    z&_create_chunk_list.<locals>.<listcomp>zMUnsupported Type, expecting one of [Tensor, DTensor, ShardedTensor] ,but got )ra   r
   _create_chunk_listr   rB   rm   r   rl   r#   rb   r)   
ValueErrorr<   )r   rW   r'   r'   r(   rn      s     


rn   )r9   mdrd   r   c              
   C   s   t |tsfzt|}W nB tk
rX } z$td|  ddt|  |W 5 d }~X Y nX t| ||S tt| dt| dddgS d S )Nz Invalid checkpoint metadata for z, z(expected BytesStorageMetadata but found r   rQ   )ra   r   rn   ro   r<   r   rS   r   )r9   rp   rd   rW   exr'   r'   r(   _create_read_items  s(    

rr   c                 C   s   t tjt|  d S r_   )r   r#   rb   _init_meta_tensor)r^   r'   r'   r(   _init_state_dict)  s    rt   )valuer   c                 C   s   t | dd}|tdkrtj j}ttjt|	 }t
| trztj|  |d}tj|| j| j|  |  d}|S t
| tjrtj| |d}|S tdt|  dn| S dS )zY
    Initializes tensor, moves it to device for torch.Tensor/DTensor on meta device.
    deviceNmeta)rv   )r>   r?   r=   stridezFound unsupported type z for meta device loading.)getattrr#   rv   distZdistributed_c10dZ_get_pg_default_devicer<   r   r   Zcurrent_devicera   r   Z
empty_likerB   Z
from_localr>   r?   r&   rx   rb   RuntimeError)ru   rv   Zdevice_typeZnew_local_tensorZdtensorr   r'   r'   r(   rs   -  s*    
rs   )=typingr   r   r   r#   Ztorch.distributeddistributedrz   Ztorch._utilsr   Z!torch.distributed._shard.metadatar   Z'torch.distributed._shard.sharded_tensorr   Ztorch.distributed._tensorr   Z torch.distributed._tensor._utilsr	   Z$torch.distributed.checkpoint.plannerr
   Ztorch.utils._pytreer   r7   r   r   r   r   r   r   r   Zplannerr   r   r   r   r   r   Z
reshardingr   r   r   str__annotations__rb   r)   r-   r8   rC   rD   rF   rJ   rS   rU   r   re   rk   rm   rn   rr   rt   rs   r'   r'   r'   r(   <module>   sT    $	    7