U yh@sddlZddlZddlmZddlmZddlmZmZm Z ddl Z ddl m Z ddl m Z ddlmZmZmZdd gZGd d d eZGd ddeZGd d d eZdS)N)contextmanager)Path) GeneratorOptionalUnion)AbstractFileSystem) url_to_fs)FileSystemBaseFileSystemReaderFileSystemWriter FsspecWriter FsspecReaderc@seZdZddddZeeeejfee e j ddfdddZ eeejfeeeejfdd d Z eeejfeeejfd d d ZeeejfeeejfddddZeejgdd ddZeeeejfedddZeeejfed ddZeeejfdd ddZdS) FileSystemN)returncCs d|_dSN)fs)selfra/var/www/html/venv/lib/python3.8/site-packages/torch/distributed/checkpoint/_fsspec_filesystem.py__init__szFileSystem.__init__)pathmoderc csH|jdk st|jj(tt|| }|VW5QRXW5QRXdSr)rAssertionErrorZ transactionfsspecopenstr)rrrstreamrrr create_streams zFileSystem.create_stream)rsuffixrcCstj||Sr)osrjoin)rrrrrr concat_path'szFileSystem.concat_pathrrcCst|\|_}|Sr)rr)rr_rrr init_path,szFileSystem.init_path)rnew_pathrcCs|j||dSr)rrename)rrr%rrrr&0szFileSystem.renamecCs|jj|dddS)NT)exist_ok)rmakedirsrrrrrmkdir5szFileSystem.mkdir checkpoint_idrcCs6t|trdSz t|Wntk r0YdSXdS)NFT) isinstancerr ValueErrorclsr,rrrvalidate_checkpoint_id8s  z!FileSystem.validate_checkpoint_idcCs |j|Sr)rexistsr)rrrr2DszFileSystem.existscCs|j|dSr)rZrmr)rrrrm_fileGszFileSystem.rm_file)__name__ __module__ __qualname__rrrrrPathLikerioIOBaserr!r$r&r* classmethodboolr1r2r3rrrrrs(   $   rc s\eZdZdZd eeejfeee e eddfdd Z e eeejfed d d Z Z S) r a` Basic implementation of StorageWriter using FFspec. This implementation makes the following assumptions and simplifications: * The checkpoint path is an empty or non-existing directory. * File creation is atomic The checkpoint consist of one file per write request plus a `.metadata` file with the serialized metadata. T逖N)rsingle_file_per_rank sync_files thread_countper_thread_copy_ahead overwritercs2tj||||||dt|_|j||_dS)a Initialize the writer pointing to `path`. Args: path: directory where the checkpoint will be written to. single_file_per_rank: Produce one file per rank instead of one file per tensor/blob. Default to True. sync_files : force files to be synced to permanent storage. Default to True. thread_count: Number of IO threads to use to write. Default to 1. per_thread_copy_ahead: How many bytes to copy from the GPU ahead of saving then. Default 10Mb. overwrite: Whether to allow overwriting existing checkpoints. Defaults to True. N. B. If sync_files is disabled, there's no guarantee that the checkpoint will be consistent in the case of a failure. )rBNsuperrrrr$r)rrr>r?r@rArB __class__rrrZszFsspecWriter.__init__r+cCs t|Srrr1r/rrrr1{sz#FsspecWriter.validate_checkpoint_id)TTr<r=T)r4r5r6__doc__rrrr7r;intrr:r1 __classcell__rrrErr Ls" !csLeZdZeeejfddfdd Zeeeejfe dddZ Z S)r Nr"cs&t|t|_|j||_dSrrCr)rErrrs zFsspecReader.__init__r+cCs t|SrrGr/rrrr1sz#FsspecReader.validate_checkpoint_id) r4r5r6rrrr7rr:r;r1rJrrrErr s)r8r contextlibrpathlibrtypingrrrrrZ fsspec.corerZ'torch.distributed.checkpoint.filesystemr r r __all__rr r rrrrs    24