U h! @sddlZddlZddlZddlZddlZddlmZddlmZm Z ddl m Z ddl m Z ddlmZmZmZmZddZdaGd d d e Zed d d dddddddg GdddejZGdddeZdS)N)suppress)cached_propertywraps)parse_qs)AbstractFileSystem)"get_package_version_without_importinfer_storage_options mirror_fromtokenizecstfdd}|S)Nc sjz ||WStk rd}z:|js&|j^}}t|trRd|krRttj||nW5d}~XYnXdS)Nzdoes not exist)OSErrorargs isinstancestrFileNotFoundErrorerrnoENOENT)r kwargs exceptionmessagefuncN/var/www/html/venv/lib/python3.8/site-packages/fsspec/implementations/arrow.pywrappers  z wrap_exceptions..wrapper)r)rrrrrwrap_exceptionss rcseZdZdZdZfddZeddZeddZ e d d Z d-d d Z ddZ ddZddZeddZeddZeddZed.ddZed/dd Zed0d!d"Zed1d#d$Zed%d&Zed'd(Zd2fd)d* Zfd+d,ZZS)3ArrowFSWrapperzwFSSpec-compatible wrapper of pyarrow.fs.FileSystem. Parameters ---------- fs : pyarrow.fs.FileSystem /c s tda||_tjf|dS)NZpyarrow)rPYARROW_VERSIONfssuper__init__)selfrr __class__rrr 3szArrowFSWrapper.__init__cCs|jjSN)r type_namer!rrrprotocol9szArrowFSWrapper.protocolcCsdt|jj|jjS)NZhdfs_)r rhostportr&rrrfsid=szArrowFSWrapper.fsidcCs*t|}|d}|dr&|dd}|S)Npathz//)r startswith)clsr+opsrrr_strip_protocolAs   zArrowFSWrapper._strip_protocolFc sN|}ddlm}fddj||D}|r<|Sdd|DSdS)Nr) FileSelectorcsg|]}|qSr) _make_entry.0entryr&rr Nsz%ArrowFSWrapper.ls..cSsg|] }|dqS)namerr3rrrr6Us)r0 pyarrow.fsr1r get_file_info)r!r+detailrr1entriesrr&rlsJs   zArrowFSWrapper.lscKs$||}|j|g\}||Sr$)r0rr9r2)r!r+rinforrrr=Ws zArrowFSWrapper.infocCs8||}z||Wntk r.YdSXdSdS)NFT)r0r=rr!r+rrrexists\s  zArrowFSWrapper.existscCsnddlm}|j|jkrd}n<|j|jkr0d}n*|j|jkrVttjt tj|j nd}|j |j ||j dS)Nr)FileType directoryfileother)r7sizetypemtime)r8r@rEZ DirectoryFileZNotFoundrrrosstrerrorr+rDrF)r!r=r@kindrrrr2es    zArrowFSWrapper._make_entryc Ks||d}||d}||d}|dtd}z6||d}t||W5QRX|j ||Wn6t k rt t |j |W5QRXYnXW5QRXdS)Nrrbz.tmp.wb)r0rstrip_opensecretsZ token_hexopenshutil copyfileobjrmove BaseExceptionrr delete_file)r!path1path2rZlstreamZ tmp_fnameZrstreamrrrcp_filexs zArrowFSWrapper.cp_filecKs2||d}||d}|j||dS)Nr)r0rNrrT)r!rWrXrrrrmvszArrowFSWrapper.mvcCs||}|j|dSr$)r0rrVr>rrrrm_files zArrowFSWrapper.rm_fileNcCsF||d}||r6|r,|j|qBtdn |j|dS)Nrz0Can't delete directories without recursive=False)r0rNisdirr delete_dir ValueErrorrV)r!r+ recursiveZmaxdepthrrrrms   zArrowFSWrapper.rmrKTc Ks|dkr |r|jj}qR|jj}n2|dkr2|jj}n |dkrD|jj}ntd|i}|dksb|sttdddkrd|d<||f|}t |||||f|S) NrKrMabz'unsupported mode for Arrow filesystem: .r compression) rZopen_input_fileZopen_input_streamZopen_output_streamZopen_append_streamr^intrsplit ArrowFile) r!r+mode block_sizeseekablermethod_kwargsstreamrrrrOs      zArrowFSWrapper._opencKs2||}|r|j|ddn|jj|dddS)NT)exist_okFr_)r0makedirsr create_dir)r!r+Zcreate_parentsrrrrmkdirs zArrowFSWrapper.mkdircCs||}|jj|dddS)NTro)r0rrq)r!r+rnrrrrps zArrowFSWrapper.makedirscCs||}|j|dSr$)r0rr]r>rrrrmdirs zArrowFSWrapper.rmdircCs||}|j|jSr$)r0rr9rFr>rrrmodifieds zArrowFSWrapper.modifiedc s&|dk|d<tj|fddd|S)N)Nrrj)startend)rcat_file)r!r+rurvrr"rrrws zArrowFSWrapper.cat_filec sd|d<tj||f|dS)NFrj)rget_file)r!ZrpathZlpathrr"rrrxszArrowFSWrapper.get_file)F)FN)rKNT)T)F)NN)__name__ __module__ __qualname____doc__Z root_markerr propertyr'rr* classmethodr0r<r=r?r2rrYrZr[r`rOrrrprsrtrwrx __classcell__rrr"rr(sB               rrmreadseektellwritereadablewritablecloserDrjc@s&eZdZdddZddZddZdS) rgNcKs.||_||_||_||_||_|_||_dSr$)r+rhrrm blocksizerir)r!rrmr+rhrirrrrr s  zArrowFile.__init__cCs|Sr$rr&rrr __enter__szArrowFile.__enter__cGs|Sr$)r)r!r rrr__exit__szArrowFile.__exit__)N)ryrzr{r rrrrrrrgs rgcs2eZdZdZdZd fdd Zed d ZZS) HadoopFileSystemz]A wrapper on top of the pyarrow.fs.HadoopFileSystem to connect it's interface with fsspecZhdfsdefaultrNc  s:ddlm}|||||||d} tjfd| i|dS)a; Parameters ---------- host: str Hostname, IP or "default" to try to read from Hadoop config port: int Port to connect on, or default from Hadoop config if 0 user: str or None If given, connect as this username kerb_ticket: str or None If given, use this ticket for authentication replication: int set replication factor of file for write operations. default value is 3. extra_conf: None or dict Passed on to HadoopFileSystem r)r)r(r)user kerb_ticket replication extra_confrN)r8rrr ) r!r(r)rrrrrrrr"rrr s zHadoopFileSystem.__init__cCst|}i}|ddr$|d|d<|ddr<|d|d<|ddrT|d|d<|ddrt|d}|ddrt|dd|d<|S)Nr(usernamerr)Z url_queryrr)rgetrre)r+r/outZqueriesrrr_get_kwargs_from_urls"s         z&HadoopFileSystem._get_kwargs_from_urls)rrNNrN) ryrzr{r|r'r staticmethodrrrrr"rrs'r)riorHrPrR contextlibr functoolsrr urllib.parserZ fsspec.specrZ fsspec.utilsrrr r rrrIOBasergrrrrrs8   .