U
    ?hJ                     @   s  d Z ddlZddlZddlmZ ddlZzddlZW n ek
rL   dZY nX zddlZW n ek
rr   dZY nX ddl	m
Z
 ddlmZmZmZmZmZmZmZmZmZ ddlmZ dd	lmZmZmZmZmZmZmZmZ e Z e Z!e Z"e Z#e Z$ej%j&edkd
dej%'de d e!gdd Z(ej%j&edkdddd Z)dd Z*ej%j+dd Z,ej%j+dd Z-ej%j+ej%j'dedfedfedfgdddgd d!d" Z.ej%j+ej%j'de!e"e$gdddgd d#d$ Z/ej%j+d%d& Z0ej%j+d'd( Z1ej%j+ej%'d)d*d+gd,d- Z2ej%j+d.d/ Z3ej%j+ej%j&edkd0dd1d2 Z4ej%j+ej%j&edkd0dd3d4 Z5ej%j&edk	d5dd6d7 Z6ej%j&edk	d8dej%'d9eeegd:d; Z7ej%j+ej%j&edkd
dej%j'd<e efe!efgd=dgd d>d? Z8ej%j+ej%j&edkd
dd@dA Z9ej%j+ej%j&edkd
dej%j&edkdddBdC Z:ej%j+dDdE Z;G dFdG dGZ<dS )HzK
Test the downloader classes and functions separately from the Pooch core.
    N)TemporaryDirectory   )Pooch)	HTTPDownloaderFTPDownloaderSFTPDownloaderDOIDownloaderchoose_downloaderFigshareRepositoryZenodoRepositoryDataverseRepository
doi_to_url)Unzip   )pooch_test_urlcheck_large_datacheck_tiny_datadata_over_ftppooch_test_figshare_urlpooch_test_zenodo_url pooch_test_zenodo_with_slash_urlpooch_test_dataverse_urlzrequires tqdm)reasonurltiny-data.txtc                 C   s   t | dd}|jdkstdS )DThe progressbar keyword argument must pass through choose_downloaderTprogressbarNr	   r   AssertionErrorr   
downloader r"   N/var/www/html/venv/lib/python3.8/site-packages/pooch/tests/test_downloaders.pytest_progressbar_kwarg_passed:   s    
r$   zrequires paramikoc                  C   s"   d} t | dd}|jdkstdS )r   /sftp://test.rebex.net/pub/example/pocketftp.pngTr   Nr   r    r"   r"   r#   "test_progressbar_kwarg_passed_sftpH   s    r&   c                	   C   s@   t t td W 5 Q R X t t td W 5 Q R X dS )z6Should raise ValueError when protocol is not supportedzhttpup://some-invalid-url.comzdoii:XXX/XXX/fileN)pytestraises
ValueErrorr	   r"   r"   r"   r#   test_unsupported_protocolP   s    r*   c               	   C   s<   t t} t dddd W 5 Q R X dt| jks8tdS )z/Should fail if data repository is not supportedz%doi:10.21105/joss.01943/file_name.txtN)r   Zoutput_filepoochz(Invalid data repository 'joss.theoj.org')r'   r(   r)   r   strvaluer   excr"   r"   r#   test_invalid_doi_repositoryY   s      r0   c               	   C   s6   t t} tdd W 5 Q R X dt| jks2tdS )z#Should fail if the DOI is not foundZNOTAREALDOI)doizIs the DOI correct?N)r'   r(   r)   r   r,   r-   r   r.   r"   r"   r#   test_doi_url_not_foundd   s    r2   zrepository,doiz10.6084/m9.figshare.14763051.v1z10.5281/zenodo.4924875z10.11588/data/TKCFEFfigshareZzenodoZ	dataverse)Zidsc              	   C   sL   t t&}t|}| ||}|jdd W 5 Q R X dt|jksHtdS )z3Should fail if the file is not found in the archivezbla.txt	file_namezFile 'bla.txt' not foundN)	r'   r(   r)   r   Z
initializedownload_urlr,   r-   r   )
repositoryr1   r/   r   repor"   r"   r#    test_figshare_url_file_not_foundl   s
    r9   c              	   C   sB   t  2}t }tj|d}|| d |d t| W 5 Q R X dS )zTest the DOI downloaderr   N)r   r   ospathjoinr   )r   local_storer!   outfiler"   r"   r#   test_doi_downloader   s
    	r?   c               	   C   sf   t  V} td }t }tj| d}|||d t |ddd}dd |D \}t| W 5 Q R X dS )zf
    Test the Zenodo downloader when the path contains a forward slash

    Related to issue #336
    z!santisoler/pooch-test-data-v1.zipztest-data.zipNdownload)actionr+   c                 S   s   g | ]}d |kr|qS )r   r"   ).0fr"   r"   r#   
<listcomp>   s      z>test_zenodo_downloader_with_slash_in_fname.<locals>.<listcomp>)r   ZENODOURL_W_SLASHr   r:   r;   r<   r   r   )r=   base_urlr!   r>   fnamesfnamer"   r"   r#   *test_zenodo_downloader_with_slash_in_fname   s    rI   c               
   C   s   t } | d| d d } | dd }d| d}t D}t }tj|d}tjt	|d	 || d |d W 5 Q R X W 5 Q R X dS )
zf
    Test if passing a Figshare url without a version warns about it, but still
    downloads it.
    N./   zThe Figshare DOI 'z#' doesn't specify which version of r   match)
FIGSHAREURLrindexr   r   r:   r;   r<   r'   ZwarnsUserWarning)r   r1   Zwarning_msgr=   r!   r>   r"   r"   r#   !test_figshare_unspecified_version   s    rS   zversion, missing, present)r   7LC08_L2SP_218074_20190114_20200829_02_T1-cropped.tar.gzcropped-before.tar.gz)r   rU   rT   c                 C   sL   d|  }d| d}t ||}dd |jD }||ks<t||ksHtdS )zG
    Test if setting the version in Figshare DOI works as expected
    z10.6084/m9.figshare.21665630.vhttps://doi.org/rK   c                 S   s   g | ]}|d  qS )namer"   )rB   itemr"   r"   r#   rD      s     z:test_figshare_data_repository_versions.<locals>.<listcomp>N)r
   api_responser   )versionmissingZpresentr1   r   r3   	filenamesr"   r"   r#   &test_figshare_data_repository_versions   s    

r]   c              
   C   sZ   t | dF}t 4}t| jd}tj|d}|||d t| W 5 Q R X W 5 Q R X dS )zTest ftp downloaderr   )portN)r   r   r   server_portr:   r;   r<   r   )	ftpserverr   r=   r!   r>   r"   r"   r#   test_ftp_downloader   s    ra   zrequires paramiko to run SFTPc               	   C   sP   t  @} tddd}d}tj| d}|||d tj|sBtW 5 Q R X dS )zTest sftp downloaderdemopasswordusernamerc   r%   pocketftp.pngN)r   r   r:   r;   r<   existsr   )r=   r!   r   r>   r"   r"   r#   test_sftp_downloader   s    rh   c                  C   sl   t  \} tddd}d}tj| d}t|d(}tt |||d W 5 Q R X W 5 Q R X W 5 Q R X dS )zFDownloader should fail when a file object rather than string is passedrb   rc   rd   r%   rf   wbN)	r   r   r:   r;   r<   openr'   r(   	TypeError)r=   r!   r   r>   Zoutfile_objr"   r"   r#   (test_sftp_downloader_fail_if_file_object   s    rl   zparamiko must be missingc               	   C   s2   t t} t  W 5 Q R X dt| jks.tdS )z+test must fail if paramiko is not installedz
'paramiko'N)r'   r(   r)   r   r,   r-   r   r.   r"   r"   r#   -test_sftp_downloader_fail_if_paramiko_missing   s    rm   ztqdm must be missingr!   c              	   C   s6   t t}| dd W 5 Q R X dt|jks2tdS )zFMake sure an error is raised if trying to use progressbar without tqdmTr   z'tqdm'N)r'   r(   r)   r,   r-   r   )r!   r/   r"   r"   r#   !test_downloader_progressbar_fails   s    rn   zurl,downloaderhttpc           
   	   C   s   |dd}t  }d}| | } tj||}|| |d | }|jdd  }t|dksdt	t
jdkrtd	}	nd
}	|dd |	kst	t| W 5 Q R X dS )z@Setup a downloader function that prints a progress bar for fetchTr   r   NrM   O   win32100%|####################A   100%|████████████████████   )r   r:   r;   r<   
readouterrerrsplitstriplenr   sysplatformr   )
r   r!   capsysr@   r=   rH   r>   capturedprintedprogressr"   r"   r#   test_downloader_progressbar	  s    	

r   c           	   
   C   s   t |d}td|jd}t |}tj|d}|||d |  }|j	dd 
 }t|dkshttjdkrxd	}nd
}|dd |kstt| W 5 Q R X W 5 Q R X dS )zESetup an FTP downloader function that prints a progress bar for fetchr   T)r   r^   Nrp   rM   rq   rr   rs   rt   ru   )r   r   r_   r   r:   r;   r<   rv   rw   rx   ry   rz   r   r{   r|   r   )	r}   r`   r   r@   r=   r>   r~   r   r   r"   r"   r#   test_downloader_progressbar_ftp&  s    
r   c              	   C   s   t dddd}t }d}tj|d}|||d |  }|jdd	  }t	|d
ks`t
tjdkrpd}nd}|dd |kst
tj|st
W 5 Q R X dS )zFSetup an SFTP downloader function that prints a progress bar for fetchTrb   rc   )r   re   rc   r%   rf   Nrp   rM   rq   rr   rs   rt   ru   )r   r   r:   r;   r<   rv   rw   rx   ry   rz   r   r{   r|   rg   )r}   r!   r=   r   r>   r~   r   r   r"   r"   r#    test_downloader_progressbar_sftp?  s    
r   c              	   C   s   G dd d}|dd}t |d}t `}d}t| }tj|d}|||d |  }|jdd 	 }	d	}
|	|
ks|t
t| W 5 Q R X dS )
zASetup a downloader function with an arbitrary progress bar class.c                   @   sD   e Zd ZdZdd Zdd Zdd Zdd	 Zd
d Ze	dd Z
dS )zEtest_downloader_arbitrary_progressbar.<locals>.MinimalProgressDisplayz&A minimalist replacement for tqdm.tqdmc                 S   s   d| _ || _d S )Nr   )counttotal)selfr   r"   r"   r#   __init__^  s    zNtest_downloader_arbitrary_progressbar.<locals>.MinimalProgressDisplay.__init__c                 S   s   t | jd t | j S )zrepresent current completionrK   )r,   r   r   r   r"   r"   r#   __repr__b  s    zNtest_downloader_arbitrary_progressbar.<locals>.MinimalProgressDisplay.__repr__c                 S   s   t d|  tjdd dS )zprint self.__repr__ to stderrrp    )fileendNprintr{   stderrr   r"   r"   r#   renderf  s    zLtest_downloader_arbitrary_progressbar.<locals>.MinimalProgressDisplay.renderc                 S   s   || _ |   dS )zmodify completion and renderN)r   r   )r   ir"   r"   r#   updatej  s    zLtest_downloader_arbitrary_progressbar.<locals>.MinimalProgressDisplay.updatec                 S   s
   d| _ dS )zset counter to 0r   N)r   r   r"   r"   r#   reseto  s    zKtest_downloader_arbitrary_progressbar.<locals>.MinimalProgressDisplay.resetc                   S   s   t dtjd dS )zprint a new empty liner   )r   Nr   r"   r"   r"   r#   closes  s    zKtest_downloader_arbitrary_progressbar.<locals>.MinimalProgressDisplay.closeN)__name__
__module____qualname____doc__r   r   r   r   r   staticmethodr   r"   r"   r"   r#   MinimalProgressDisplay[  s   r   N)r   r   zlarge-data.txtrp   rM   z336/336)r   r   BASEURLr:   r;   r<   rv   rw   rx   ry   r   r   )r}   r   Zpbarr@   r=   rH   r   r>   r~   r   r   r"   r"   r#   %test_downloader_arbitrary_progressbarW  s    

r   c                   @   s  e Zd ZdZdZde Zde ZdZdZdZ	ddeeed	ed
e	 deidgdZ
ddeeed	ee	deidgdZddeeed	ee	deidd	ed
e	 deidgdZejdde
fdefdefgdd Zejdde
fdefgdd Zejde
egdd ZdS )TestZenodoAPISupportz0
    Test support for different Zenodo APIs
    i@ z10.0001/zenodo.rV   zmy-file.zipzMhttps://zenodo.org/api/files/513d7033-93a2-4eeb-821c-2fb0bbab0012/my-file.zipZ 2942bfabb3d05332b66eb128e0842cffz 2021-20-19T08:00:00.000000+00:00z$513d7033-93a2-4eeb-821c-2fb0bbab0012md5:r   )idkeychecksumlinks)createdmodifiedr   r1   doi_urlfiles)r   filenamer   r   zapi_version, api_responselegacynewinvalidc              	   C   st   | d| j | t| j| jd}|d|_|dkrL|j|kspt	n$d}t
jt|d |j}W 5 Q R X dS )z@
        Test if the API version is correctly detected.
        /zenodo.r1   Zarchive_urlr   r   z0Couldn't determine the version of the Zenodo APIrN   N)expect_request
article_idrespond_with_jsonr   r1   r   url_forbase_api_urlapi_versionr   r'   r(   r)   )r   
httpserverr   rY   r!   msgr"   r"   r#   test_api_version  s    z%TestZenodoAPISupport.test_api_versionc                 C   s   | d| j | t| j| jd}|d|_|j| j	d}|dkrZ|| j
ks|tn"d| j d| j	 d}||ks|td	S )
zK
        Test if the download url is correct for each API version.
        r   r   r   r4   r   zhttps://zenodo.org/records/z/files/z?download=1N)r   r   r   r   r1   r   r   r   r6   r5   file_urlr   )r   r   r   rY   r!   r6   Zexpected_urlr"   r"   r#   test_download_url  s    	z&TestZenodoAPISupport.test_download_urlrY   c                 C   sj   | d| j | td|d}t| j| jd}|d|_|	| |j
| jd| j iksftdS )zX
        Test if population of registry is correctly done for each API version.
        r   r   )rF   r;   r   r   N)r   r   r   r   r   r1   r   r   r   Zpopulate_registryregistryr5   file_checksumr   )r   r   Ztmp_pathrY   Zpuppyr!   r"   r"   r#   test_populate_registry  s    	
z+TestZenodoAPISupport.test_populate_registryN)r   r   r   r   r   r1   r   r5   r   r   Zlegacy_api_responseZnew_api_responseZinvalid_api_responser'   markparametrizer   r   r   r"   r"   r"   r#   r     s   

   	 

r   )=r   r:   r{   tempfiler   r'   ZtqdmImportErrorZparamikor   r   Zdownloadersr   r   r   r   r	   r
   r   r   r   Z
processorsr   utilsr   r   r   r   r   r   r   r   r   rP   Z	ZENODOURLrE   ZDATAVERSEURLr   Zskipifr   r$   r&   r*   networkr0   r2   r9   r?   rI   rS   r]   ra   rh   rl   rm   rn   r   r   r   r   r   r"   r"   r"   r#   <module>   s   

,(
	


		







2