U
    ?h3g                     @   s,  d Z ddlZddlZddlmZ ddlmZ ddlZddlm	Z	m
Z
mZmZmZ ddlmZmZmZ ddlmZmZ dd	lmZ dd
lmZmZ ddlmZmZmZmZmZmZmZm Z m!Z!m"Z"m#Z# e$ee%j&d Z'e Z(e Z)e Z*e Z+e Z,e Z-ddiZ.ej/dd Z0ej1j2dd Z3ej1j2dd Z4ej1j2dd Z5dd Z6ej1j2ej1j7de)e*e+e-gddddgdd d! Z8ej1j2ej1j7de)e*e+e-gddddgdd"d# Z9G d$d% d%Z:ej1j2d&d' Z;G d(d) d)Z<ej1j2d*d+ Z=ej1j2d,d- Z>ej1j2d.d/ Z?ej1j2d0d1 Z@d2d3 ZAd4d5 ZBd6d7 ZCej1j2d8d9 ZDd:d; ZEd<d= ZFd>d? ZGd@dA ZHdBdC ZIdDdE ZJdFdG ZKej1j2dHdI ZLdJdK ZMdLdM ZNej1j2dNdO ZOdPdQ ZPdRdS ZQdTdU ZRej1j2ej17dVddWgdXdY ZSej1j2ej1j7de*e+e-gdddgddZd[ ZTej1j2d\d] ZUd^d_ ZVdS )`z+
Test the core class and factory function.
    N)Path)TemporaryDirectory   )createPoochretrievedownload_actionstream_download)
get_loggertemporary_fileos_cache)	file_hashhash_matches)core)HTTPDownloaderFTPDownloader   )pooch_test_urldata_over_ftppooch_test_figshare_urlpooch_test_zenodo_url pooch_test_zenodo_with_slash_urlpooch_test_dataverse_urlpooch_test_registrycheck_tiny_datacheck_large_datacapture_logmirror_directorydatatiny-data.txtZ@098h0894dba14b12085eacb204284b97e362f4f3e5a5807693cc90ef415c1b2dc                 C   s
   t t| S )z
    Mirror the test data folder on a temporary directory. Needed to avoid
    permission errors when pooch is installed on a non-writable path.
    )r   DATA_DIR)Ztmp_path r!   G/var/www/html/venv/lib/python3.8/site-packages/pooch/tests/test_core.pydata_dir_mirror5   s    r#   c               
   C   s  t  } d}t| }t L}t|d| d}| }| d dksHtd|ksTtt| |ksdtW 5 Q R X ||t| d kstt	| t
|t| kstt $}t|d| d}| dkstW 5 Q R X t (}t|t| | d}| dkstW 5 Q R X W 5 Q R X dS )z'Try downloading some data with retriever   N)
known_hashpathr   DownloadingSHA256 hash of downloaded file: )r   BASEURLr   r   getvaluesplitAssertionErrorREGISTRYlenr   r   local_store	data_fileurllog_filefnamelogsr!   r!   r"   test_retrieve>   s$    
r6   c               
   C   s   t  } d}t| }t N}t|d| |d}| }| d dksHtd|ksTtt| |ksdtW 5 Q R X |tj	|d kstt
| t|t| kstW 5 Q R X dS )zATry downloading some data with retrieve and setting the file namer   N)r$   r%   r4   r   r&   r'   r   )r   r)   r   r   r*   r+   r,   r-   osr%   r   r   r/   r!   r!   r"   test_retrieve_fnameX   s    r8   c               
   C   s   d} t |  }td|  }zt d}t|d| d}|	 }|
 d dksLtttd |ksdtd|ksptt|  |kstW 5 Q R X |t| kstt| t|t|  kstW 5 tjt|rtt| X dS )zETry downloading some data with retrieve to the default cache locationr   poochN)r$   r4   r   r&   zSHA256 hash of downloaded file)r)   r   r7   r%   existsstrremover   r   r*   r+   r,   resolver-   r   r   )r1   r2   Zexpected_locationr3   r4   r5   r!   r!   r"   test_retrieve_default_pathk   s     r>   c                 C   s<   t | dtd}t| d }|d}||ks0tt| dS )zASetup a pooch that already has the local data and test the fetch.some bogus URLr%   base_urlregistryr   N)r   r-   r;   fetchr,   r   )r#   puptruer4   r!   r!   r"   test_pooch_local   s
    
rF   r2   httpsZfigshareZzenodoZ	dataverse)Zidsc              
   C   s   t  }t|}d| d i}t|dt|d}t H}|d}| }| d dksZt| d d| dksvtW 5 Q R X t	| t  }|d}| dkstW 5 Q R X W 5 Q R X d	S )
z:Have pooch download the file from URL that is not base_urlr   r(   r%   rA   rB   urlsr   r&   ''.N)
r   r   r   r-   r   rC   r*   r+   r,   r   )r2   r0   r%   rI   rD   r3   r4   r5   r!   r!   r"   test_pooch_custom_url   s    
&
rM   c              
   C   s   t  }t|}t|d }t|| td}t H}|d}| }| d dksXt	| d d| dkstt	W 5 Q R X ||kst	t
| t|td kst	t  }|d}| dkst	W 5 Q R X W 5 Q R X d	S )
z:Setup a pooch that has no local data and needs to downloadr   r@   r   r&   rJ   rK   rL   r(   N)r   r   r;   r   r-   r   rC   r*   r+   r,   r   r   )r2   r0   r%   	true_pathrD   r3   r4   r5   r!   r!   r"   test_pooch_download   s    
&
rO   c                   @   s    e Zd ZdZdd Zdd ZdS )FakeHashMatchesz8Create a fake version of hash_matches that fails n timesc                 C   s   || _ d| _d S )Nr   )	nfailuresfailed)selfrQ   r!   r!   r"   __init__   s    zFakeHashMatches.__init__c                 O   s6   | j | jk r,|  j d7  _ t|d df|S t||S )z#Fail n times before finally passingr   r   Zbla)rR   rQ   r   )rS   argskwargsr!   r!   r"   r      s    zFakeHashMatches.hash_matchesN)__name__
__module____qualname____doc__rT   r   r!   r!   r!   r"   rP      s   rP   c                 C   s   t  }| tdtdj t|}t|ttd}t	
t"}t }|d W 5 Q R X W 5 Q R X dt|kstt|  d}t|dkst|d d	st|d d
| dstW 5 Q R X dS )z2Check that retrying the download is off by defaultr      r@   r   does not match the known hash
r   r   r&   rK   rL   N)r   setattrr   rP   r   r   r   r)   r-   pytestraises
ValueErrorr   rC   r;   r,   r*   stripr+   r.   
startswithendswithmonkeypatchr0   r%   rD   errorr3   r5   r!   r!   r"   (test_pooch_download_retry_off_by_default   s    rh   c                   @   s    e Zd ZdZdd Zdd ZdS )	FakeSleepz<Create a fake version of sleep that logs the specified timesc                 C   s
   g | _ d S )N)times)rS   r!   r!   r"   rT      s    zFakeSleep.__init__c                 C   s   | j | dS )z Store the time and doesn't sleepN)rj   append)rS   Zsecsr!   r!   r"   sleep   s    zFakeSleep.sleepN)rW   rX   rY   rZ   rT   rl   r!   r!   r!   r"   ri      s   ri   c                 C   s  t  t}| tdtdj t }| tjd|j t|}t	|d }d}t
|tt|d}t }|d}|  d}	t|	d| kst|	d d	st|	d d
| dsttt|dd|	dd D ]>\}
}d|kst|
dkrdnd}d|
 d| d|kstq|jdddddddddddgksBtW 5 Q R X ||ksZtt| t|td ksxtW 5 Q R X dS )z?Check that retrying the download works if the hash is differentr      rl   r   r%   rA   rB   Zretry_if_failedr]   r   r   r&   rK   rL   rJ   NFailed to downloadsr(   zdownload again z
 more time.r   r[                  	   
   )r   r^   r   rP   r   ri   timerl   r   r;   r   r)   r-   r   rC   r*   rb   r+   r.   r,   rc   rd   ziprangerj   r   r   )rf   r0   Z	fakesleepr%   rN   retriesrD   r3   r4   r5   ilinepluralr!   r!   r"   test_pooch_download_retry   s6    
   
"0r   c                 C   s   t  }| tdtdj t|}t|ttdd}t	
t"}t }|d W 5 Q R X W 5 Q R X |  d}t|dkst|d d	st|d d
| dstd|d kstd|d kstdt|kstW 5 Q R X dS )zFCheck that retrying the download fails after the set amount of retriesr   r[   r   rn   r   r]   r   r   r&   rK   rL   ro   zdownload again 1 more time.r\   N)r   r^   r   rP   r   r   r   r)   r-   r_   r`   ra   r   rC   r*   rb   r+   r.   r,   rc   rd   r;   re   r!   r!   r"   *test_pooch_download_retry_fails_eventually  s    r   c               
   C   sp   t  `} t| }dtd i}t|dt|d}td }|d}| dksPtW 5 Q R X t	| W 5 Q R X dS )zHSetup a pooch and check that no logging happens when the level is raisedr   r(   rH   CRITICALN)
r   r   r)   r   r-   r   rC   r*   r,   r   )r0   r%   rI   rD   r3   r4   r!   r!   r"   test_pooch_logging_level)  s    

r   c               
   C   s  t  } t| }t|d }t|ddd}|d W 5 Q R X t|ttd}t H}|	d}|
 }| d dks|t| d	 d
| dkstW 5 Q R X ||kstt| t|td kstt  }|	d}|
 dkstW 5 Q R X W 5 Q R X dS )zFSetup a pooch that already has the local data but the file is outdatedr   wutf-8encodingdifferent datar@   r   UpdatingrJ   rK   rL   r(   N)r   r   r;   openwriter   r)   r-   r   rC   r*   r+   r,   r   r   )r0   r%   rN   finrD   r3   r4   r5   r!   r!   r"   test_pooch_update8  s"    
&
r   c               
   C   s~   t  n} t| }t|d }t|ddd}|d W 5 Q R X t|ttdd}t	t
 |d W 5 Q R X W 5 Q R X dS )	z$Test that disallowing updates works.r   r   r   r   r   Fr%   rA   rB   Zallow_updatesN)r   r   r;   r   r   r   r)   r-   r_   r`   ra   rC   )r0   r%   rN   r   rD   r!   r!   r"   test_pooch_update_disallowedT  s    r   c               
   C   s   d} zdt j| < t n}t|}t|d }t|ddd}|d W 5 Q R X t|t	t
| d}tt |d W 5 Q R X W 5 Q R X W 5 t j|  X d	S )
zDTest that disallowing updates works through an environment variable.ZMYPROJECT_DISALLOW_UPDATESFalser   r   r   r   r   r   N)r7   environpopr   r   r;   r   r   r   r)   r-   r_   r`   ra   rC   )Zvariable_namer0   r%   rN   r   rD   r!   r!   r"   (test_pooch_update_disallowed_environmenth  s"    
"r   c                  C   s(   d} t | dtd}|j| d ks$tdS )zR
    Test if pooch.create appends a trailing slash to the base url if missing
    zhttps://mybase.urlN)rA   rB   r%   /)r   r    rA   r,   )rA   rD   r!   r!   r"   ,test_pooch_create_base_url_no_trailing_slash  s    r   c                 C   s:  t  }tj|}t|ttd}t p}t	t
}|d W 5 Q R X dt|jks\t| }| d dksxt| d d| dkstW 5 Q R X W 5 Q R X t| ttd}t t}t	t
}|d W 5 Q R X dt|jkst| }| d d	kst| d d|  dks,tW 5 Q R X d
S )z>Raise an exception if the file hash doesn't match the registryr@   r   z(tiny-data.txt)r   r&   rJ   rK   rL   r   N)r   r7   r%   abspathr   r)   REGISTRY_CORRUPTEDr   r_   r`   ra   rC   r;   valuer,   r*   r+   )r#   r0   r%   rD   r3   rg   r5   r!   r!   r"   test_pooch_corrupted  s$    0r   c               	   C   s2   t ddtd} tt | d W 5 Q R X dS )z=Should raise an exception if the file is not in the registry.zit shouldn't matterzthis shouldn't eitherr@   zthis-file-does-not-exit.csvN)r   r-   r_   r`   ra   rC   rD   r!   r!   r"   test_pooch_file_not_in_registry  s      r   c                  C   sL   t ddd} | tjtd | jtks.t| j	
 tt
 ksHtdS )z,Loading the registry from a file should workr(   r%   rA   registry.txtNr   load_registryr7   r%   joinr    rB   r-   r,   registry_filessortlistr   r!   r!   r"   test_pooch_load_registry  s    r   c                  C   sL   t ddd} | tjtd | jtks.t| j	
 tt
 ksHtdS )z8Loading the registry from a file and strip line commentsr(   r   zregistry_comments.txtNr   r   r!   r!   r"   !test_pooch_load_registry_comments  s    r   c               	   C   s   t jtd} tddd}t| d}|| W 5 Q R X |jtksHt	|j
 tt ksbt	tddd}t| ddd}|| W 5 Q R X |jtkst	|j
 tt kst	dS )	z'Loading the registry from a file objectr   r(   r   rbrr   r   N)r7   r%   r   r    r   r   r   rB   r-   r,   r   r   r   )r%   rD   r   r!   r!   r"    test_pooch_load_registry_fileobj  s    r   c                  C   sD   t ddd} | tjtd | jtks.t| j	ddiks@tdS )z8Load the registry from a file with a custom URL insertedr(   r   zregistry-custom-url.txtr   zhttps://some-site/tiny-data.txtN)
r   r   r7   r%   r   r    rB   r-   r,   rI   r   r!   r!   r"   #test_pooch_load_registry_custom_url  s    r   c               	   C   s<   t ddi d} tt | tjtd W 5 Q R X dS )z?Should raise an exception when a line doesn't have two elementsr(   r@   zregistry-invalid.txtN)	r   r_   r`   IOErrorr   r7   r%   r   r    r   r!   r!   r"   %test_pooch_load_registry_invalid_line  s    r   c                  C   s@   t ddd} | tjtd d| jks.td| jks<tdS )zCShould check that spaces in filenames are allowed in registry filesr(   r   zregistry-spaces.txtzfile with spaces.txtzother with spaces.txtN)r   r   r7   r%   r   r    rB   r,   r   r!   r!   r"   $test_pooch_load_registry_with_spaces  s    r   c                  C   sn   t tttd} | dstt ttd td} | dr<tddi}|t t tt|d} | drjtdS )FShould correctly check availability of existing and non existing filesr@   r   z
wrong-url/znot-a-real-data-file.txtZnotarealhashN)r   r    r)   r-   is_availabler,   update)rD   rB   r!   r!   r"   test_check_availability  s    
r   c              	   C   sf   t | dR}tt|dddddd}t| jd}|jd|dsFt|jd	|drXtW 5 Q R X d
S )r   r   r(   Z@baee0894dba14b12085eacb204284b97e362f4f3e5a5807693cc90ef415c1b2dZjdjdjdjdflld)r   doesnot_exist.zipr@   )port
downloaderr   N)r   r   r    replacer   Zserver_portr   r,   )Z	ftpserverr2   rD   r   r!   r!   r"   test_check_availability_on_ftp  s    
r   c               	   C   sF   dd } t tttd}d}tjt|d |jd| d W 5 Q R X dS )	z@Should raise an exception if the downloader doesn't support thisc                 S   s   dS )z,A downloader that doesn't support check_onlyNr!   )r2   outputr9   r!   r!   r"   r     s    z>test_check_availability_invalid_downloader.<locals>.downloaderr@   z%does not support availability checks.)matchr   r   N)r   r    r)   r-   r_   r`   NotImplementedErrorr   )r   rD   msgr!   r!   r"   *test_check_availability_invalid_downloader  s
    r   c           	   
   C   s   dd }t  }t|}t|ttd}t \}|jd|d}| }| }t	|dks\t
|d  d dkstt
|d	 d
kst
W 5 Q R X |  jrt
t| t  }|d}| dkst
W 5 Q R X W 5 Q R X dS )z%Setup a downloader function for fetchc                 S   s   t  d t | || dS )z2Download through HTTP and warn that we're doing itdownloader executedN)r
   infor   )r2   Zoutput_filerD   r!   r!   r"   download  s    z,test_fetch_with_downloader.<locals>.downloadr@   zlarge-data.txtr   r   r   r&   r   r   r(   N)r   r   r   r)   r-   r   rC   r*   
splitlinesr.   r,   r+   Z
readouterrerrr   )	Zcapsysr   r0   r%   rD   r3   r4   r5   linesr!   r!   r"   test_fetch_with_downloader  s     
r   c              	   C   sH   t | tddid}tt}|d W 5 Q R X dt|jksDtdS )z!Test an invalid hashing algorithmr   z	blah:1234r@   z'blah'N)	r   r)   r_   r`   ra   rC   r;   r   r,   )r#   rD   excr!   r!   r"   test_invalid_hash_alg4  s      r   c              	   C   s   t | d }t| t|d}| }W 5 Q R X dD ]V}t|}|| d| d|  i}t| d|d}||	dkst
t| q6dS )z2Test different hashing algorithms using local datar   r   )sha512md5:r?   r@   N)r;   r   r   readhashlibnewr   	hexdigestr   rC   r,   )r#   r4   r   r   algZhasherrB   rD   r!   r!   r"   #test_alternative_hashing_algorithms?  s    

r   c               
   C   s   t tddd\} }| dks t|dks,tt }t t|dd\} }W 5 Q R X | dks^t|dksjtt B}t|d	d
d}|d W 5 Q R X t t|t|d\} }W 5 Q R X | dkst|dkstdS )z>Test that the right action is performed based on file existingzthis_file_does_not_exist.txtN)r$   r   r&   znot the correct hashr   r   r   r   r   z	some datarC   ZFetching)r   r   r,   r   r   r   r   )actionZverbtmpr   r!   r!   r"   test_download_actionN  s      
"r   r4   zsubdir/tiny-data.txtc              	   C   sr   t d |  }t|  }t }t H}t||  }| r:tt||||dd | sXttt	| W 5 Q R X dS )z9Check that downloading a file over HTTP works as expectedzstore/N)r9   )
r)   r-   r   r   r   r:   r,   r	   r   r;   )r4   r2   r$   r   r0   Zdestinationr!   r!   r"   test_stream_downloadc  s    r   c              	   C   sz   t  j}tj|}t|| d}|  t|jdks:td|jksHtd|jksVt|jD ]}|	| q\W 5 Q R X dS )z;Check that the registry is correctly populated from the APIr   r   r   z	store.zipN)
r   r7   r%   r   r   load_registry_from_doir.   rB   r,   rC   r2   r0   r%   rD   filenamer!   r!   r"   test_load_registry_from_dois  s    
r   c               	   C   sp   t } t \}tj|}t|| d}|  t|jdks>t	d|jksLt	|jD ]}|
| qRW 5 Q R X dS )zs
    Check that the registry is correctly populated from the Zenodo API when
    the filename contains a slash
    r   r   z!santisoler/pooch-test-data-v1.zipN)ZENODOURL_W_SLASHr   r7   r%   r   r   r   r.   rB   r,   rC   r   r!   r!   r"   -test_load_registry_from_doi_zenodo_with_slash  s    
r   c               	   C   s@   t dtd} tt}|   W 5 Q R X dt|jks<tdS )z(Check that non-DOI URLs produce an errorr(   r   zonly implemented for DOIsN)	r   r)   r_   r`   ra   r   r;   r   r,   )rD   r   r!   r!   r"   !test_wrong_load_registry_from_doi  s    r   )WrZ   r   r7   pathlibr   tempfiler   r_   r   r   r   r   r   r	   utilsr
   r   r   hashesr   r   r(   Zdownloadersr   r   r   r   r   r   r   r   r   r   r   r   r   r;   __file__parentr    r-   r)   ZFIGSHAREURLZ	ZENODOURLr   ZDATAVERSEURLr   Zfixturer#   marknetworkr6   r8   r>   rF   ZparametrizerM   rO   rP   rh   ri   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r!   r!   r!   r"   <module>   s   4 



	





 


	
	


