U
    f/e+                     @   s  d Z ddlmZmZ ddlZddlZddlZddlm	  m
Z ddlmZ ddlmZ ddlmZ ddlmZ ejjejddd	d
dgejdddgejdddgdd Zejdd Zejddddd Zejdd Zejde G dd dZdS ) zl
Tests parsers ability to read and parse non-local files
and hence require a network connection to be read.
    )BytesIOStringION)	DataFrame)read_featherread_csvzcompress_type, extension)gzip.gz)bz2.bz2)zipz.zip)xzz.xzmodeexplicitinferenginepythoncc                 C   s   t | |||| d S )N)check_compressed_urls)salaries_tablecompress_type	extensionr   r    r   G/tmp/pip-unpacked-wheel-tiezk1ph/pandas/tests/io/parser/test_network.pytest_compressed_urls   s    r   c                 C   s8   d}|| }|dkr|}t |d||d}t||  d S )NzXhttps://github.com/pandas-dev/pandas/raw/master/pandas/tests/io/parser/data/salaries.csvr   	)sepcompressionr   r   tmassert_frame_equal)r   r   r   r   r   base_urlurlZ	url_tabler   r   r   r   "   s    r   z"https://raw.githubusercontent.com/T)Zcheck_before_testc                  C   s(   d} t | ddd}|jd dks$tdS )zN
    read_csv should honor the requested encoding for URLs.

    GH 10424
    zihttps://raw.githubusercontent.com/pandas-dev/pandas/master/pandas/tests/io/parser/data/unicode_series.csvzlatin-1N)encodingheader)      u$   Á köldum klaka (Cold Fever) (1994))r   locAssertionError)pathdfr   r   r   test_url_encoding_csv4   s    r+   c                 C   s   t | ddddS )z DataFrame with the tips dataset.iodatacsvtips.csvr   )datapathr   r   r   tips_dfC   s    r1   s3_resourcec                   @   s   e Zd Zeddd Zdd Zdd Zdd	 Zd
d Z	dd Z
dd Zdd Zdd Zdd Zejjddddd Zejjdddeddd Zdd Zd d! Zd"d# Zedd$d% Zd&S )'TestS3s3fsc                 C   sx   dD ]>\}}t d| ||d}t|ts,t|jr6tt|| qt d|d}t|ts^t|jrhtt|| d S )N) N)r	   r   )r   r
   s3://pandas-test/tips.csv)r   storage_optionszs3://cant_get_it/tips.csvr8   r   
isinstancer   r(   emptyr   r    selfr1   s3soextcompr*   r   r   r   test_parse_public_s3_bucketL   s    

z"TestS3.test_parse_public_s3_bucketc                 C   s@   t dd|d}t|tst|jr&tt|jd d | d S )Nzs3n://pandas-test/tips.csv
   nrowsr8   r   r;   r   r(   r<   r   r    ilocr>   r1   r?   r*   r   r   r   test_parse_public_s3n_bucketa   s    
z#TestS3.test_parse_public_s3n_bucketc                 C   s@   t dd|d}t|tst|jr&tt|jd d | d S )Nzs3a://pandas-test/tips.csvrC   rD   rF   rH   r   r   r   test_parse_public_s3a_bucketi   s    
z#TestS3.test_parse_public_s3a_bucketc                 C   sT   dD ]J\}}t d| d||d}t|ts.t|jr8tt|jd d | qd S )Nr5   r7   rC   )rE   r   r8   rF   r=   r   r   r   !test_parse_public_s3_bucket_nrowsp   s    
z(TestS3.test_parse_public_s3_bucket_nrowsc           
   
   C   s   d}dD ]\}}t d| |||dd}|j|ks4tdD ]J}| }t|tsRt|jr\t|j|| ||d   }	t	|	| q8W 5 Q R X qd S )N   r5   r7   )	chunksizer   r8   r   r&      r&   
r   rM   r(   Z	get_chunkr;   r   r<   rG   r   r    
r>   r1   r?   rM   r@   rA   Z	df_readerZi_chunkr*   Ztrue_dfr   r   r   #test_parse_public_s3_bucket_chunked|   s&    
 
z*TestS3.test_parse_public_s3_bucket_chunkedc           
   
   C   s   d}dD ]\}}t d| ||d|dd}|j|ks6tdD ]J}| }t|tsTt|jr^t|j|| ||d   }	t	|	| q:W 5 Q R X qd S )NrL   r5   r7   r   )rM   r   r   r8   rN   r&   rP   rQ   r   r   r   *test_parse_public_s3_bucket_chunked_python   s(    
 
z1TestS3.test_parse_public_s3_bucket_chunked_pythonc                 C   sJ   dD ]@\}}t d| d||d}t|ts.t|jr8tt|| qd S )Nr5   r7   r   r   r   r8   r:   r=   r   r   r   "test_parse_public_s3_bucket_python   s    
z)TestS3.test_parse_public_s3_bucket_pythonc                 C   sF   dD ]<}t d| dd|d}t|ts*t|jr4tt|| qd S )N)r6   r	   r   r7   r   r   rT   r:   )r>   r1   r?   r@   r*   r   r   r   test_infer_s3_compression   s    
z TestS3.test_infer_s3_compressionc                 C   sV   dD ]L\}}t d| dd||d}t|ts0t|jr:tt|jd d | qd S )Nr5   r7   r   rC   )r   rE   r   r8   rF   r=   r   r   r   (test_parse_public_s3_bucket_nrows_python   s    
z/TestS3.test_parse_public_s3_bucket_nrows_pythonc              	   C   sP   d}t jt|d td|d W 5 Q R X t jt|d td W 5 Q R X d S )N#The specified bucket does not existmatchzs3://nyqpug/asdf.csvr9   zs3://cant_get_it/file.csv)pytestraisesIOErrorr   )r>   r?   msgr   r   r   test_read_s3_fails   s
    zTestS3.test_read_s3_failszGH#39155 s3fs upgradeF)reasonstrictc              	   C   s@   dd l }t|jjf}tj|dd |jd|d W 5 Q R X d S )Nr   rX   rY   z/s3://an_s3_bucket_data_doesnt_exit/not_real.csvr9   )botocoreFileNotFoundError
exceptionsClientErrorr[   r\   to_csvr>   r1   r?   rb   errorr   r   r   test_write_s3_csv_fails   s     zTestS3.test_write_s3_csv_failsZpyarrowc              	   C   s@   dd l }t|jjf}tj|dd |jd|d W 5 Q R X d S )Nr   rX   rY   z3s3://an_s3_bucket_data_doesnt_exit/not_real.parquetr9   )rb   rc   rd   re   r[   r\   Z
to_parquetrg   r   r   r   test_write_s3_parquet_fails   s    z"TestS3.test_write_s3_parquet_failsc              	   C   sj   |j jjddd}t|d  }t|dd}W 5 Q R X t|tsHt|j	rRtt|}t
|| d S )Npandas-testr/   )BucketKeyBodyutf8)r#   )metaclientZ
get_objectr   readr   r;   r   r(   r<   r   r    )r>   r2   Z	tips_fileZ	s3_objectbufferresultexpectedr   r   r   $test_read_csv_handles_boto_s3_object   s     
z+TestS3.test_read_csv_handles_boto_s3_objectc              	   C   s   dd l }ttjddtdd}t }|| t|	 
d}|djd|d	 |j  |jtjd
d, tdd|d ddd |jD kstW 5 Q R X d S )Nr   i    Zabcd)columnszutf-8rk   zlarge-file.csv)rm   rn   r4   )loggerzs3://pandas-test/large-file.csvrL   rD   )r   i  T c                 s   s   | ]}|j d d V  qdS )N)args).0xr   r   r   	<genexpr>  s     z8TestS3.test_read_csv_chunked_download.<locals>.<genexpr>)r4   r   nprandomZrandnlistr   rf   r   getvalueencoderl   Z
put_objectZS3FileSystemZclear_instance_cacheZat_levelloggingDEBUGr   recordsr(   )r>   r2   Zcaplogr?   r4   r*   Zstr_bufbufr   r   r   test_read_csv_chunked_download  s    

z%TestS3.test_read_csv_chunked_downloadc                 C   s   t d|d}t|| d S )Nzs3://pandas-test/tips#1.csvr9   r   )r>   r1   r?   rt   r   r   r   test_read_s3_with_hash_in_key   s    z$TestS3.test_read_s3_with_hash_in_keyc                 C   s$   t |}t d|d}t|| d S )Nz's3://pandas-test/simple_dataset.featherr9   )r   r   r    )r>   Zfeather_filer?   ru   resr   r   r   test_read_feather_s3_file_path%  s     z%TestS3.test_read_feather_s3_file_pathN)__name__
__module____qualname__tdZ
skip_if_norB   rI   rJ   rK   rR   rS   rU   rV   rW   r_   r[   markZxfailri   rj   rv   r   r   r   r   r   r   r   r3   I   s*   


r3   ) __doc__r,   r   r   r   Znumpyr   r[   Zpandas.util._test_decoratorsutilZ_test_decoratorsr   Zpandasr   Zpandas._testingZ_testingr   Zpandas.io.feather_formatr   Zpandas.io.parsersr   r   networkZparametrizer   r   r+   Zfixturer1   ZusefixturesZskip_if_not_us_localer3   r   r   r   r   <module>   s4   




