U
    f/e                     @   s  d Z ddlZddlmZ ddlZddlZddlmZmZ ddl	Z
ddlZddlmZ ddlmZmZ ddlmZmZ ddlm  mZ ddlZddlmZ ddlmZ dd	lm Z m!Z!m"Z"m#Z#m$Z$ zddl%Z%d
Z&W n e'k
r   dZ&Y nX z0e  edde(d ddl)Z)W 5 Q R X d
Z*W n e'k
r6   dZ*Y nX ej+dZ,ej-ej.dej+j/e* pfeddkdddej.dej+j/e& dddgddd Z0ej-dd Z1ej-dd Z2ej-dd  Z3ej-d!d" Z4ej-d#d$ Z5ej-ej6ej7j8ej6ej7j9ej6ej7j:ej;d%d&ej;d'd&ej;d(d&ej;d)d&gdd*d+ Z<dHd-d.Z=d/d0 Z>d1d2 Z?d3d4 Z@d5d6 ZAd7d8 ZBd9d: ZCd;d< ZDd=d> ZEG d?d@ d@ZFG dAdB dBeFZGej+dCG dDdE dEeFZHG dFdG dGeFZIdS )Iz test parquet compat     N)BytesIO)catch_warningsfilterwarnings)
get_option)PY38is_platform_windows)pa_version_under1p0pa_version_under2p0)Version)FastParquetImplPyArrowImpl
get_engineread_parquet
to_parquetTFignorez	`np.bool`)categoryz4ignore:RangeIndex.* is deprecated:DeprecationWarningfastparquetmode.data_managerarrayz4fastparquet is not installed or ArrayManager is usedreason)Zmarkspyarrowpyarrow is not installed)paramsc                 C   s   | j S Nparamrequest r   @/tmp/pip-unpacked-wheel-tiezk1ph/pandas/tests/io/test_parquet.pyengine@   s    r!   c                   C   s   t std dS )Nr   r   )_HAVE_PYARROWpytestskipr   r   r   r    paU   s    
r%   c                   C   s*   t std ntddkr&td dS )Nzfastparquet is not installedr   r   z.ArrayManager is not supported with fastparquetr   )_HAVE_FASTPARQUETr#   r$   r   r   r   r   r    fp\   s
    
r'   c                   C   s   t dddgddS )N         foo)AB)pd	DataFramer   r   r   r    	df_compate   s    r0   c               
   C   sD   t tdttddtjdddddd	dgt jd
ddd} | S )Nabcr(            @      @float64dtypeTF20130101r*   periods)abdef)r.   r/   listrangenparange
date_range)dfr   r   r    df_cross_compatj   s    rF   c                   C   s   t tddtjdgdd dgdddgddd	gttd
dtdddtjdddddtjdgdddgt jdddt 	dt j
t 	dgdS )Nr1   r;   c   foo   bars   bazr+   barbazr(   r2   r*      u1r3   r4   r5   r6          @g      @TFr8   r9   Z20130103)stringZstring_with_nanZstring_with_nonebytesunicodeintZuintfloatZfloat_with_nanbooldatetimeZdatetime_with_nat)r.   r/   r@   rB   nanrA   rC   astyperD   	TimestampZNaTr   r   r   r    df_full|   s$    

rY   z2019-01-04T16:41:24+0200z%Y-%m-%dT%H:%M:%S%zz2019-01-04T16:41:24+0215z2019-01-04T16:41:24-0200z2019-01-04T16:41:24-0215c                 C   s   | j S r   r   r   r   r   r    timezone_aware_date_list   s    rZ   r)   c
              	      s   p
ddipi dkr |r4|d< |d<  fdd}
dkrtt  |
|	 W 5 Q R X n|
|	 dS )a  Verify parquet serializer and deserializer produce the same results.

    Performs a pandas to disk and disk to pandas round trip,
    then compares the 2 resulting DataFrames to verify equality.

    Parameters
    ----------
    df: Dataframe
    engine: str, optional
        'pyarrow' or 'fastparquet'
    path: str, optional
    write_kwargs: dict of str:str, optional
    read_kwargs: dict of str:str, optional
    expected: DataFrame, optional
        Expected deserialization result, otherwise will be equal to `df`
    check_names: list of str, optional
        Closed set of column names to be compared
    check_like: bool, optional
        If True, ignore the order of index & columns.
    repeat: int, optional
        How many times to repeat the test
    compressionNr!   c              
      sV   t | D ]H}jf tdd tf}W 5 Q R X tj| d qd S )NTrecord)check_names
check_likecheck_dtype)rA   r   r   r   tmassert_frame_equal)repeat_actualr`   r_   r^   rE   expectedpathread_kwargswrite_kwargsr   r    compare   s    z!check_round_trip.<locals>.compare)ra   ensure_clean)rE   r!   rh   rj   ri   rg   r^   r_   r`   rc   rk   r   rf   r    check_round_trip   s    "
rm   c              	   C   s.   d}t jt|d t| dd W 5 Q R X d S )Nz.engine must be one of 'pyarrow', 'fastparquet'matchr+   rJ   )r#   raises
ValueErrorrm   )r0   msgr   r   r    test_invalid_engine   s    rs   c              	   C   s$   t dd t|  W 5 Q R X d S )Nio.parquet.enginer   r.   option_contextrm   )r0   r%   r   r   r    test_options_py   s    rw   c              	   C   s$   t dd t|  W 5 Q R X d S )Nrt   r   ru   )r0   r'   r   r   r    test_options_fp   s    rx   c              	   C   s$   t dd t|  W 5 Q R X d S )Nrt   autoru   )r0   r'   r%   r   r   r    test_options_auto   s    rz   c              	   C   s  t tdtstt tdts$ttdd< t tdtsDtt tdtsVtt tdtshtW 5 Q R X tdd< t tdtstt tdtstt tdtstW 5 Q R X tdd> t tdtstt tdtstt tdtstW 5 Q R X d S )Nr   r   rt   ry   )
isinstancer   r   AssertionErrorr   r.   rv   )r'   r%   r   r   r    test_options_get_engine   s    r}   c               	   C   s.  ddl m}  | d}| d}ts(dnttjt|k }tsBdnttjt|k }to\| }tof| }|s*|s*|rd| d}t	j
t|d td	 W 5 Q R X n&d
}t	j
t|d td	 W 5 Q R X |rd| d}t	j
t|d td	 W 5 Q R X n&d}t	j
t|d td	 W 5 Q R X d S )Nr   )VERSIONSr   r   FzPandas requires version .z. or newer of .pyarrow.rn   ry   z%Missing optional dependency .pyarrow.z. or newer of .fastparquet.z)Missing optional dependency .fastparquet.)Zpandas.compat._optionalr~   getr"   r
   r   __version__r&   r   r#   rp   ImportErrorr   )r~   Z
pa_min_verZ
fp_min_verZhave_pa_bad_versionZhave_fp_bad_versionZhave_usable_paZhave_usable_fpro   r   r   r    "test_get_engine_auto_error_message  s8    



r   c              	   C   sj   | }t  T}|j||d d t||d}t || t||ddgd}t ||ddg  W 5 Q R X d S )Nr!   r[   r!   r;   r=   r!   columns)ra   rl   r   r   rb   )rF   r%   r'   rE   rh   resultr   r   r    test_cross_engine_pa_fp>  s    
r   c              
   C   s   |}t  j}|j||d d tddD t||d}t || t||ddgd}t ||ddg  W 5 Q R X W 5 Q R X d S )Nr   Tr\   r   r;   r=   r   )ra   rl   r   r   r   rb   )r   rF   r%   r'   rE   rh   r   r   r   r    test_cross_engine_fp_paL  s    
r   c                   @   s*   e Zd Zdd Zdd Zejdd ZdS )Basec              
   C   sB   t  0}tj||d t|||d d W 5 Q R X W 5 Q R X d S )Nrn   r[   )ra   rl   r#   rp   r   )selfrE   r!   excerr_msgrh   r   r   r    check_error_on_write[  s    
zBase.check_error_on_writec              
   C   s>   t  ,}t | t|||d d W 5 Q R X W 5 Q R X d S )Nr   )ra   rl   Zexternal_error_raisedr   )r   rE   r!   r   rh   r   r   r    check_external_error_on_writea  s    
z"Base.check_external_error_on_writec                 C   s.   |dkrt | d}t|}t|| d S )Nry   zfhttps://raw.githubusercontent.com/pandas-dev/pandas/master/pandas/tests/io/data/parquet/simple.parquet)r#   importorskipr   ra   rb   )r   r0   r!   urlrE   r   r   r    test_parquet_read_from_urlg  s    
zBase.test_parquet_read_from_urlN)__name__
__module____qualname__r   r   ra   networkr   r   r   r   r    r   Z  s   r   c                   @   s   e Zd Zdd Zdd Zdd Zejddd	d
dgdd Z	dd Z
dd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zd d! ZdS )"	TestBasicc              	   C   sJ   t dddgddt dtdddgfD ]}d}| ||t| q,d S )Nr(   r)   r*   r+   r8   z+to_parquet only supports IO with DataFrames)r.   SeriesrX   rB   r   r   rq   )r   r!   objrr   r   r   r    
test_errort  s    zTestBasic.test_errorc                 C   s6   t tdttddd}ddg|_t|| d S )Nr1   r(   r2   rO   rR   r+   rJ   )r.   r/   r@   rA   r   rm   )r   r!   rE   r   r   r    test_columns_dtypes  s    
zTestBasic.test_columns_dtypesc                 C   s   t tdttddd}d}ddg|_| ||t| ddg|_| ||t| td	ddddtd	ddddg|_| ||t| d S )
Nr1   r(   r2   r   %parquet must have string column namesr   rH   rI   i  )r.   r/   r@   rA   r   r   rq   rU   )r   r!   rE   rr   r   r   r    test_columns_dtypes_invalid  s    

z%TestBasic.test_columns_dtypes_invalidr[   Ngzipsnappybrotlic                 C   sP   |dkrt d n|dkr&t d tddddgi}t||d|id d S )	Nr   r   r,   r(   r)   r*   r[   rj   )r#   r   r.   r/   rm   )r   r!   r[   rE   r   r   r    test_compression  s    
zTestBasic.test_compressionc                 C   sJ   t tdttddd}t dtdi}t|||ddgid d S )Nr1   r(   r2   r   rO   r   rg   ri   )r.   r/   r@   rA   rm   )r   r!   rE   rg   r   r   r    test_read_columns  s       zTestBasic.test_read_columnsc                 C   s   |dk}t ddddgi}t|| dddgt jdddtd	dddgg}|D ]2}||_t|t jrt|jd |_t|||d
 qPdddg|_d|j_	t|| d S )Nr   r,   r(   r)   r*   r2   r8   r9   r1   )r^   r   r+   )
r.   r/   rm   rD   r@   indexr{   ZDatetimeIndex
_with_freqname)r   r!   r^   rE   Zindexesr   r   r   r    test_write_index  s     
zTestBasic.test_write_indexc                 C   s>   |}t ddddgi}t jdddg}||_t|| d S )Nr,   r(   r)   r*   r;   r(   r;   r)   r<   r(   )r.   r/   
MultiIndexfrom_tuplesr   rm   )r   r%   r!   rE   r   r   r   r    test_write_multiindex  s
    zTestBasic.test_write_multiindexc                 C   s   |}t jdddd}t jtjdt| dtdd}t jj	d	d
g|gddgd}|j
d d}||fD ]4}||_t|| t||dddgi|ddg d qhd S )Nz01-Jan-2018z01-Dec-2018ZMS)freqr)   r*   ABCr   Level1Level2leveldate)namesr   r,   r-   )ri   rg   )r.   rD   r/   rB   randomrandnlenr@   r   Zfrom_productcopyr   rm   )r   r%   r!   datesrE   Zindex1index2r   r   r   r    test_multiindex_with_columns  s"    $
 
  
 
z&TestBasic.test_multiindex_with_columnsc              	   C   s   t dddgdddgd}d dd	}|jd
d}t||||d t jdddgdddgddddgd}t||||d ddddddddgddddddddgg}t jttddd tdD d|d}|jd
d}t||||d d S )Nr(   r)   r*   qrs)r;   r<   F)r[   r   T)Zdroprj   rg   ZzyxZwvuZtsr)r   rJ   rK   r+   quxonetwo   c                 S   s   g | ]
}| qS r   r   ).0ir   r   r    
<listcomp>  s     z7TestBasic.test_write_ignoring_index.<locals>.<listcomp>)r   r   )r.   r/   Zreset_indexrm   r@   rA   )r   r!   rE   rj   rg   arraysr   r   r    test_write_ignoring_index  s$    
  z#TestBasic.test_write_ignoring_indexc                 C   sB   t jdddg}t jtjdd|d}d}| ||t| d S )Nr   r   r   r2   r*   r   Y\s*parquet must have string column names for all values in\s*each level of the MultiIndex)	r.   r   r   r/   rB   r   r   r   rq   )r   r!   Z
mi_columnsrE   rr   r   r   r    test_write_column_multiindex  s
    z&TestBasic.test_write_column_multiindexc              	   C   sh   |}ddddddddgddddddddgg}t jtjdd|d}d	d
g|j_d}| ||t| d S )NrJ   rK   r+   r   r(   r)   r   r   r   r   r   )	r.   r/   rB   r   r   r   r   r   rq   r   r%   r!   r   rE   rr   r   r   r    &test_write_column_multiindex_nonstring	  s    z0TestBasic.test_write_column_multiindex_nonstringc              	   C   s^   |}ddddddddgddddddddgg}t jtjdd|d}d	d
g|j_t|| d S )NrJ   rK   r+   r   r   r   r   r   Z	ColLevel1Z	ColLevel2)r.   r/   rB   r   r   r   r   rm   r   r%   r!   r   rE   r   r   r    #test_write_column_multiindex_string  s    z-TestBasic.test_write_column_multiindex_stringc                 C   s>   |}ddddg}t jtjdd|d}d|j_t|| d S )	NrJ   rK   r+   r   r   r2   r   Z	StringCol)r.   r/   rB   r   r   r   r   rm   r   r   r   r    test_write_column_index_string*  s
    z(TestBasic.test_write_column_index_stringc                 C   sH   |}ddddg}t jtjdd|d}d|j_d}| ||t| d S )	Nr(   r)   r*   r2   r   r   ZNonStringColr   )	r.   r/   rB   r   r   r   r   r   rq   r   r   r   r    !test_write_column_index_nonstring6  s    z+TestBasic.test_write_column_index_nonstring)r   r   r   r   r   r   r#   markparametrizer   r   r   r   r   r   r   r   r   r   r   r   r   r   r    r   s  s   
		 
r   z8ignore:CategoricalBlock is deprecated:DeprecationWarningc                   @   sd  e Zd Zdd Zdd Zdd Zdd Zd	d
 Zdd Ze	j
je oFeddddd Zdd Zede	j
ddgg gdd Zeddd Zeddd Zdd Zd d! Ze	j
d"eejgd#d$ Zd%d& Zd'd( Zedd)d* Zejdd+d,d-d. Zedd/d0 Z edd1d2 Z!d3d4 Z"d5d6 Z#ejdd+d,d7d8 Z$d9d: Z%d;S )<TestParquetPyArrowc                 C   sB   |}t jdddd}|d }||d< dd dg|d< t|| d S )Nr8   r*   Europe/Brusselsr:   tzdatetime_tzTZbool_with_none)r.   rD   r   rm   )r   r%   rY   rE   dtir   r   r    
test_basicE  s    
zTestParquetPyArrow.test_basicc                 C   s<   |}t jdddd|d< t|||ddg dddgid	 d S )
Nr8   r*   r   r   r   rO   rR   r   r   )r.   rD   rm   )r   r%   rY   rE   r   r   r    test_basic_subset_columnsQ  s    

z,TestParquetPyArrow.test_basic_subset_columnsc                 C   s:   |j |d}t|tstt|}t|}t|| d S )Nr   )r   r{   rP   r|   r   r   ra   rb   )r   r%   rY   Z	buf_bytesZ
buf_streamresr   r   r    *test_to_bytes_without_path_or_buf_provided_  s
    z=TestParquetPyArrow.test_to_bytes_without_path_or_buf_providedc                 C   s8   t jtdddtdd }| ||td d S )N   r2   r*   aaar   zDuplicate column names found	r.   r/   rB   rC   Zreshaper@   r   r   rq   r   r%   rE   r   r   r    test_duplicate_columnsj  s    $z)TestParquetPyArrow.test_duplicate_columnsc                 C   sN   t dt jdddi}| ||t t ddddgi}| ||tj d S )Nr;   1 dayr*   r9   r(   rN   )r.   r/   timedelta_ranger   NotImplementedErrorr   ZArrowExceptionr   r   r   r    test_unsupportedo  s    z#TestParquetPyArrow.test_unsupportedc                 C   sx   t  }t td|d< t jddddd dgt dddgd|d< t jddddddgddd	gd
d|d< t|| d S )NZabcdefr;   rJ   r+   rK   r6   r<   rG   r=   T)
categoriesZordered)r.   r/   Categoricalr@   ZCategoricalDtyperm   r   r   r   r    test_categoricalz  s    
  
z#TestParquetPyArrow.test_categoricalzlocalhost connection rejectedF)r   strictc                 C   s4   t d}|jf |}d|i}t||d||d d S )Ns3fs
filesystemzpandas-test/pyarrow.parquetrh   ri   rj   )r#   r   ZS3FileSystemrm   )r   r0   s3_resourcer%   s3sor   Zs3kwr   r   r    test_s3_roundtrip_explicit_fs  s    
z0TestParquetPyArrow.test_s3_roundtrip_explicit_fsc                 C   s   d|i}t ||d||d d S )Nstorage_optionsz s3://pandas-test/pyarrow.parquetr   rm   )r   r0   r   r%   r   r   r   r    test_s3_roundtrip  s    z$TestParquetPyArrow.test_s3_roundtripr   partition_colr,   c              
   C   sV   |  }|r.tstrdnd}|| |||< t|||dd|i|d |dddd d S )	NZint32r   zs3://pandas-test/parquet_dirr   )partition_colsr[   r   Tr(   )rg   rh   ri   rj   r_   rc   )r   r   r	   rW   rm   )r   r0   r   r%   r   r   Zexpected_dfZpartition_col_typer   r   r    test_s3_roundtrip_for_dir  s0    z,TestParquetPyArrow.test_s3_roundtrip_for_dirr   c                 C   s(   t  }|| t|}t|| d S r   )r   r   r   ra   rb   )r   r0   bufferZdf_from_bufr   r   r    test_read_file_like_obj_support  s    
z2TestParquetPyArrow.test_read_file_like_obj_supportc              	   C   sb   | dd | dd tjtdd td W 5 Q R X tjtdd |d W 5 Q R X d S )NHOMEZTestingUserUSERPROFILEz.*TestingUser.*rn   z~/file.parquet)Zsetenvr#   rp   OSErrorr   r   )r   r0   Zmonkeypatchr   r   r    test_expand_user  s    z#TestParquetPyArrow.test_expand_userc              	   C   s   ddg}|}t  l}|j||d d dd lm} |j|dd}t|jjdksTt	|jjt
|ksht	t|j|jks|t	W 5 Q R X d S )NrT   rR   r   r[   r   FZvalidate_schemar)   ra   ensure_clean_dirr   pyarrow.parquetparquetZParquetDatasetr   Z
partitionsZpartition_namesr|   setr   shape)r   r%   rY   r   rE   rh   pqdatasetr   r   r    test_partition_cols_supported  s    
z0TestParquetPyArrow.test_partition_cols_supportedc           	   	   C   s   d}|g}|}t  l}|j||d d dd lm} |j|dd}t|jjdksVt	|jjt
|ksjt	t|j|jks~t	W 5 Q R X d S )NrT   r  r   Fr  r(   r  )	r   r%   rY   r   partition_cols_listrE   rh   r	  r
  r   r   r    test_partition_cols_string  s    
z-TestParquetPyArrow.test_partition_cols_string	path_typec           	   	   C   sP   d}|g}|}t  0}||}|j||d t|j|jksBtW 5 Q R X d S )Nr-   )r   )ra   r  r   r   r  r|   )	r   r%   r0   r  r   r  rE   Zpath_strrh   r   r   r    test_partition_cols_pathlib  s    
z.TestParquetPyArrow.test_partition_cols_pathlibc                 C   s   t  }t|| d S r   )r.   r/   rm   r   r   r   r    test_empty_dataframe  s    z'TestParquetPyArrow.test_empty_dataframec                 C   sV   dd l }tdddgi}||jd| dg}|t}t||d|i|d d S )Nr   xr(   )typeschemar   )	r   r.   r/   r  fieldZbool_rW   rT   rm   )r   r%   r   rE   r  Zout_dfr   r   r    test_write_with_schema  s
    
z)TestParquetPyArrow.test_write_with_schemac                 C   sz   t t jdddgddt jdddgddt jdd dgd	dd
}t|| t dt jdddd gddi}t|| d S )Nr(   r)   r*   Int64r6   ZUInt32r;   rG   rO   )r;   r<   rG   )r.   r/   r   rm   r   r   r   r     test_additional_extension_arrays  s    
 z3TestParquetPyArrow.test_additional_extension_arraysz1.0.0)min_versionc              	   C   sV   t dt jdd dgddi}t d|" t|||d| dd W 5 Q R X d S )	Nr;   rG   zstring[pyarrow]r6   string_storagezstring[]rg   )r.   r/   r   rv   rm   rW   )r   r%   r  rE   r   r   r     test_pyarrow_backed_string_array,  s    z3TestParquetPyArrow.test_pyarrow_backed_string_arrayc                 C   s(   t dt jddddi}t|| d S )Nr=   z
2012-01-01r*   D)r:   r   )r.   r/   period_rangerm   r   r   r   r    test_additional_extension_types3  s     z2TestParquetPyArrow.test_additional_extension_typesc                 C   s  dd l m} ttdddd gdtdddd gdtddd	d gtd
dd
d gd}t &}||| t|}t|d
d}W 5 Q R X |d j	t
	dkstttjdddd gddtjdddd gddtjddd	d gddtjd
dd
d gddd}t|| d S )Nr   r(   r)   r*   Zint64Zuint8r;   r<   rG   TF)r;   r<   rG   r=   )use_nullable_dtypesr5   r  r6   ZUInt8rO   boolean)r  r  r   tabler   ra   rl   Zwrite_tabler   r7   rB   r|   r.   r/   rb   )r   r%   r	  r"  rh   Zresult1Zresult2rg   r   r   r    test_use_nullable_dtypes@  s*    
z+TestParquetPyArrow.test_use_nullable_dtypesc                 C   s0   t dt jddddi}t||ddid d S )	Nr;   z
2017-01-01Z1n
   r   r:   versionz2.0r   )r.   r/   rD   rm   r   r   r   r    test_timestamp_nanoseconds]  s    z-TestParquetPyArrow.test_timestamp_nanosecondsc                 C   s:   t st  d|g }tj|d|id}t||dd d S )N   index_as_colr   dataF)r`   )r	   r#   r$   r.   r/   rm   )r   r%   rZ   idxrE   r   r   r    test_timezone_aware_indexc  s
    
z,TestParquetPyArrow.test_timezone_aware_indexc              	   C   s^   t dttddi}t $}||| t||dgdd}W 5 Q R X t|dksZt	d S )Nr;   r   r*   r;   z==r   F)filtersZuse_legacy_datasetr(   )
r.   r/   r@   rA   ra   rl   r   r   r   r|   )r   r%   rE   rh   r   r   r   r    test_filter_row_groupsv  s    
   z)TestParquetPyArrow.test_filter_row_groupsc              	   C   s~   t jtjdddddgd}t }||| t||}W 5 Q R X |rdt	|j
t jjjsztnt	|j
t jjjsztd S )Nr$  r*   r,   r-   Cr   )r.   r/   rB   r   r   ra   rl   r   r   r{   Z_mgrcoreZ	internalsZArrayManagerr|   ZBlockManager)r   r%   Zusing_array_managerrE   rh   r   r   r   r    test_read_parquet_manager  s    
z,TestParquetPyArrow.test_read_parquet_managerN)&r   r   r   r   r   r   r   r   r   r#   r   Zxfailr   r   r   r   tdZ
skip_if_nor   r   r   r   r  r  strpathlibPathr  r  r  r  r  r  r#  r'  r-  r0  r3  r   r   r   r    r   C  sX   
&








r   c                   @   s   e Zd Zdd Zejjdddd Zdd Zd	d
 Z	dd Z
dd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd ZdS ) TestParquetFastParquetc                 C   sF   |}t jdddd}|d }||d< t jddd|d< t|| d S )	Nr8   r*   z
US/Easternr   r   r   r9   	timedelta)r.   rD   r   r   rm   )r   r'   rY   rE   r   r   r   r    r     s    
z!TestParquetFastParquet.test_basicznot supportedr   c                 C   s<   t jtdddtdd }d}| ||t| d S )Nr   r2   r*   r   r   z9Cannot create parquet dataset with duplicate column namesr   r   r'   rE   rr   r   r   r    r     s    $z-TestParquetFastParquet.test_duplicate_columnsc                 C   s@   t ddd dgi}t jddtjdgidd}t|||d d S )	Nr;   TFg      ?g        Zfloat16r6   r  )r.   r/   rB   rV   rm   r   r'   rE   rg   r   r   r    test_bool_with_none  s    z*TestParquetFastParquet.test_bool_with_nonec                 C   sV   t dt jddddi}| ||td  t ddddgi}d}| ||t| d S )	Nr;   Z2013Mr*   r%  r(   rN   z"Can't infer object conversion type)r.   r/   r  r   rq   r:  r   r   r    r     s
    z'TestParquetFastParquet.test_unsupportedc                 C   s&   t dt tdi}t|| d S )Nr;   r1   )r.   r/   r   r@   rm   )r   r'   rE   r   r   r    r     s    z'TestParquetFastParquet.test_categoricalc              	   C   sf   dt tddi}t|}t (}|j||d dd t||dgd}W 5 Q R X t|dksbt	d S )Nr;   r   r*   r(   )r[   Zrow_group_offsetsr.  )r/  )
r@   rA   r.   r/   ra   rl   r   r   r   r|   )r   r'   r=   rE   rh   r   r   r   r    r0    s    

z-TestParquetFastParquet.test_filter_row_groupsc                 C   s    t ||dd|id |dd d S )Nz$s3://pandas-test/fastparquet.parquetr   )r[   r   r   r   )r   r0   r   r'   r   r   r   r    r     s    z(TestParquetFastParquet.test_s3_roundtripc              	   C   sl   ddg}|}t  N}|j|d|d d tj|s8tdd l}||dj	}t
|dks^tW 5 Q R X d S )NrT   rR   r   r!   r   r[   r   Fr)   ra   r  r   osrh   existsr|   r   ZParquetFileZcatsr   r   r'   rY   r   rE   rh   r   Zactual_partition_colsr   r   r    r    s    
z4TestParquetFastParquet.test_partition_cols_supportedc              	   C   sh   d}|}t  N}|j|d|d d tj|s4tdd l}||dj	}t
|dksZtW 5 Q R X d S )NrT   r   r>  r   Fr(   r?  rB  r   r   r    r    s    
z1TestParquetFastParquet.test_partition_cols_stringc              	   C   sl   ddg}|}t  N}|j|dd |d tj|s8tdd l}||dj	}t
|dks^tW 5 Q R X d S )NrT   rR   r   )r!   r[   partition_onr   Fr)   r?  rB  r   r   r    test_partition_on_supported  s    
z2TestParquetFastParquet.test_partition_on_supportedc              
   C   sV   ddg}|}d}t jt|d. t }|j|dd ||d W 5 Q R X W 5 Q R X d S )NrT   rR   zYCannot use both partition_on and partition_cols. Use partition_cols for partitioning datarn   r   )r!   r[   rC  r   )r#   rp   rq   ra   r  r   )r   r'   rY   r   rE   rr   rh   r   r   r    3test_error_on_using_partition_cols_and_partition_on  s    
zJTestParquetFastParquet.test_error_on_using_partition_cols_and_partition_onc                 C   s*   t  }| }d|j_t|||d d S )Nr   r  r.   r/   r   r   r   rm   r;  r   r   r    r    s    z+TestParquetFastParquet.test_empty_dataframec                 C   s>   d|g }t j|d|id}| }d|j_t|||d d S )Nr(  r)  r*  r   r  rF  )r   r'   rZ   r,  rE   rg   r   r   r    r-    s
    
z0TestParquetFastParquet.test_timezone_aware_indexc              
   C   s\   t dddgi}t 8}|| tjtdd t|ddd W 5 Q R X W 5 Q R X d S )	Nr;   r(   r)   z!not supported for the fastparquetrn   r   T)r!   r   )	r.   r/   ra   rl   r   r#   rp   rq   r   )r   r'   rE   rh   r   r   r    &test_use_nullable_dtypes_not_supported  s
    

z=TestParquetFastParquet.test_use_nullable_dtypes_not_supportedN)r   r   r   r   r#   r   r$   r   r<  r   r   r0  r   r  r  rD  rE  r  r-  rG  r   r   r   r    r8    s   	

	r8  )	NNNNNTFTr)   )J__doc__rU   ior   r@  r6  warningsr   r   ZnumpyrB   r#   Zpandas._configr   Zpandas.compatr   r   Zpandas.compat.pyarrowr   r	   Zpandas.util._test_decoratorsutilZ_test_decoratorsr4  Zpandasr.   Zpandas._testingZ_testingra   Zpandas.util.versionr
   Zpandas.io.parquetr   r   r   r   r   r   r"   r   DeprecationWarningr   r&   r   Z
pytestmarkZfixturer   Zskipifr!   r%   r'   r0   rF   rY   nowtimezoneutcminmaxstrptimerZ   rm   rs   rw   rx   rz   r}   r   r   r   r   r   r   r8  r   r   r   r    <module>   s   

 






         
A+ Q
  L