U
    k/e                    @   s  d dl Z d dlZd dlZd dlZd dlZd dlZd dlmZ d dl	m
Z
m	Z	mZmZmZ d dlZd dlmZ d dlZd dlmZ d dlZd dlmZmZ d dlmZmZmZ d dl m!  mZ" d dl#m$Z$ d dl%Z&zd dl%m'Z( W n e)k
r   Y nX z(d dl*Z+d dl,mZ- dd	l.m/Z/m0Z0 W n e)k
r6   Y nX ej1j*Z2dddZ3dddZ4dddZ5dddZ6d	ddZ7G dd dZ8G dd dZ9ej1:dddddd d!d"d#gd$d% Z;ej1:dddddd d!d"d#gd&d' Z<G d(d) d)Z=G d*d+ d+Z>G d,d- d-Z?G d.d/ d/Z@G d0d1 d1ZAG d2d3 d3ZBd4d5 ZCd6d7 ZDG d8d9 d9ZEd:d; ZFd<d= ZGej1:d>d?gd@gfdAdB ZHdCdD ZIdEdF ZJdGdH ZKeLe"Me"jNdIdJ ZOdKdL ZPdMdN ZQdOdP ZRdQdR ZSdSdT ZTdUdV ZUdWdX ZVdYdZ ZWd[d\ ZXd]d^ ZYd_d` ZZdadb Z[dcdd Z\dedf Z]dgdh Z^didj Z_dkdl Z`dmdn Zadodp Zbej1:dqe&c drfe&d dsfe&e dtfe&f dufe&g dvfe&h dwfe&i dxfe&j dyfe&k dzfe&l d{fe&m d|fe&n d}fe&o d~fe&p dfe&q dfe&r dfe&jrdddfe&s dfe&te&te&f dfe&udddfe&vddfe&vdddfe&wddfe&xddfgdd Zydd Zzdd Z{dd Z|dd Z}dd Z~dd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zd
ddZdd Zdd Zdd Zdd Zdd Zdd ZG dd de&jZdddZdd ZddĄ ZddƄ ZG ddȄ de&jZddʄ Zdd̄ Zdd΄ ZddЄ Zej1:ddddggddՄ Zddׄ Zej1j*ddل Zddۄ Zdd݄ Zdd߄ Zdd Zdd Zej1j'ej1ddd Zdd Zej1:ddddgej1:dddgej1:de	ddde	dddgdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd  Zdd Zdd ZdS (      N)OrderedDict)datedatetimetime	timedeltatimezone)get_logical_type_pandas_api)invoke_scriptrandom_asciirands)Version)parquet   )dataframe_with_arraysdataframe_with_listsd   c                 C   sN  t tj| tjdtj| tjdtj| tjdtj| tjdtj| tjdtj| tjdtj| tj	dtj| tj
dtj| tjdtj| tjdtj| dktjd| ddtjd| ddtjd| ddtjd| ddtjd| ddtjd| d	dtjd| d
dtjd| dddd t| D d gdd t| d D  d g dg|  dS )Ndtyper   2016-01-01T00:00:00.001datetime64[s]datetime64[ms]datetime64[us]datetime64[ns]timedelta64[s]timedelta64[ms]timedelta64[us]timedelta64[ns]c                 S   s   g | ]}t |qS  str.0xr   r   =/tmp/pip-unpacked-wheel-seu8352k/pyarrow/tests/test_pandas.py
<listcomp>R   s     z%_alltypes_example.<locals>.<listcomp>c                 S   s   g | ]}t |qS r   r   r!   r   r   r$   r%   S   s         )uint8uint16uint32uint64int8int16int32int64float32float64boolzdatetime[s]zdatetime[ms]zdatetime[us]zdatetime[ns]r   r   r   r   r    Zstr_with_nullsZ	empty_str)pd	DataFramenparanger(   r)   r*   r+   r-   r.   r/   r0   r1   randomrandnrangesizer   r   r$   _alltypes_example9   s@     r<   FTc              	   C   s   |r
t jnt j}|j| |||r"dndd}	|	j|d}
|rJ|	j|sJt|d krV| }t	 . t
ddt tj|
|||rdndd	 W 5 Q R X d S )
Nr&   r   )schemapreserve_indexZnthreadsuse_threadsignoreelementwise comparison failedequivF)check_dtypeZcheck_index_type)paRecordBatchTablefrom_pandas	to_pandasr=   equalsAssertionErrorwarningscatch_warningsfilterwarningsDeprecationWarningtmassert_frame_equal)dfexpectedr@   expected_schemarD   r=   r>   as_batchklasstableresultr   r   r$   _check_pandas_roundtripX   s(    

  
rY   c                 C   s^   t j| d|d}|d k	r$|d kr$|}|d k	r:|j|ks:ttj| | jd}t	| | d S )NTrH   typename)
rE   arrayr[   rK   r3   SeriesrI   r]   rP   assert_series_equal)stype_expected_pa_typearrrX   r   r   r$   _check_series_roundtrips   s    re   c                 C   s   t j| d||d}| }t| }|d kr@|j| ksVtn|j||B  ksVt|d kr|d krrt| }nt| 	 }d ||	 < t
jt||dd d S )NT)rH   maskr[   FZcheck_names)rE   r^   rI   r3   isnull
null_countsumrK   r_   copyrP   r`   )valuesrS   rf   r[   rd   rX   Zvalues_nullsr   r   r$   _check_array_roundtrip   s    
rm   c                 C   s(   t j| d|d}| }t||  d S )NTrZ   rE   r^   rI   nptassert_array_equal)Znp_arrayr[   rd   rX   r   r   r$   "_check_array_from_pandas_roundtrip   s    rq   c                   @   s0  e Zd ZdZdd Zdd Zdd Zdd	 Zd
d Zdd Z	dd Z
dd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zd d! Zd"d# Zd$d% Zd&d' Zd(d) Zd*d+ Zd,d- Zd.d/ Zd0d1 Zd2d3 Zd4d5 Zd6d7 Zd8d9 Zd:d; Z d<d= Z!d>d? Z"d@dA Z#dBdC Z$dDdE Z%dFdG Z&dHdI Z'dJS )KTestConvertMetadataz9
    Conversion tests for Pandas metadata & indices.
    c                 C   s8   t ddddgi}tj|}|djdks4td S )Nr   r   r&      0)r3   r4   rE   rG   rH   fieldr]   rK   selfrR   rW   r   r   r$   test_non_string_columns   s    z+TestConvertMetadata.test_non_string_columnsc              	   C   s   t dddgdddgd}|d}tt& tj|}|	dj
d	ksRtW 5 Q R X | }t|jj
|j_
tt t||d
d W 5 Q R X d S )N      ?       @      @      @      @      @)r   r   r   1TrS   r>   )r3   r4   	set_indexpytestwarnsUserWarningrE   rG   rH   ru   r]   rK   rk   r    indexrY   )rw   rR   rW   rS   r   r   r$   "test_non_string_columns_with_index   s    
z6TestConvertMetadata.test_non_string_columns_with_indexc                 C   s   t jdddgdddgdddgdddgd}tjj|ddgd}tj|ddg }||sbttjj|ddgd}tj|ddg }||std S )	Nr   r&   rs         )r   r   r&   r   columns)r3   r4   rE   rG   rH   rJ   rK   rF   )rw   rR   rW   rS   Zrecord_batch_tableZrecord_batch_expectedr   r   r$   test_from_pandas_with_columns   s     z1TestConvertMetadata.test_from_pandas_with_columnsc                 C   s.   t ddddgi}dg|j_t|dd d S )Ndatar   r&   rs   aTr>   )r3   r4   r   namesrY   rw   rR   r   r   r$   %test_column_index_names_are_preserved   s    
z9TestConvertMetadata.test_column_index_names_are_preservedc                 C   s6   t jtjddt jdddddd}t|dd	 d S )
Nr   rs   z
2021-01-01Z50DZCET)periodsfreqtzr   Tr   )r3   r4   r5   r7   r8   
date_rangerY   r   r   r   r$   test_column_index_names_with_tz   s
    z3TestConvertMetadata.test_column_index_names_with_tzc                 C   s  d}t jdddddgit jddd|d	d
}t jdddddgit ddd
}tj|}tj|}t|jdksxt|	 }t
|| t|jt jstt|jddkst|jj|kst|	 }t
|| t|jt jstt|jddkst|jjd kstd S )Nfoor   r   r&   rs   r   r      stepr]   r   r         r   )r3   r4   
RangeIndexrE   rG   rH   lenr=   rK   rI   rP   rQ   
isinstancer   r	   Zget_rangeindex_attributer]   )rw   Z
index_namerR   df2rW   Ztable_no_index_namerX   result2r   r   r$   test_range_index_shortcut   s(    
z-TestConvertMetadata.test_range_index_shortcutc                 C   sl   t jdddddgit jddddd	d
}tjj|dd}|jdksFtd|jksTt|	 }t
|| d S )Nr   r   r&   rs   r   r   r   r   r   r   Tr   )r3   r4   r   rE   rG   rH   Znum_columnsrK   Zcolumn_namesrI   rP   rQ   )rw   rR   rW   restoredr   r   r$   $test_range_index_force_serialization   s    z8TestConvertMetadata.test_range_index_force_serializationc              	   C   sL   t jtjddddgd}t  tjdd t|dd	 W 5 Q R X d S )
Nr   r&   r   br   erroractionTr   )	r3   r4   r5   r7   r8   rL   rM   simplefilterrY   r   r   r   r$   test_rangeindex_doesnt_warn   s    
z/TestConvertMetadata.test_rangeindex_doesnt_warnc                 C   s<   t jddgddgg}t jdddg|d}t|d	d
 d S )NonetwoXYr   r   r&   r   rs   cr   Tr   r3   
MultiIndexfrom_arraysr4   rY   rw   r   rR   r   r   r$   test_multiindex_columns  s     z+TestConvertMetadata.test_multiindex_columnsc                 C   sJ   t jjddgt ddggddgd}t jdd	d
g|d}t|dd d S )Nr   r   z
2017-08-01z
2017-08-02Zlevel_1Zlevel_2r   r   r   r   r   Tr   )r3   r   r   DatetimeIndexr4   rY   r   r   r   r$   #test_multiindex_columns_with_dtypes	  s    z7TestConvertMetadata.test_multiindex_columns_with_dtypesc                 C   s   t jdgt jdgtdd}t|dd t jdgt jdgtdd}t|dd t jdgt jtdddgddd}t|dd d S )	Nr   r   r   Tr   皙?  object)r3   r4   Indexr   rY   r   r   r   r   r$   (test_multiindex_with_column_dtype_object  s     
z<TestConvertMetadata.test_multiindex_with_column_dtype_objectc                 C   s<   t jddgddgg}t jdddg|d}t|d	d
 d S )N   あ   いr   r   r   r   r   r   Tr   r   r   r   r   r$   test_multiindex_columns_unicode&  s    z3TestConvertMetadata.test_multiindex_columns_unicodec              	   C   s\   t jddgddgg}t jdddg|d}t  tjd	d
 t|dd W 5 Q R X d S )Nr   r   r   r   r   r   r   r   r   r   Tr   )r3   r   r   r4   rL   rM   r   rY   r   r   r   r$   test_multiindex_doesnt_warn+  s
    
z/TestConvertMetadata.test_multiindex_doesnt_warnc                 C   s    t dddg}t|dd d S )Nr   r   r   Tr   r3   r4   rY   r   r   r   r$   test_integer_index_column4  s    z-TestConvertMetadata.test_integer_index_columnc              	   C   sV  t jdddgt jjdddgddd	ggd d
gddd dgd}tt tjj	|dd}W 5 Q R X |j
j}|d \}}}}}|d dkst|d |d kst|d d kst|d dkst|d dkst|d |d kst|d \}	}
|	dkst|d |	kst|d d ks t|
d
ks.t|d |
ks@t|d |
ksRtd S )N)r   r   @)r&   r   g@)rs   r   g?r   r   r   rs   r&   r   r   r   __index_level_0__)r   r   Tr   r   r]   
field_nameNoneindex_columns)r3   r4   r   r   r   r   r   rE   rG   rH   r=   pandas_metadatarK   )rw   rR   tjscol1Zcol2Zcol3Zidx0r   Z
idx0_descrZ	foo_descrr   r   r$   test_index_metadata_field_name8  s2    z2TestConvertMetadata.test_index_metadata_field_namec                 C   s   t jdddgt jtdddd}tjj|dd	}|jj}|d
 \}|d d ksTt	|d dksdt	|d dkstt	|d }|d dkst	|d dkst	d S )Nr   r   rz   r&   r   r{   rs   r   r|   defcategoryr   r   Tr   column_indexesr]   pandas_typeZcategorical
numpy_typer,   metadataZnum_categoriesrs   orderedF)
r3   r4   r   listrE   rG   rH   r=   r   rK   rw   rR   r   r   r   Zmdr   r   r$   test_categorical_column_indexZ  s    
z1TestConvertMetadata.test_categorical_column_indexc                 C   s   t jdddgt jtdddd}tjj|dd	}|jj}|d
 \}|d dksTt	|d |d ksht	|d dksxt	|d dkst	|d }t
|dkst	|d dkst	d S )Nr   r   r   r   Zstringzr\   r   Tr   r   r]   r   r   r   r   unicoder   r   encodingUTF-8)r3   r4   r   r   rE   rG   rH   r=   r   rK   r   r   r   r   r$   test_string_column_indexk  s    
z,TestConvertMetadata.test_string_column_indexc                 C   s   t jdddgt jddddd}tjj|d	d
}|jj}|d \}|d d ksRt|d dksbt|d dksrt|d }|d dkstd S )Nr   r   r   
2017-01-01rs   America/New_Yorkstartr   r   r   Tr   r   r]   r   
datetimetzr   r   r   r   )	r3   r4   r   rE   rG   rH   r=   r   rK   r   r   r   r$   test_datetimetz_column_index~  s       
z0TestConvertMetadata.test_datetimetz_column_indexc                 C   s4   t dt jddddi}|d}t|dd d S )Nr   r   rs   r   r   Tr   )r3   r4   r   r   rY   r   r   r   r$   test_datetimetz_row_index  s       
z-TestConvertMetadata.test_datetimetz_row_indexc                 C   sF   t dddgdddgd}|jd|d< |d}t|dd d S )	Nr   r&   rs   r   r   r   r   Tr   )r3   r4   r   astyper   rY   r   r   r   r$   test_categorical_row_index  s    
z.TestConvertMetadata.test_categorical_row_indexc              	   C   s<   t jddgtdd}tt tj| W 5 Q R X d S )Nr   r   aar   )	r3   r4   r   r   raises
ValueErrorrE   rG   rH   r   r   r   r$   *test_duplicate_column_names_does_not_crash  s    z>TestConvertMetadata.test_duplicate_column_names_does_not_crashc              
   C   s   ddgddgg}|D ]x}t jj|dgdd}t j|gdg}t j|||g}tt |	  W 5 Q R X tt |	  W 5 Q R X qd S )Nr   r   r   Fsafer   )
rE   DictionaryArrayr   rF   rG   from_batchesr   r   
IndexErrorrI   )rw   indicesZindsrd   batchrW   r   r   r$   #test_dictionary_indices_boundscheck  s    z7TestConvertMetadata.test_dictionary_indices_boundscheckc                 C   s&   t jddgidgd}t|dd d S )Nr   r   u   うr   Tr   r   r   r   r   r$   *test_unicode_with_unicode_column_and_index  s    z>TestConvertMetadata.test_unicode_with_unicode_column_and_indexc              
   C   s   t ddgddgd}ddgddgdd	gfD ]r}t j|td
|_tt tj	
| W 5 Q R X | }|jjt|_tt t||dd W 5 Q R X q.d S )Nr   r&   rs   r   r   r      a2      ?r   Tr   )r3   r4   r   r   r   r   r   r   rE   rG   rH   rk   rl   r   r    rY   )rw   rR   colsrS   r   r   r$   test_mixed_column_names  s    z+TestConvertMetadata.test_mixed_column_namesc                 C   s   t dt tj  kr"t dk r0n n
td dg}d }||i}t|}tjj	|dd}|
 }|jd |jd kst|jjd |jjd kst|jd |kstd S )	N2.0.02.1.0Regression in pandas 2.0.0r   r   Tr   r   )r   r3   __version__r   skipencoder4   rE   rG   rH   rI   rl   rK   r   r   )rw   Zcolumn_datakeyr   rR   r   r   r   r   r$   test_binary_column_name  s    &

z+TestConvertMetadata.test_binary_column_namec                 C   sP   d}t t|}tjjdddg|gddgd}tjd|i|d}t|d	d
 d S )Nrs   r   barZfoobarZsome_numbersr   numbersr   Tr   )r   r9   r3   r   r   r4   rY   )rw   Znum_rowsr  r   rR   r   r   r$    test_multiindex_duplicate_values  s    z4TestConvertMetadata.test_multiindex_duplicate_valuesc                 C   sb   t dddgi}tj|}|jj}d|ks2t|d d }|d dksNt|d	 d
ks^td S )Nr   s
   some_bytesZsome_unicodemixedr   r   r   bytesr   r   r3   r4   rE   rG   rH   r=   r   rK   )rw   rR   rW   r   data_columnr   r   r$   test_metadata_with_mixed_types  s    z2TestConvertMetadata.test_metadata_with_mixed_typesc                 C   sb   t jdddgdddgddd	d
gd}tj|}|jdd}||j  }t	
|| d S )Nr   r&   rs   r   r  bazr   r   r   Zthreer   T)Zignore_metadata)r3   r4   rE   rG   rH   rI   castr=   remove_metadatarP   rQ   )rw   rR   rW   rX   rS   r   r   r$   test_ignore_metadata  s    z(TestConvertMetadata.test_ignore_metadatac                 C   s   t ddgdddgdgd gi}ttjdtt dg}tjj||d	}|jj	}d
|ksft
|d d }|d dkst
|d dkst
d S )Nr   r   r&   rs   r   r   r   r[   r=   r
  r   r   r   zlist[int64]r   r   )r3   r4   rE   r=   ru   list_r/   rG   rH   r   rK   )rw   rR   r=   rW   r   r  r   r   r$   test_list_metadata  s    " z&TestConvertMetadata.test_list_metadatac                 C   sN   t dddddddgi}tj|}|jj}|d d d	 d
ksJtd S )Ndictsr   r&   r   rs   r   r   r   r   r   r  )rw   rR   rW   r   r   r   r$   test_struct_metadata  s    z(TestConvertMetadata.test_struct_metadatac                 C   s   t dtdtd gi}tj|}|jj}d|ks@t	|d d }|d dks\t	|d	 d
kslt	|d dddkst	d S )Ndecimals394092382910493.12341234678z314292388910493.12343437128r
  r   r   r   decimalr   r   r         )Z	precisionZscale)
r3   r4   r  DecimalrE   rG   rH   r=   r   rK   )rw   rS   rW   r   r  r   r   r$   test_decimal_metadata  s    
z)TestConvertMetadata.test_decimal_metadatac                 C   s   t jdddgddt jdddd	fD ]}t jd
ddgdddgd|d}tj|}|d
}| }|dg }t	|j
t jrd |j
_t|| |d
}| }t||dg jdd q$d S )Nr   r   r   r   r\   r   rs   Europe/Brusselsr   r   r   r&   皙?皙?333333?r   r   TZdrop)r3   r   r   r4   rE   rG   rH   Zremove_columnrI   r   r   r   r   rP   rQ   reset_index)rw   r   rR   rW   Ztable_subsetrX   rS   Ztable_subset2r   r   r$   !test_table_column_subset_metadata  s$    


z5TestConvertMetadata.test_table_column_subset_metadatac                 C   sv   t ttdttdtdd}tj|	ddg}|
ddg}| }|ddg 	d}t|| d S )Nr   )firstsecondvaluer(  r)  r*  )r3   r4   r   r9   r5   r6   rE   rG   rH   r   selectrI   rP   rQ   )rw   rR   rW   ZsubsetrX   rS   r   r   r$   'test_to_pandas_column_subset_multiindex2  s    

z;TestConvertMetadata.test_to_pandas_column_subset_multiindexc              	   C   s   dgddgd g}g g g g}t dtj|tt dfdtj|tt dfg}tjt| t|	 }tj
|g}| }tj
|}|jj}| }	tt d|fd|fg}
t|	|
 |d ddd dd	d
ddd ddd
gkstd S )Ntestr   r   c1r  c2r   r   zlist[unicode])r]   r   r   r   r   zlist[empty])r   rE   r^   r  stringrF   r   r   rl   keysrG   r   rI   rH   r=   r   r3   r4   rP   rQ   rK   )rw   r.  r/  arraysrbtblrR   Ztbl2Zmd2r   rS   r   r   r$   test_empty_list_metadata@  s>    


z,TestConvertMetadata.test_empty_list_metadatac                 C   s@   t dddgdddgd}tj|}|jjd d k	s<td S )Nr   r&   rs   r   pandas_versionr  rv   r   r   r$   test_metadata_pandas_versionl  s    z0TestConvertMetadata.test_metadata_pandas_versionc           
      C   s   t dt jdddi}tj|}|d tjddd}|dt	d|j
|}tdtjdddfg}tjj||d	}| }|d jd|d< ||fD ](}|jjd k	st| }	t|	| qd S )
Nr   z
2020-01-01rs   r   nsUTCr   r   r  )r3   r4   r   rE   rG   rH   r  	timestampZ
set_columnru   r[   r=   rk   dttz_localizer   rK   rI   rP   rQ   )
rw   rR   rW   Znew_colZ
new_table1r=   Z
new_table2rS   Z	new_tablerX   r   r   r$   test_mismatch_metadata_schemaq  s       z1TestConvertMetadata.test_mismatch_metadata_schemaN)(__name__
__module____qualname____doc__rx   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r  r	  r  r  r  r  r  r'  r,  r5  r7  r?  r   r   r   r$   rr      sJ   
		"
	
,rr   c                   @   s   e Zd ZdZdd Zdd Zdd Zdd	 Zd
d Zdd Z	dd Z
dd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zd d! Zd"d# Zd$d% Zd&d' Zd(d) Zd*d+ Zd,d- Zd.S )/TestConvertPrimitiveTypesz>
    Conversion tests for primitive (e.g. numeric) types.
    c           
      C   s   i }g }dt  fdt  fdt  fg}d}|D ]4\}}tj|}||||< |t 	|| q2t
|}t |}	t||	d d S )Nf2f4f8r   rT   )rE   float16r0   r1   r5   r7   r8   r   appendru   r3   r4   r=   rY   )
rw   r   fieldsdtypes
num_valuesZnumpy_dtypearrow_dtyperl   rR   r=   r   r   r$   test_float_no_nulls  s    




z-TestConvertPrimitiveTypes.test_float_no_nullsc                 C   s  d}t jjdd|ddk }dt fdt fdt fg}dddg}g }g }g }|D ]Z\}}	t j||}
tj	|
d	|d
}|
| |
t||	 t j|
|< |
|
 qVtjtt|||d}tj||}|jt|st| }t|| d S )Nr   r   
   r:   rs   rE  rF  rG  T)rH   rf   r   )r5   r7   randintrE   rI  r0   r1   r8   r   r^   rJ  ru   nanr3   r4   dictziprG   r   r=   rJ   rK   rI   rP   rQ   )rw   rM  	null_maskrL  r   expected_colsr2  rK  r]   rN  rl   rd   ex_framerW   rX   r   r   r$   test_float_nulls  s0    





z*TestConvertPrimitiveTypes.test_float_nullsc                 C   sr   t dddtjgi}ttjdt ddg}tjj	||dd}|d 
 d	d
d gks^tt||  d S )Nr   ry   rz   TnullableFr=   r   r   r   r&   )r3   r4   r5   NaNrE   r=   ru   r-   rG   rH   	to_pylistrK   rP   rQ   rI   rw   rR   r=   rW   r   r   r$   test_float_nulls_to_ints  s
    z2TestConvertPrimitiveTypes.test_float_nulls_to_intsc                 C   s>   t dddd dg}t dddd dg}t||t d d S )N        ry   rz   g      FTrS   r[   )r3   r_   rm   rE   bool_rw   ra   rS   r   r   r$   test_float_nulls_to_boolean  s    z5TestConvertPrimitiveTypes.test_float_nulls_to_booleanc                 C   sF   t dtjg}tj|dd}|jdks,tt|d 	 sBtd S )Nr`  FrH   r   r   )
r3   r_   r5   rR  rE   r^   ri   rK   isnanas_py)rw   ra   rd   r   r   r$   'test_series_from_pandas_false_respected  s    zATestConvertPrimitiveTypes.test_series_from_pandas_false_respectedc                 C   s  t  }g }dt fdt fdt fdt fdt fdt fdt fdt	 fd	t fd
t	 fg
}d}|D ]h\}}t
|}t
jjt|jt
t
jjt|jt
t
jj|d}||||< |t|| qzt|}	t|}
t|	|
d d S )Ni1i2i4i8u1u2u4u8longlong	ulonglongr   r:   rH  )r   rE   r,   r-   r.   r/   r(   r)   r*   r+   r5   Ziinfor7   rQ  maxminint_r   rJ  ru   r3   r4   r=   rY   )rw   r   rK  numpy_dtypesrM  r   rN  inforl   rR   r=   r   r   r$   test_integer_no_nulls  s2    
 

 

 

 

 



z/TestConvertPrimitiveTypes.test_integer_no_nullsc                 C   s   t  }ddddddddd	d
ddddddddg}|D ]}tjd|d||< q2t|}t| | D ]"}t|}|	 |
 ksdtqdd S )Nri  rj  rk  rl  rm  rn  ro  rp  byteZubyteshortZushortZintcZuintcru  Zuintrq  rr     r   )r   r5   r6   r3   r4   rY   rl   rE   r^   r]  tolistrK   )rw   r   rv  r   rR   np_arrrd   r   r   r$   test_all_integer_types  s(            

z0TestConvertPrimitiveTypes.test_all_integer_typesc                 C   s   ddddddddg}|D ]}d	D ]|}t jd
ddg|| d}||d d d fD ]L}|jjrzt|}| | kstqNt	t
 t|}W 5 Q R X qNq qd S )Nri  rj  rk  rl  rm  rn  ro  rp  z=<>r   r&   *   r   )r5   r^   r   ZisnativerE   r]  r|  rK   r   r   NotImplementedError)rw   
int_dtypesr=  orderr   r}  rd   r   r   r$   test_integer_byteorder  s    
z0TestConvertPrimitiveTypes.test_integer_byteorderc                 C   s   ddddddddg}d	}t jjd
d|ddk }g }g }|D ]L}t jjd
d	|d}tj||d}|| |d}	t j|	|< ||	 q:tj	t
t|||d}
tj||}| }t||
 d S )Nri  rj  rk  rl  rm  rn  ro  rp  r   r   rP  r:   rs   rf   rG  r   )r5   r7   rQ  rE   r^   rJ  r   rR  r3   r4   rS  rT  rG   r   rI   rP   rQ   )rw   r  rM  rU  rV  r2  r]   rl   rd   rS   rW  rW   rX   r   r   r$   test_integer_with_nulls  s$    


z1TestConvertPrimitiveTypes.test_integer_with_nullsc                 C   sF   t jddd}t }tj||d}t|d}||sBtd S )NrP  r/   r   r  r,   )r5   r6   rE   r,   r^   r   rJ   rK   )rw   rd   Ztarget_typerX   rS   r   r   r$    test_array_from_pandas_type_cast,  s
    z:TestConvertPrimitiveTypes.test_array_from_pandas_type_castc                 C   sV   d}t jd tdt j|dki}tdt }t	|g}t
||d d S )Nr   r   boolsrH  )r5   r7   seedr3   r4   r8   rE   ru   rb  r=   rY   )rw   rM  rR   ru   r=   r   r   r$   test_boolean_no_nulls5  s    z/TestConvertPrimitiveTypes.test_boolean_no_nullsc                 C   s   d}t jd t jjdd|ddk }t jjdd|ddk }tj||d}|t}d ||< tdt	 }t
|g}td|i}tj|gdg}	|	j
|st|	 }
t|
| d S )	Nr   r   rP  r:   rs   r   r  r  )r5   r7   r  rQ  rE   r^   r   r   ru   rb  r=   r3   r4   rG   r   rJ   rK   rI   rP   rQ   )rw   rM  rf   rl   rd   rS   ru   r=   rW  rW   rX   r   r   r$   test_boolean_nulls?  s    
z,TestConvertPrimitiveTypes.test_boolean_nullsc                 C   sF   t dddddgd }t dddddgd }t||t d d S )NTFr&   r   r   ra  )r3   r_   rm   rE   r/   rc  r   r   r$   test_boolean_to_intV  s    z-TestConvertPrimitiveTypes.test_boolean_to_intc              	   C   sj   t jdddddgd td}t dddddgd }d}tjtj|d t||t d	 W 5 Q R X d S )
NTFr&   r   r   r   zExpected integer, got boolmatchra  )	r3   r_   r   r   r   rE   ArrowTypeErrorrm   r/   )rw   ra   rS   Zexpected_msgr   r   r$   test_boolean_objects_to_int\  s
    z5TestConvertPrimitiveTypes.test_boolean_objects_to_intc                 C   sF   t dddd dgd }t dddd dgd }t||t d d S )NTFr&   ry   r`  ra  )r3   r_   rm   rE   r1   rc  r   r   r$   test_boolean_nulls_to_floatd  s    z5TestConvertPrimitiveTypes.test_boolean_nulls_to_floatc                 C   s(   t jtjdddddgd}t| d S )N)rs   r&   r2   r   r   r   r   )r3   r4   r5   onesrY   r   r   r   r$   test_boolean_multiple_columnsj  s    z7TestConvertPrimitiveTypes.test_boolean_multiple_columnsc                 C   sn   t jd dt dgd td}td|i}tdt|i}tdt }t	|g}t
|||d d S )Nr   g      @r   r   floatsrS   rT   )r5   r^   r1   r   r3   r4   
to_numericrE   ru   r=   rY   rw   rd   rR   rS   ru   r=   r   r   r$   test_float_object_nullso  s    z1TestConvertPrimitiveTypes.test_float_object_nullsc              	   C   s   t tjddtjg}t t t t t	 t
 t t g}|D ]}tj||d}tjd ddd g|d}||stt d|i}ttd|g}tjj||dd}|d	 d	|sNtqNd S )
Nry   rz   r  r   r&   Z	has_nullsFr=   r>   r   )r3   r_   r5   rR  rE   r,   r-   r.   r/   r(   r)   r*   r+   r^   rJ   rK   r4   r=   ru   rG   rH   chunk)rw   ra   typestyrX   rS   rR   r=   r   r   r$   test_float_with_null_as_integerx  s"       
z9TestConvertPrimitiveTypes.test_float_with_null_as_integerc                 C   sn   t jd dt dgd td}td|i}tdt|i}tdt }t	|g}t
|||d d S )Nr   rs   r   r   intsr  )r5   r^   r/   r   r3   r4   r  rE   ru   r=   rY   r  r   r   r$   test_int_object_nulls  s    z/TestConvertPrimitiveTypes.test_int_object_nullsc                 C   sR   t jdd dgd td}td|i}tdt }t|g}t	||d d S )NFTr   r   r  rH  )
r5   r^   r   r3   r4   rE   ru   rb  r=   rY   )rw   rd   rR   ru   r=   r   r   r$   test_boolean_object_nulls  s
    z3TestConvertPrimitiveTypes.test_boolean_object_nullsc                    s8   t jd gtd  fdd}|t  |t  d S )Nr   c                    s4   t j | d}|j| kst|d  d ks0td S )Nr  r   )rE   r^   r[   rK   rg  )r   a2rd   r   r$   _check_type  s    zJTestConvertPrimitiveTypes.test_all_nulls_cast_numeric.<locals>._check_type)r5   r^   r   rE   r.   r1   )rw   r  r   r  r$   test_all_nulls_cast_numeric  s    z5TestConvertPrimitiveTypes.test_all_nulls_cast_numericc                 C   s   t jdt jgt jd}tj|t d}| \}}t|t jsDt|dksPtt|t js`tt |snttj|t dd}| \}}t|t jst|dkst|d kstd S )Nr   r   r  Tr[   rH   )	r5   r^   rR  rI  rE   r]  r   rK   rf  )rw   rd   r   r#   yr   r   r$   test_half_floats_from_numpy  s    z5TestConvertPrimitiveTypes.test_half_floats_from_numpyN)r@  rA  rB  rC  rO  rX  r_  rd  rh  rx  r~  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r   r   r   r$   rD    s.   	
		rD  r   ri  rj  rk  rl  rm  rn  ro  rp  c                 C   sz   d}t jjdd|ddk }t jjdd|| d}tj||d}| rX|d}d ||< n|}|jd	d
}t j	|| d S Nr   r   rP  r:   rs   r;   r   r  OTinteger_object_nulls)
r5   r7   rQ  rE   r^   anyr   rI   testingZassert_equal)r   rM  rU  rl   r^   rS   rX   r   r   r$   &test_array_integer_object_nulls_option  s    

r  c                 C   s   d}t jjdd|ddk }t jjdd|| d}tj||d}| rX|d}d ||< n|}t| |i}tj	
|g| g}|jd	d
}t|| d S r  )r5   r7   rQ  rE   r^   r  r   r3   r4   rG   r   rI   rP   rQ   )r   rM  rU  rl   r^   rS   rW   rX   r   r   r$   &test_table_integer_object_nulls_option  s    

r  c                   @   s  e Zd ZdZdd Zdd Zejdddd	d
gdd Z	dd Z
dd Zee ejB ejdddd Zdd Zdd Zdd Zdd Zejddeddddddggd d! Zejd"d#d$gd%d& Zejd'd#d$gd(d) Zejd*e e e de de d	e d
e dd+e dd+e d	d+e d
d+g
d,d- Z!ejd*e e e de de d	e d
e dd+e dd+e d	d+e d
d+g
d.d/ Z"d0d1 Z#d2d3 Z$d4d5 Z%d6d7 Z&d8d9 Z'd:d; Z(d<d= Z)d>d? Z*d@dA Z+dBdC Z,dDdE Z-ejdFe e gdGdH Z.dIdJ Z/ejj0e1dKe1ej2  kope1dLk n  dMdNdOdP Z3ejdddd	d
gdQdR Z4ejdddd	d
gdSdT Z5dUdV Z6dS )WTestConvertDateTimeLikeTypeszQ
    Conversion tests for datetime- and timestamp-like types (date64, etc.).
    c                 C   sL   t dtjdddgddi}tdtd}t|g}t||d d S )	N
datetime642007-07-13T01:23:34.1234567892006-01-13T12:34:56.4325397842010-08-13T05:46:57.437699912r   r   r9  rH  	r3   r4   r5   r^   rE   ru   r<  r=   rY   rw   rR   ru   r=   r   r   r$   #test_timestamps_notimezone_no_nulls  s      z@TestConvertDateTimeLikeTypes.test_timestamps_notimezone_no_nullsc                 C   sL   t dtjdd dgddi}tdtd}t|g}t||d d S )Nr  r  r  r   r   r9  rH  r  r  r   r   r$    test_timestamps_notimezone_nulls  s      z=TestConvertDateTimeLikeTypes.test_timestamps_notimezone_nullsunitra   msusr9  c                 C   s   t tjt dk r$|dkr$td tdtjdddgd| d	d
i}|d j	d|d< t
| t|d  tdtjdd ddgd| d	d
i}|d j	d|d< t
| d S )Nr   r9  z0pandas < 2.0 only supports nanosecond datetime64r  2007-07-13T01:23:34.1232006-01-13T12:34:56.4322010-08-13T05:46:57.437zdatetime64[]r   
US/Easternr  r  r  )r   r3   r  r   r  r4   r5   r^   r=  r>  rY   re   )rw   r  rR   r   r   r$   test_timestamps_with_timezone  s8    
 
 
z:TestConvertDateTimeLikeTypes.test_timestamps_with_timezonec                 C   s   dd t dD }tdtj|tdi}tj|}t|d 	dtj
sPt| }tdtj|ddi}t|| d S )Nc                 S   s   g | ]}t  t|d  qS ))days)r   todayr   r!   r   r   r$   r%   #  s     zETestConvertDateTimeLikeTypes.test_python_datetime.<locals>.<listcomp>rP  r   r   r   r   )r9   r3   r4   r_   r   rE   rG   rH   r   r  TimestampArrayrK   rI   rP   rQ   )rw   
date_arrayrR   rW   rX   expected_dfr   r   r$   test_python_datetime!  s      z1TestConvertDateTimeLikeTypes.test_python_datetimec              
   C   sZ   t d}|j|d|dfD ]2}tdddddd|dg}td	|i}t| q"d S )
Npytzr  r   r   r{     -   tzinfor   )	r   importorskiputcr   FixedOffsetr   r3   r4   rY   )rw   r  r   rl   rR   r   r   r$   %test_python_datetime_with_pytz_tzinfo4  s
    
zBTestConvertDateTimeLikeTypes.test_python_datetime_with_pytz_tzinfoN)deadlinec              	   C   sL   t |dkrtd tdddddd|dg}td	|i}t|d
d d S )N)zbuild/etc/localtimeZFactoryz Localtime timezone not supportedr   r   r{  r  r  r  r   FrD   )r    r   r  r   r3   r4   rY   )rw   r   rl   rR   r   r   r$   'test_python_datetime_with_pytz_timezone<  s
    
zDTestConvertDateTimeLikeTypes.test_python_datetime_with_pytz_timezonec           
   	   C   s   t d}ddlm} tdddddd|jd	g}tjd
|i|d}t|dd d}|t|d}|	|d }tdddddd|d	g}tdddddd|d	g}tjd
|i|d}tjd
|i|d}	t||	dd d S )Nr  r   )r   r   r   r{  r  r  r  r   r   Tr   )hours<   r   )
r   r  r   r   r  r3   r4   rY   r   r  )
rw   r  r   rl   rR   r  Ztz_timezoneZtz_pytzZ
values_expZdf_expr   r   r$   )test_python_datetime_with_timezone_tzinfoE  s    
zFTestConvertDateTimeLikeTypes.test_python_datetime_with_timezone_tzinfoc                 C   s   G dd dt }|ddddddg}tdtj|tdi}tj|}t|d 	dtj
sbt| }tdtj|ddi}t|d |d< t|| d S )	Nc                   @   s   e Zd ZdZdS )zNTestConvertDateTimeLikeTypes.test_python_datetime_subclass.<locals>.MyDatetimer`  N)r@  rA  rB  Z
nanosecondr   r   r   r$   
MyDatetimeZ  s   r    r   r   r   r   r   )r   r3   r4   r_   r   rE   rG   rH   r   r  r  rK   rI   to_datetimerP   rQ   )rw   r  r  rR   rW   rX   r  r   r   r$   test_python_datetime_subclassX  s    z:TestConvertDateTimeLikeTypes.test_python_datetime_subclassc                 C   s   G dd dt }|dddg}tdtj|tdi}tj|}t|d 	dtj
s\t| }tdtjt dddgtdi}t|| d S )Nc                   @   s   e Zd ZdS )zFTestConvertDateTimeLikeTypes.test_python_date_subclass.<locals>.MyDateNr@  rA  rB  r   r   r   r$   MyDates  s   r  r  r   r   r   r   )r   r3   r4   r_   r   rE   rG   rH   r   r  ZDate32ArrayrK   rI   r5   r^   rP   rQ   )rw   r  r  rR   rW   rX   r  r   r   r$   test_python_date_subclassq  s    z6TestConvertDateTimeLikeTypes.test_python_date_subclassc                 C   sV   t tdddd g}t |g}| }t jj|t  d}||	dsRt
d S )N  rP  r  r  date32)rE   r^   r   chunked_arrayrI   ArrayrH   r  rJ   r  rK   )rw   rd   r   ra   arr2r   r   r$   test_datetime64_to_date32  s
    z6TestConvertDateTimeLikeTypes.test_datetime64_to_date32rf   TFc                 C   s   t ddddddg}tjj|t |d}ttdd	d
tdd	dtdd	dtdd	d
tdd	dtdd	dg}tj||t d}|	|st
d S )Nz2018-05-10T00:00:00z2018-05-11T00:00:00z2018-05-12T00:00:00z2018-05-10T10:24:01z2018-05-11T10:24:01z2018-05-12T10:24:01r[   rf   r   r   rP  r  r{  rf   r[   )r3   r  rE   r  rH   date64r5   r^   r   rJ   rK   )rw   rf   ra   rd   r   rS   r   r   r$   test_pandas_datetime_to_date64  s&    





z;TestConvertDateTimeLikeTypes.test_pandas_datetime_to_date64zcoerce_to_ns,expected_dtype)Fr   )Tr   c           
      C   s   t dddd t dddt dddg}tjdd dd	gd
d}ttjtdk rPd}tjdd dd	g|d}t|t|gg}|D ]`}|j|d}|	t
}	|j|	jkstt||	 |jd|d}|j|jkstt|| qd S )Nr  r       r&   r  
2000-01-01z
1970-01-01z
2040-02-26datetime64[D]r   r   r   coerce_temporal_nanosecondsFdate_as_objectr  )r   r5   r^   r   r3   r  rE   r  rI   r   r   r   rK   ro   rp   )
rw   coerce_to_nsZexpected_dtyper   Zexpected_daysrS   ZobjectsobjrX   Zexpected_objr   r   r$   test_array_types_date_as_object  s<    






z<TestConvertDateTimeLikeTypes.test_array_types_date_as_objectzcoerce_to_ns,expected_typec              
   C   s~   t dtdddd tdddtdddgi}tjj|dd	}|jd|d
}| }tj|	||dd tj||dd d S )Nr   r  r   r  r  r&   r  Fr   r  Tr  )
r3   r4   r   rE   rG   rH   rI   rP   rQ   r   )rw   r  expected_typerR   rW   Zdf_datetimeZ	df_objectr   r   r$   !test_table_convert_date_as_object  s"     


z>TestConvertDateTimeLikeTypes.test_table_convert_date_as_object
arrow_typer:  c                 C   s   t dddtdddg}t|}t||}|jddd}d }t|dr\|j	d k	r\d}t
d	| }t||| d S )
Nr  r     TFr  r  r   r:  r9  )r   r   r3   r_   rE   r^   r  rI   hasattrr   r<  to_pandas_dtyperP   r`   r   )rw   r  r   rS   rd   rX   expected_tzr  r   r   r$   &test_array_coerce_temporal_nanoseconds  s    
 zCTestConvertDateTimeLikeTypes.test_array_coerce_temporal_nanosecondsc           	      C   s   t dddtdddg}ttd|g}td|i}tjt|g|d}|j	ddd}d }t
|d	rz|jd k	rzd
}td| }t||| d S )Nr  r   r  r   r  TFr  r   r:  r9  )r   r   rE   r=   ru   r3   r4   rW   r^   rI   r  r   r<  r  rP   rQ   r   )	rw   r  r   r=   r  rW   Z	result_dfr  r  r   r   r$   &test_table_coerce_temporal_nanoseconds  s     zCTestConvertDateTimeLikeTypes.test_table_coerce_temporal_nanosecondsc              
   C   s   t dtdddd tdddtdddgi}tjj|dd	}tdt }tj|g|jj	d
}|j
|srt| }t|| d S )Nr   r  r   r  r  r&   r  Fr   r   )r3   r4   r   rE   rG   rH   ru   r  r=   r   rJ   rK   rI   rP   rQ   )rw   rR   rW   ru   rT   rX   r   r   r$   test_date_infer   s     


z,TestConvertDateTimeLikeTypes.test_date_inferc                 C   sx   t jtdddtdddgdd}ddg}tj|t |d}t jd tdddgdd}tj|dd	}||sttd S )
Nr  r   rs   r  r   TFr  re  )r5   r^   r   rE   rJ   rK   )rw   rd   rf   rX   rS   r   r   r$   test_date_mask  s    z+TestConvertDateTimeLikeTypes.test_date_maskc                 C   st  t jtdddd tdddtdddgtd}t jdddd	gd
d}|dd }t ddddg}t }t }tj||d}tj||d}tj|||d}	tj|||d}
||	st	||
st	ddg}tj
||g|}t jddddgdd}tjj|d< |d}tj||d|d}|jdd}t|| | }|d}tj||d|d}t|| d S )Nr  r   rs   r   r   kC  r   lC  mC  r.   r/    \&FTr  r  r  r  z
2017-04-03z
2017-04-04z
2017-04-05r  r   r   )r  r  r   )r  r   )r5   r^   r   r   r   rE   r  r  rJ   rK   rG   r   r3   NaTr*  r4   rI   rP   rQ   )rw   rd   Zarr_i4Zarr_i8rf   Zt32Zt64Za32Za64Za32_expectedZa64_expectedZcolnamesrW   	ex_valuesZex_datetime64msZexpected_pandasZtable_pandasZtable_pandas_objectsZ
ex_objectsZexpected_pandas_objectsr   r   r$   test_date_objects_typed  sZ    





z4TestConvertDateTimeLikeTypes.test_date_objects_typedc                 C   s   t tdd }tjtdddtj|gtd}tjtdddd d gtd}tj|dd}tj|dd}||spt	|j
dks~t	tjtjgddjt kst	tj|gddjt kst	d S )NNAr  r   r   Tre  r&   )getattrr3   r5   r^   r   r  r   rE   rJ   rK   ri   r[   null)rw   Zpd_NArl   Zvalues_with_nonerX   rS   r   r   r$   test_pandas_null_valuesJ  s     z4TestConvertDateTimeLikeTypes.test_pandas_null_valuesc                 C   s   t  }t  }tjdddgdd}|dd }t j||d}t j||d}td	d
d}|d  |ksnt|d  |kstd S )Nr   r  r  r.   r   r/   r  r  r  r   rs   r   )	rE   r  r  r5   r^   r   r   rg  rK   )rw   t1t2rd   r  a1r  rS   r   r   r$   test_dates_from_integersY  s    z5TestConvertDateTimeLikeTypes.test_dates_from_integersc                 C   s  t ddddt ddddg}td}tj|d g td	}t|}|j|ksRt|d
  |d
 ksjt|d  |d kst|d  d kstt	
d|i}tj|}|d
 |sttjdd |D dd	}tj|tdd}|d
  |d
 ksttj|d tdd}	|	d
  |d
 ks:ttj|d dtdd}
|
d
  |d
 jddkszttj|d dtdd}|d
  |d
 jd
dkstd S )Nr   r&   rs   L  r   r   r   r  r   r   timesc                 S   s   g | ]}t |qS r   _pytime_to_microsr"   vr   r   r$   r%   {  s     zHTestConvertDateTimeLikeTypes.test_pytime_from_pandas.<locals>.<listcomp>r/   r    r9  rk  r  microsecond@B ra   )r   rE   time64r5   r^   r   r[   rK   rg  r3   r4   rF   rH   rJ   r   time32replace)rw   pytimesr  ZaobjsparrrR   r   rd   r  r  a3a4r   r   r$   test_pytime_from_pandasg  s8    

"z4TestConvertDateTimeLikeTypes.test_pytime_from_pandasc                 C   s  t ddddt ddddt dddg}t|d d d g }td	d
 |d d D d g }tdd
 |d d D d g }tjdd
 |D dd}tjdd
 |D dd}tjdddgtd}tj||tdd}tj|d |tdd}tj|d d|tdd}	tj|d d|tdd}
ddddg}tj	|||	|
g|}||f||f|	|f|
|ffD ]&\}}|
 }|j|k sttqt|
 }tj||||d|d}t|| d S ) Nr   r&   rs   r  r   r   r   r   c                 S   s   g | ]}|j d dqS )r  r  r  r!   r   r   r$   r%     s   zJTestConvertDateTimeLikeTypes.test_arrow_time_to_pandas.<locals>.<listcomp>c                 S   s   g | ]}|j d dqS )r   r  r!  r!   r   r   r$   r%     s   c                 S   s   g | ]}t |qS r   r  r  r   r   r$   r%     s     r/   r   c                 S   s   g | ]}t |qS r   r  r  r   r   r$   r%     s     FTr  r  r  r9  rk  r  r  ra   
time64[us]
time64[ns]
time32[ms]	time32[s])r"  r#  r$  r%  r   )r   r5   r^   r2   rE   r  r   r  rF   r   rI   rl   allrK   r3   r4   rP   rQ   )rw   r  rS   Zexpected_msZ
expected_srd   rU  r  r  r  r  r   r   Zexpected_valuesZresult_pandasrR   r  r   r   r$   test_arrow_time_to_pandas  sd    




z6TestConvertDateTimeLikeTypes.test_arrow_time_to_pandasc                 C   s|   t jdd ddgdd}t| t jdd ddgd	d}t| t jd
d ddgdd}t| t jdd ddgdd}t| d S )Nr  r  r  r   r   z2007-07-13T01:23:34.123456z2006-01-13T12:34:56.432539z2010-08-13T05:46:57.437699r   r  r  r  r   z2007-07-13T01:23:34z2006-01-13T12:34:56z2010-08-13T05:46:57r   r5   r^   rq   )rw   Zdatetime64_nsZdatetime64_usZdatetime64_msZdatetime64_sr   r   r$   test_numpy_datetime64_columns  sH    z:TestConvertDateTimeLikeTypes.test_numpy_datetime64_columnsc                 C   s   t tjt dkrtd tdddgtd}ttj	dddgdd}|
 }t|| t|g}|
 }t|| d S )Nr   z0pandas >= 2.0 supports non-nanosecond datetime64r   r&   rs   r  )r  )r   r3   r  r   r  rE   r^   r<  r_   r  rI   rP   r`   r  )rw   rd   rS   ra   r   r   r$   &test_timestamp_to_pandas_coerces_to_ns  s    
zCTestConvertDateTimeLikeTypes.test_timestamp_to_pandas_coerces_to_nsc                 C   s   dD ]}dD ]}t tdddgt j||d}t d|i}d}tjt|d |jdd	 W 5 Q R X tjt|d |jdd	 W 5 Q R X tjt|d |	djdd	 W 5 Q R X |jd
dd |jd
dd |	djd
dd qqd S )Nra   r  r  )Nr   r   r;  r   z'would result in out of bounds timestampr  Tr  F)r   r  )
rE   r^   r   r<  rW   r   r   r   rI   column)rw   r  r   rd   rW   msgr   r   r$   &test_timestamp_to_pandas_out_of_bounds  s"     
 zCTestConvertDateTimeLikeTypes.test_timestamp_to_pandas_out_of_boundsc                 C   sN   t dt jg t ddi}| }tdtjg ddi}t	|| d S )Nr   r  r  r   r   )
rE   rW   r  r<  rI   r3   r4   r_   rP   rQ   )rw   rW   rX   rS   r   r   r$   &test_timestamp_to_pandas_empty_chunked  s    zCTestConvertDateTimeLikeTypes.test_timestamp_to_pandas_empty_chunkedr   c                 C   s&   t jdd ddgdd}t||d d S )N
2007-07-13z
2006-01-15z
2010-08-19r  r   r  r(  )rw   r   Zdatetime64_dr   r   r$   test_numpy_datetime64_day_unit  s    z;TestConvertDateTimeLikeTypes.test_numpy_datetime64_day_unitc                 C   sz   t dddg}ttdddtdddtdddg}tjj||d}td tdddd g}tj||svt	d S )NTFi  r   i  i  r  )
r5   r^   r3   r_   r   rE   r  rH   rJ   rK   )rw   mr   rX   rS   r   r   r$   %test_array_from_pandas_date_with_mask  s    


zBTestConvertDateTimeLikeTypes.test_array_from_pandas_date_with_maskz1.16.0z1.16.1z#Until numpy/numpy#12745 is resolved)reasonc                 C   s*   t dt dt jgi}t|dd d S )Nr   z2012-11-11 00:00:00+01:00Fr  )r3   r4   	Timestampr  rY   r   r   r   r$   test_fixed_offset_timezone#  s    
z7TestConvertDateTimeLikeTypes.test_fixed_offset_timezonec                 C   sj   t tjt dk rd}tdtjdddgd| dd	i}tdt|}t	|g}t
||d
 d S )Nr   r9  timedelta64r   l     qa    @B1timedelta64[r  r   rH  r   r3   r  r4   r5   r^   rE   ru   durationr=   rY   rw   r  rR   ru   r=   r   r   r$   test_timedeltas_no_nulls3  s     
z5TestConvertDateTimeLikeTypes.test_timedeltas_no_nullsc                 C   sj   t tjt dk rd}tdtjdd dgd| ddi}tdt|}t	|g}t
||d	 d S )
Nr   r9  r7  r   r8  r9  r  r   rH  r:  r<  r   r   r$   test_timedeltas_nullsB  s     
z2TestConvertDateTimeLikeTypes.test_timedeltas_nullsc              
   C   sN   ddl m} tdd |dddddgi}tdt fg}t||d d S )	Nr   )
DateOffsetZdate_offseti  rs   iX  )r  monthsmicrosecondsZnanosecondsrH  )Zpandas.tseries.offsetsr?  r3   r4   rE   r=   Zmonth_day_nano_intervalrY   )rw   r?  rR   r=   r   r   r$   test_month_day_nano_intervalQ  s     z9TestConvertDateTimeLikeTypes.test_month_day_nano_interval)7r@  rA  rB  rC  r  r  r   markparametrizer  r  r  hgivenstnonepastZ	timezonessettingsr  r  r  r  r  r5   r^   r  r  r  rE   r  r  r<  r  r  r  r  r  r
  r  r   r'  r)  r*  r.  r/  r1  r3  Zskipifr   r  r6  r=  r>  rB  r   r   r   r$   r    s   





  

 


  

 


	1%0!
	&


r  c                   @   s   e Zd Zdd Zdd Zejjdd Zejjej	ddd	gd
d Z
dd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zd d! Zd"d# Zd$d% Zd&d' Zd(d) Zd*S )+TestConvertStringLikeTypesc                 C   st   d}dd ddt jg}td|| i}tdt }t|g}dd ddd g}td|| i}t|||d d S )Nr  r   r  u   mañanastringsr  )	r5   rR  r3   r4   rE   ru   r0  r=   rY   )rw   repeatsrl   rR   ru   r=   r  rS   r   r   r$   test_pandas_unicoded  s    z.TestConvertStringLikeTypes.test_pandas_unicodec                 C   sr   ddd t ddtjg}td|i}tj|}|d jt	 ksFt
ddd ddd g}td|i}t|| d S )Nqux   foos   barzrL  r      qux)	bytearrayr5   rR  r3   r4   rE   rG   rH   r[   binaryrK   rY   )rw   rl   rR   rW   Zvalues2rS   r   r   r$   test_bytes_to_binaryo  s    z/TestConvertStringLikeTypes.test_bytes_to_binaryc                 C   s   dd }dd }t d|gd |g dgd  i}t|d }t|tjsRt|jdks`td }tj	|}|d jdkstd S )	N   x krL     r#   r&   r   )
r3   r4   rE   r^   r   ChunkedArrayrK   
num_chunksrG   rH   )rw   v1v2rR   rd   rW   r   r   r$   test_bytes_exceed_2gbz  s     z0TestConvertStringLikeTypes.test_bytes_exceed_2gbcharr#   rU  c                 C   s   |d }|d }t d|ggd |gg dgg i}tj|d dd}t|tjsXt|jdksftt|	d	d
ks|tt|	ddkstd S )NrV  rW  rL  rX  rU  Tre  r&   r      r   )
r3   r4   rE   r^   r   rY  rK   rZ  r   r  )rw   r^  r[  r\  rR   rd   r   r   r$   +test_auto_chunking_pandas_series_of_strings  s     zFTestConvertStringLikeTypes.test_auto_chunking_pandas_series_of_stringsc                 C   s   dd t dd d dg}td|i}ttdtdg}tjj||d}|jd j	|d j	ksft
|jd j|d jkst
| }t|| d S )NrP     bar   heyrL  rs   r  r   )rR  r3   r4   rE   r=   ru   rS  rG   rH   r[   rK   r]   rI   rP   rQ   )rw   rl   rR   r=   rW   rX   r   r   r$   test_fixed_size_bytes  s    z0TestConvertStringLikeTypes.test_fixed_size_bytesc              	   C   sd   dd dd d dg}t d|i}ttdtdg}ttj tj	j
||d W 5 Q R X d S )NrP  s   barb  rL  rs   r  )r3   r4   rE   r=   ru   rS  r   r   ArrowInvalidrG   rH   )rw   rl   rR   r=   r   r   r$   5test_fixed_size_bytes_does_not_accept_varying_lengths  s
    zPTestConvertStringLikeTypes.test_fixed_size_bytes_does_not_accept_varying_lengthsc                 C   s&   t dddd g}t|t d d S )N   123    r   rb   )r3   r_   re   rE   rS  rw   ra   r   r   r$   test_variable_size_bytes  s    z3TestConvertStringLikeTypes.test_variable_size_bytesc                 C   sB   t tdtdtdd g}t|t d t|t d d S )Nrf  rg  r   rh  rc   )r3   r_   rR  re   rE   rS  ri  r   r   r$   test_binary_from_bytearray  s
    z5TestConvertStringLikeTypes.test_binary_from_bytearrayc                 C   sP   t dddd g}t|t d t d|i}t|tdt fgd d S )Nrf  rg  r   rh  r   r  )r3   r_   re   rE   large_binaryr4   rY   r=   rw   ra   rR   r   r   r$   test_large_binary  s     z,TestConvertStringLikeTypes.test_large_binaryc                 C   sP   t dddd g}t|t d t d|i}t|tdt fgd d S )NZ123r'   r   rh  r  )r3   r_   re   rE   large_stringr4   rY   r=   rn  r   r   r$   test_large_string  s     z,TestConvertStringLikeTypes.test_large_stringc           
      C   s   dddddg}t d|i}tdt }t|g}tjj||d}|jdd}t d|i}t	j
||dd |jdd}t dt |i}	t	j
||	dd d S )Nr'   rL  r  Fstrings_to_categoricalTr  )r3   r4   rE   ru   r0  r=   rG   rH   rI   rP   rQ   Categorical)
rw   rl   rR   ru   r=   rW   result1	expected1r   	expected2r   r   r$   test_table_empty_str  s    z/TestConvertStringLikeTypes.test_table_empty_strc                 C   s   dddddg}t d|i}tdt }t|g}tjj||d}t d|i}t dt |i}|j	dgd}t
j||dd |j	g d}	t
j|	|dd |j	dd}
t
j|
|dd |j	t d}t
j||dd d S )Nr'   rL  r  
categoriesTr  )rL  )r3   r4   rE   ru   r0  r=   rG   rH   rt  rI   rP   rQ   tuple)rw   rl   rR   ru   r=   rW   Zexpected_strZexpected_catru  r   result3Zresult4r   r   r$   test_selective_categoricals  s    z6TestConvertStringLikeTypes.test_selective_categoricalsc                 C   s8   t jg t  d}t jj|gdgd}|jdgd d S )Nr  colr2  r   ry  )rE   r^   r.   rG   r   rI   )rw   r^   rW   r   r   r$   &test_to_pandas_categorical_zero_length  s    zATestConvertStringLikeTypes.test_to_pandas_categorical_zero_lengthc                 C   sN   t ddddg }t jj|gdgd}|jdgd}| |sJtd S )Nr   r  r~  r  ry  )rE   r^   dictionary_encoderG   r   rI   rJ   rK   )rw   r^   rW   rX   r   r   r$   ,test_to_pandas_categories_already_dictionary  s    zGTestConvertStringLikeTypes.test_to_pandas_categories_already_dictionaryc              	   C   s   dddddg}t d|i}tdt }t|g}tjj||d}|jdd}t dt 	|i}t
j||dd ttj |jddd	 W 5 Q R X d S )
Nr   r   r   rL  r  Trr  r  rs  zero_copy_only)r3   r4   rE   ru   r0  r=   rG   rH   rI   rt  rP   rQ   r   r   rd  rw   rl   rR   ru   r=   rW   rX   rS   r   r   r$   (test_table_str_to_categorical_without_na  s    zCTestConvertStringLikeTypes.test_table_str_to_categorical_without_nac              	   C   s   d ddt jg}td|i}tdt }t|g}tjj	||d}|j
dd}tdt|i}tj||dd ttj |j
ddd W 5 Q R X d S )	Nr   r   rL  r  Trr  r  r  )r5   rR  r3   r4   rE   ru   r0  r=   rG   rH   rI   rt  rP   rQ   r   r   rd  r  r   r   r$   %test_table_str_to_categorical_with_na  s    z@TestConvertStringLikeTypes.test_table_str_to_categorical_with_nac                 C   s2   t tjdgtdt  }|jt  ks.td S )NrU  r   )rE   r^   r5   r   r0  r[   rK   )rw   	convertedr   r   r$   test_array_of_bytes_to_strings  s    z9TestConvertStringLikeTypes.test_array_of_bytes_to_stringsc              	   C   s>   t jtjjdd" ttjdgtdt  W 5 Q R X d S )Nzwas not a utf8 stringr  s   r   )	r   r   rE   librd  r^   r5   r   r0  rw   r   r   r$   'test_array_of_bytes_to_strings_bad_data  s
    zBTestConvertStringLikeTypes.test_array_of_bytes_to_strings_bad_datac              	   C   s,  t jdddgdd}tj|tdd}tjt|tdd}||sNtt dd	dg}tj|td|d
}tjdd dgtdd}||sttjtj	j
dd. t jdddgdd}tj|tdd W 5 Q R X tjtj	j
dd. t jdddgdd}tj|tdd W 5 Q R X d S )NrP  ra  s   bazz|S3r   rs   r  FTr  z)Got bytestring of length 3 \(expected 4\)r  r   z*Got bytestring of length 12 \(expected 3\)z|U3)r5   r^   rE   rS  r   rJ   rK   r   r   r  rd  )rw   rd   r  rS   rf   r   r   r$   ,test_numpy_string_array_to_fixed_size_binary!  s&    
zGTestConvertStringLikeTypes.test_numpy_string_array_to_fixed_size_binaryN)r@  rA  rB  rN  rT  r   rC  large_memoryr]  rD  r`  rc  re  rj  rl  ro  rq  rx  r}  r  r  r  r  r  r  r  r   r   r   r$   rK  b  s,   

rK  c                   @   s  e Zd ZdZededgZededgZededgZe	j
de	jeed	d
dde	jeedddde	jeeddddgdd Ze	j
de	jedde	jedde	jeddgdd Zdd Zdd Zdd Zdd Zd d! Zd"S )#TestConvertDecimalTypesz,
    Conversion test for decimal types.
    z	-1234.123z1234.439z-129934.123331z129534.123731r  z-314292388910493.12343437128)rl   r  r   rs   	decimal32idr{  r   	decimal64r  r  
decimal128c                 C   sR   t d|i}tjj|dd}td|}tj|g|jjd}|j|sNt	d S )Nr  Fr   r  )
r3   r4   rE   rG   rH   ru   r=   r   rJ   rK   )rw   rl   r  rS   rW   ru   rT   r   r   r$   test_decimal_from_pandasJ  s
    z0TestConvertDecimalTypes.test_decimal_from_pandasrl   c                 C   s2   t d|i}tj|}| }t|| d S )Nr  )r3   r4   rE   rG   rH   rI   rP   rQ   )rw   rl   rS   r  rR   r   r   r$   test_decimal_to_pandasX  s    z.TestConvertDecimalTypes.test_decimal_to_pandasc              	   C   s   t dg}tdd}ttj tj||d W 5 Q R X t dg}tdd}ttj tj||d W 5 Q R X d S )Nz1.234rP  r&   r  z1.2345rs   )r  r  rE   r  r   r   rd  r^   )rw   data1Ztype1data2Ztype2r   r   r$   "test_decimal_fails_with_truncationc  s    z:TestConvertDecimalTypes.test_decimal_fails_with_truncationc                 C   s   t dt dg}t|}t|}| |ks8t|jt	ddksNttj|t	ddd}t dt dg}| |kstd S )	Nz0.01z0.001rs   r{  r   r  z0.01000z0.00100)
r  r  r3   r_   rE   r^   r]  rK   r[   r  )rw   r   seriesr^   rS   r   r   r$   &test_decimal_with_different_precisionsn  s    

z>TestConvertDecimalTypes.test_decimal_with_different_precisionsc                 C   sP   t tdd g}t|tddd t d gd }t|tddd d S )N3.14r{  r   rh  r&   r3   r_   r  r  re   rE   r  rw   r  r   r   r$   $test_decimal_with_None_explicit_type|  s    z<TestConvertDecimalTypes.test_decimal_with_None_explicit_typec                 C   s,   t tdd g}t|tddd d S )Nr  rs   r&   rk  r  r  r   r   r$   !test_decimal_with_None_infer_type  s    z9TestConvertDecimalTypes.test_decimal_with_None_infer_typec                 C   s0   ddidt did}tj|}t| d S )Nr   r   z0.0r   )r  r  r3   r4   	from_dictrY   )rw   Ztmpdirr   rR   r   r   r$   test_strided_objects  s
    z,TestConvertDecimalTypes.test_strided_objectsN)r@  rA  rB  rC  r  r  r  r  r  r   rC  rD  paramrE   r  r  r  r  r  r  r  r   r   r   r$   r  9  s6   
	
r  c                
   @   s  e Zd ZdZdd Zdd Zdd Zdd	 Zd
d Zdd Z	dd Z
dd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zd d! Zd"d# Zd$d% Zd&d' Zd(d) Zd*d+ Zejd,ejd-d.gd/gd0gd0d/gd0gfejd1d2gd3gd0gd0d3gd0gfejd0d0gd0gd0gd0d0gd0gfgd4d5 Z d6d7 Z!d8d9 Z"d:d; Z#d<d= Z$d>d? Z%ejj&d@dA Z'dBdC Z(dDdE Z)dFdG Z*dHdI Z+dJdK Z,dLdM Z-d0S )NTestConvertListTypesz,
    Conversion tests for list<> types.
    c                 C   st   t  \}}t|||d tjj||dd}||jj}|j|sHt	|j
D ] }||}t|| |jd qNd S Nr=   rT   Fr  r  )r   rY   rE   rG   rH   with_metadatar=   r   rJ   rK   r   ru   rm   r[   rw   rR   r=   rW   rT   r,  ru   r   r   r$   test_column_of_arrays  s    


z*TestConvertListTypes.test_column_of_arraysc                 C   s   d}t jt jd|dt jd|dd t jd|dgtd}tt }tj||d}|d  tt	dkspt
|d  tt	dkst
|d  d kst
|d	  dgkst
d S )
Nri  rP  r   r   r   r  r   r&   rs   )r5   r^   r6   r   rE   r  r,   rg  r   r9   rK   )rw   r   rd   rb   r  r   r   r$   test_column_of_arrays_to_py  s    z0TestConvertListTypes.test_column_of_arrays_to_pyc                 C   s   t jddgdggt t  d}t jj|gdgd}| }tdddgdggi}t	
|| |d  }t	jt||d dd d S )NTFr  r   r   r   rg   )rE   r^   r  rb  rG   r   rI   r3   r4   rP   rQ   r`   r_   )rw   r^   rW   rR   r  ra   r   r   r$   test_column_of_boolean_list  s    "z0TestConvertListTypes.test_column_of_boolean_listc                 C   s   t jtdtdgtdggt t ddd}t jj|gdgd}| }t	
dtdtdgtdggi}t|| d S )	Nr   r   z3.3r&   r   r  r   r   )rE   r^   r  r  r  r  rG   r   rI   r3   r4   rP   rQ   )rw   r^   rW   rR   r  r   r   r$   test_column_of_decimal_list  s    

z0TestConvertListTypes.test_column_of_decimal_listc              
   C   s   t tjtjtjgtd}tt t	t t
tddgfD ]X}tj||d}tjd d g|d}||s|ttt tj|j|d W 5 Q R X qJd S )Nr   f0r.   r  )r3   r_   r5   r^   rR  r   rE   r  r/   
large_liststructru   rJ   rK   r   r   	TypeErrorrl   )rw   ra   r  rX   rS   r   r   r$   +test_nested_types_from_ndarray_null_entries  s    z@TestConvertListTypes.test_nested_types_from_ndarray_null_entriesc                 C   st   t  \}}t|||d tjj||dd}||jj}|j|sHt	|j
D ] }||}t|| |jd qNd S r  )r   rY   rE   rG   rH   r  r=   r   rJ   rK   r   ru   rm   r[   r  r   r   r$   test_column_of_lists  s    


z)TestConvertListTypes.test_column_of_listsc                 C   sZ   g dddgddddgg dgg}t dd |D }t|}t | }t|| d S )	Nr&   rs   r   r   r   r   c                 S   s   g | ]}t j|td qS )r   )r5   r^   float)r"   ra   r   r   r$   r%     s     zITestConvertListTypes.test_column_of_lists_first_empty.<locals>.<listcomp>)r3   r_   rE   r^   rI   rP   r`   )rw   Z	num_listsr  rd   rX   r   r   r$    test_column_of_lists_first_empty  s
    
z5TestConvertListTypes.test_column_of_lists_first_emptyc                 C   s   t dtjddgd ddgddgddgd	d
ggtdi}ttdtt	 g}tj
j|d d |d}tj
j|dd  |d}t||g}| }t|| d S )Nlistsr   r&   rs   r   r   r   r   r   	   r   r  )r3   r4   r5   r^   r   rE   r=   ru   r  r/   rG   rH   concat_tablesrI   rP   rQ   )rw   rR   r=   r  r  rW   rX   r   r   r$   test_column_of_lists_chunked  s*     z1TestConvertListTypes.test_column_of_lists_chunkedc              	   C   s^   t dtjg tdi}ttdtt	 g}tj
j||d}| }t|| d S )Nr  r   r  )r3   r4   r5   r^   r   rE   r=   ru   r  r/   rG   rH   rI   rP   rQ   )rw   rR   r=   rW   rX   r   r   r$   "test_empty_column_of_lists_chunked  s     z7TestConvertListTypes.test_empty_column_of_lists_chunkedc           
   	   C   s   ddgddgddgddgd	d
gddgddgddgg}ddgddgg}t |}t |}t jj|gdgd}t jj|gdgd}t ||g}| }td|| i}	t	||	 d S )Nr   r   r&   rs   r   r   r   r   rP  r  r{                 r   r        r   r   )
rE   r^   rG   r   r  rI   r3   r4   rP   rQ   )
rw   r  r  r  r  r  r  ZconcatenatedrX   rS   r   r   r$   test_column_of_lists_chunked2  s      

z2TestConvertListTypes.test_column_of_lists_chunked2c                 C   sP   t  \}}tj|gd dd}|d jd d d }|jd dksDtt| d S )Nr   T)Zignore_indexr/   rs   r   r   )r   r3   concatrl   stridesrK   rm   )rw   rR   r=   rd   r   r   r$   test_column_of_lists_strided+  s
    
z1TestConvertListTypes.test_column_of_lists_stridedc                 C   s   t jd d gd gtd}t|}tt|}||s<t|jtt	 ksTtt jd d d d gt jd d gtdgtd}t|}td d d d gd d gg}||std S )Nr   )
r5   r^   r   rE   r   rJ   rK   r[   r  r	  )rw   r   rd   rS   r  r   r   r$   test_nested_lists_all_none4  s    

z/TestConvertListTypes.test_nested_lists_all_nonec                 C   sR   t g g g g}t|}tt|}||s6t|jtt	 ksNtd S N)
r3   r_   rE   r^   r   rJ   rK   r[   r  r	  rw   r   rd   rS   r   r   r$   test_nested_lists_all_emptyC  s
    
z0TestConvertListTypes.test_nested_lists_all_emptyc                 C   sR   t g dgg}t|}tt|}||s6t|jtt	 ksNtd S Nr   )
r3   r_   rE   r^   r   rJ   rK   r[   r  r0  r  r   r   r$   test_nested_list_first_emptyK  s
    
z1TestConvertListTypes.test_nested_list_first_emptyc                 C   s   t tjdddgddd g}t|}t|j}tjdddgd gtt d}||sdt	||srt	t tjdddgddd g}t|}tjdddgd gtt
 d}||st	d S )Nr   r&   rs   ri  r   r  rF  )r3   r_   r5   r^   rE   rl   r  r,   rJ   rK   r0   )rw   r   rX   r   rS   Zdata3r|  Z	expected3r   r   r$   test_nested_smaller_intsS  s    
"
"z-TestConvertListTypes.test_nested_smaller_intsc                 C   s   t dtjdgddggfdddgddggfdd dgd	d
ggfdd dgd	d
ggd gfg}t|}ttdtt	 tdtt	 tdtt
 tdttt
 g}t||d d S )NZnan_intsr   r&   rs   r  r   strsr   r   dZnested_strsrH  )r   r5   rR  r3   r4   rE   r=   ru   r  r/   r0  rY   rw   r   rR   rT   r   r   r$   test_infer_listsb  s    
z%TestConvertListTypes.test_infer_listsc           
      C   s   t jt  dd}t t  }ddddgd dddd	gd
dddgg}t j||d}t j||d}| }| }t||D ](\}}	|d kr|	d kstt||	 qzd S )Nr   )Z	list_sizer   r   r&   rs   r   r   r   r   r  rP  r  r  )	rE   r  r/   r^   rI   rT  rK   ro   rp   )
rw   Zfixed_tyZvariable_tyr   Z	fixed_arrZvariable_arrrX   rS   leftrightr   r   r$   test_fixed_size_listt  s    $z)TestConvertListTypes.test_fixed_size_listc              	   C   sj   t dtjddgtjdtjddgtjdgfg}t|}ttdt	t g}t
||d d S )Nr  r   r   r   r&   rs   rH  )r   r5   r^   r/   r3   r4   rE   r=   ru   r  rY   r  r   r   r$   test_infer_numpy_array  s    
z+TestConvertListTypes.test_infer_numpy_arrayc              	   C   s   t dddgt  }t ddgddgdd	ggt t  }t j||gd
dg}t jdddg|}t	dddgdgdddgdddd	gdgg}t	|
 }t   tddt t|| W 5 Q R X d S )Nr   r&   rs   r   r   r   r  eff1rE  r   )r  rE  rA   rB   )rE   r^   r.   r  r0  StructArrayr   	ListArrayr3   r_   rI   rL   rM   rN   rO   rP   r`   )rw   r  rL  structsr   rS   r  r   r   r$   test_to_list_of_structs_pandas  s$    
z3TestConvertListTypes.test_to_list_of_structs_pandasc                 C   s   t tjt dkr.t tjt dk r.td dddgfddd	gfggd
g fddd dgfgdddgfggg}t|tt	t
 tt
 }| }t|}t   tddt t|| W 5 Q R X d S )N1.25.0.dev0r   $Regression in pandas with numpy 1.25r   r   r   r  r   r  r  rO  Nquuxr  quzr  grA   rB   )r   r5   r  r3   r   r  rE   r^   r  map_utf8rI   r_   rL   rM   rN   rO   rP   r`   )rw   r   rd   r  rS   r   r   r$   test_to_list_of_maps_pandas  s    
"(

z0TestConvertListTypes.test_to_list_of_maps_pandasc           	   
   C   sN  t tjt dkr.t tjt dk r.td tdddddd	dgd
d}tdgdgddgddgg dddggt	t
 dd}tjdddg||}tjdd
dg|}| }tdddgfdddgfggdg fdggg}|d
d }tdg fdggg}t , tddt t|| t|| W 5 Q R X dS )zL
        A slightly more rigorous test for chunk/slice combinations
        r  r   r  rA   r   r  r  rO  r  r   r   r   r   r   r  Nr  r&   r   r   r  rB   )r   r5   r  r3   r   r  rE   r^   slicer  r0  MapArrayr   r  rI   r_   rL   rM   rN   rO   rP   r`   )	rw   r1  itemsmaprd   r  rS   Zseries_slicedZexpected_slicedr   r   r$   "test_to_list_of_maps_pandas_sliced  sH    
     
z7TestConvertListTypes.test_to_list_of_maps_pandas_slicedzt,data,expectedr   r&   rs   NZaaabbr   c                 C   sZ   t dddg}t|}tjj||t| d}tjj|t| d|sVt	d S )NTFr  r  )
r5   r^   r3   r_   rE   r  rH   r  rJ   rK   )rw   r   r   rS   r2  ra   rX   r   r   r$   ,test_array_from_pandas_typed_array_with_mask  s    

zATestConvertListTypes.test_array_from_pandas_typed_array_with_maskc                 C   sX   t jdtd}|g  tt dddg|d}tj	|}|
 }t|| d S )N)rs   r   r   r   3r   )r5   emptyr   fillr3   r4   r^   rE   rG   rH   rI   rP   rQ   )rw   Zempty_list_arrayrR   r4  rX   r   r   r$   test_empty_list_roundtrip  s    
z.TestConvertListTypes.test_empty_list_roundtripc                 C   s`   t  \}}|D ]L}||j j}tjt||jd}t|}|j|jksLt||stqd S Nr  )	r   r]   rl   rE   r^   r   r[   rK   rJ   )rw   rR   r=   ru   rd   rS   rX   r   r   r$   test_array_from_nested_arrays	  s    

z2TestConvertListTypes.test_array_from_nested_arraysc              	   C   s   t jdddgdggd gt t t  d }t N tddtj	 tddt
 tj|tjdddgdggd gtd	d
d W 5 Q R X d S )Nr   r&   rs   r   r  rA   z&Creating an ndarray from ragged nestedrB   r   Frg   )rE   r^   r  r/   rI   rL   rM   rN   r5   ZVisibleDeprecationWarningrO   rP   r`   r3   r_   r   ri  r   r   r$   test_nested_large_list	  s     

 z+TestConvertListTypes.test_nested_large_listc              	   C   s   t jt jfD ]}t jddgd dgg g|t  d }tj|t	ddgd dgg gdd	 t jddgd dgg g|t 
 d }tj|t	ddgd dgg gdd	 qd S )
Nr   r  ccr  s   aas   bbs   ccFrg   )rE   r  r  r^   rm  rI   rP   r`   r3   r_   rp  )rw   Zlist_type_factoryra   r   r   r$   test_large_binary_list#	  s"    

 

 z+TestConvertListTypes.test_large_binary_listc                 C   sz   t ddd dg }t jdddddg|}t| }t	|
 | |ddd dg}d |d< t	|
 | d S )Nr   r  r   r   rs   r   r&   )rE   r^   r  r  r   r3   r_   r]  rP   r`   rI   take)rw   childrd   rS   r   r   r$   test_list_of_dictionary2	  s    z,TestConvertListTypes.test_list_of_dictionaryc                 C   s~   d}t jttj|dfddt|d}tj	|}|d }|j
dksNtt|ddksdtt|d	d	ksztd S )
Ni    i   r(   r   r   r   r&   i r   )r3   r4   r  r   r5   zerosr9   rE   rG   rH   rZ  rK   r   r  )rw   nrR   rW   Zcolumn_ar   r   r$   #test_auto_chunking_on_list_overflow?	  s    z8TestConvertListTypes.test_auto_chunking_on_list_overflowc                 C   sX   ddgdgdddgdgg}t d|i}tdtt t fg}t||d	 d S )
Nr   r      br&      crs      dr      er      fr      gr   r  r  )r3   r4   rE   r=   r  rS  r.   rY   )rw   r   rR   r=   r   r   r$   test_map_array_roundtripN	  s     z-TestConvertListTypes.test_map_array_roundtripc                 C   s   ddgdgdddgdgg}dd	 |D }t j|t t  t  d
}t j|t t  t  d
}t ||g}t|| }| }t	j
||dd d S )Nr  r  r  r  r  r  r  c                 S   s   g | ]}d d |D qS )c                 S   s   g | ]\}}||d  fqS )r&   r   r"   kr  r   r   r$   r%   ^	  s     zJTestConvertListTypes.test_map_array_chunked.<locals>.<listcomp>.<listcomp>r   r"   rowr   r   r$   r%   ^	  s     z?TestConvertListTypes.test_map_array_chunked.<locals>.<listcomp>r  Frg   )rE   r^   r  rS  r.   r  r3   r_   rI   rP   r`   )rw   r  r  arr1r  rd   rS   actualr   r   r$   test_map_array_chunkedY	  s    z+TestConvertListTypes.test_map_array_chunkedc                 C   sj   ddgd dddgdgg}dd |D }t |}tj|tt t d	}| }tj	||d
d d S )Nr  r  r  r  )r  Nr  c                 S   s&   g | ]}|d k	rdd |D nd qS )Nc                 S   s(   g | ] \}}||d k	rt |nd fqS r  )r  r  r   r   r$   r%   o	  s     zMTestConvertListTypes.test_map_array_with_nulls.<locals>.<listcomp>.<listcomp>r   r
  r   r   r$   r%   o	  s    zBTestConvertListTypes.test_map_array_with_nulls.<locals>.<listcomp>r  Frg   )
r3   r_   rE   r^   r  rS  r.   rI   rP   r`   )rw   r   rS   rd   r  r   r   r$   test_map_array_with_nullsh	  s    
z.TestConvertListTypes.test_map_array_with_nullsc                 C   s   t dddg}t dddddg }t ttt|}t j|||}t	dd	d
gddgg}|
 }tj||dd d S )Nr   rs   r   r   r   r   r  )r   r   )r   r   )r&   r   )rs   r   )r   r  Frg   )rE   r^   r  r   r9   r   r  r   r3   r_   rI   rP   r`   )rw   offsetsr  r1  rd   rS   r  r   r   r$   !test_map_array_dictionary_encodedw	  s    z6TestConvertListTypes.test_map_array_dictionary_encodedc           	   	   C   sp  t ddgdddgd dd gddgg}t |d	d|ddg}| }tjd
dgdddgd dtjggdd}t||D ]*\}}|d kr||kstq|t	
|| q|td
dddddtjgg}t	
|d	 j| |dd }tjdddgd dtjggdd}t||D ]0\}}|d kr2||ks>tnt	
|| qtddddtjgg}t	
|d	 j| d S )Nr   r&   rs   r   r   r   r   r   r   ry   rz   r{   r|   r}   r~   r   r   )rE   r^   r  r  to_numpyr5   r\  rT  rK   ro   rp   base)	rw   rd   Zchunked_arrr}  rS   r  r  Zexpected_baseZnp_arr_slicedr   r   r$   test_list_no_duplicate_base	  s,    & 
z0TestConvertListTypes.test_list_no_duplicate_basec              
   C   s   t jjt ddddgt dddddd gt dd	dgd
}|jdd}tjddgd dtjggdd}t||D ]*\}}|d kr||kstqrt	
|| qrd S )Nr   r&   r   r   r   c   rs   FT)r  rl   rf   r  ry   rz   r{   r   r   )rE   r  r   r^   r  r5   r\  rT  rK   ro   rp   )rw   rd   r}  rS   r  r  r   r   r$   test_list_values_behind_null	  s    z1TestConvertListTypes.test_list_values_behind_null).r@  rA  rB  rC  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r   rC  rD  rE   r/   r0  r	  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r   r   r   r$   r    sd   	'



		
r  c                   @   sh   e Zd ZdZdd Zdd Zdd Zdd	 Zd
d Ze	j
je	j
jdd Zdd Zdd Zdd ZdS )TestConvertStructTypesz,
    Conversion tests for struct types.
    c              	   C   sf   t dddddddgi}tdtdt fdt fgfg}t||d	 t|||d
 d S )Nr  r   r&   r   rs   r   r   r   rH  r  )r3   r4   rE   r=   r  r/   rY   )rw   rR   rT   r   r   r$   test_pandas_roundtrip	  s       z,TestConvertStructTypes.test_pandas_roundtripc                 C   s   t jd ddgt  d}t jdd dgt  d}t jddd gt  d}t j|||gdd	d
g}td ddddd ddddd dg}t|	 }t
|| d S )Nr&   rs   r  r   r   TFr  r  r  )r  r  r  )rE   r^   r/   r0  rb  r  r   r3   r_   rI   rP   r`   )rw   r  r  r  rd   rS   r  r   r   r$   test_to_pandas	  s    


z%TestConvertStructTypes.test_to_pandasc           
      C   s   t   t }tjdgt d}tjdgt d}tj|gdg}tj|gdg}t||g}t	
ddiddig}t	
| }t|| ~~~~~~t }	|	|kstd S )Nr   r  r&   r  )gccollectrE   total_allocated_bytesr^   r/   r  r   r  r3   r_   rI   rP   r`   rK   )
rw   Zbytes_startZints1ints2r  r  rd   rS   r  Z	bytes_endr   r   r$   test_to_pandas_multiple_chunks	  s*    z5TestConvertStructTypes.test_to_pandas_multiple_chunksc                 C   s\  t dt jfdt jfg}ttdt tdt g}t jg |d}tj||d}| g kslt	t jddg|d}tj||d}| dd	d
ddd
gkst	tj|t dd	g|d}| dd	d
d gkst	t g }tg }t jg |d}tj||d}| g ks"t	t jddg|d}tj||d}| i i gksXt	d S )Nr#   )Zy_titler  r  r   r  )r  T)+   Fr  T)r#   r  r   Fr  r   )
r5   r   r.   rb  rE   r  ru   r^   r]  rK   rw   r=  r  r   rd   r   r   r$   test_from_numpy	  s0    


z&TestConvertStructTypes.test_from_numpyc                 C   s  t dt dt jfdt jfgfdt jfdt jfg}|jdksDtt	t
dt	t
dt t
dt gt
dt t
dt g}t jg |d}tj||d}| g kstt jd	d
g|d}tj||d}| ddddddddddddgkstd S )Nr#   xxyyr  zr{  r   r  ))r   Tr&   r   ))rs   Fr   r  r   T)r#  r$  r&   r   )r#   r  r%  rs   Fr   r  )r5   r   r,   rb  r-   object_itemsizerK   rE   r  ru   r0  r^   r]  r!  r   r   r$   test_from_numpy_nested
  s6    z-TestConvertStructTypes.test_from_numpy_nestedc                    s  d}t dt jfdg}d|j }d| }|||j  }t j||d}t j||d< ||d< t j|d |d dk < t	t
dt t
dt g}tj||d	d
}|jdkstdd  d fdd	}	|	|| ~t j|dk }
tj|||
d	d}|jdkst|	|||
 ~d S )Nl        r#   )r  r   i      .r   r  r#  Tr  r&   c                 s   s   |   D ]}|E d H  qd S r  )Z
iterchunks)rd   r  r   r   r$   iter_chunked_array<
  s    zHTestConvertStructTypes.test_from_numpy_large.<locals>.iter_chunked_arrayc           	   	      s   t | t |kst|d }|d }t | D ]\}}zr| }|d k	r`|| r`|d kstnH|| }t|r|d d kstn|d |kst|d || kstW q0 tk
r   td|  Y q0X q0d S )Nr#   r  zFailed at index)r   rK   	enumeraterg  r5   rf  	Exceptionprint)	rd   r   rf   xsZysir  r  r#   r*  r   r$   check@
  s     

z;TestConvertStructTypes.test_from_numpy_large.<locals>.check)r[   rf   rH   )N)r5   r   r1   r'  r  r7   Zrandom_samplerR  rE   r  ru   rS  r^   rZ  rK   )rw   Ztarget_sizer=  bsblockr  r   r  rd   r1  rf   r   r0  r$   test_from_numpy_large(
  s.    

z,TestConvertStructTypes.test_from_numpy_largec              	   C   s   t t dt  t dt  g}tdtjfdtjfg}tjg |d}tj	t
dd t j||d W 5 Q R X tg }tj	tdd t j||d W 5 Q R X d S )	Nr#   r  r%  r   zMissing field 'y'r  r  zExpected struct array)rE   r  ru   r.   rb  r5   r   r^   r   r   r   r  )rw   r  r=  r   r   r   r$   test_from_numpy_bad_input_
  s     
z0TestConvertStructTypes.test_from_numpy_bad_inputc                 C   s   t dddgi}t dddddddgi}td	t fd
t fg}t|d }t||d |d td|fg}t	||||d d S )NZtuples)r   r&   rs   r   r   r&   r   rs   r   r   r   ra  )rS   r=   rT   )
r3   r4   rE   r  r/   r5   asarrayrm   r=   rY   )rw   rR   r  Zstruct_typerd   rT   r   r   r$   test_from_tuplesn
  s$        z'TestConvertStructTypes.test_from_tuplesc                    s   ddg t dddg t ddd g g}t jj| d}tdd |D  } fd	d
|D }t|}t	|
 | |dd dg}d |d< t	|
 | d S )Nr  r  i  i  r   r   c                 s   s   | ]}|  V  qd S r  )r]  )r"   r  r   r   r$   	<genexpr>
  s     zCTestConvertStructTypes.test_struct_of_dictionary.<locals>.<genexpr>c                    s   g | ]}t t |qS r   )rS  rT  r
  r   r   r$   r%   
  s     zDTestConvertStructTypes.test_struct_of_dictionary.<locals>.<listcomp>r   r&   r   )rE   r^   r  r  r   rT  r3   r_   rP   r`   rI   r  )rw   childrenrd   Zrows_as_tuplesZrows_as_dictsrS   r   r   r$   test_struct_of_dictionary
  s    
z0TestConvertStructTypes.test_struct_of_dictionaryN)r@  rA  rB  rC  r  r  r  r"  r(  r   rC  Zslowr  r4  r5  r8  r;  r   r   r   r$   r  	  s   5r  c                   @   sp   e Zd ZdZdd Zdd Zdd Zdd	 Zd
d Zdd Z	dd Z
dd Zdd Zdd Zdd Zdd ZdS )TestZeroCopyConversionz@
    Tests that zero-copy conversion works with some types.
    c                 C   s.   t dddgjdd}t|dddg d S )Nr   r   r&   Tr  rn   )rw   rX   r   r   r$   test_zero_copy_success
  s    z-TestZeroCopyConversion.test_zero_copy_successc                 C   s^   t jtddgtjdgdd}|jdd}tddg}tj	t
|t
|dd d S )	Nr   r   r/   r   Tr  Frg   )rE   r   r   r5   r^   rI   r3   rt  rP   r`   r_   )rw   rd   rX   rl   r   r   r$   test_zero_copy_dictionaries
  s    z2TestZeroCopyConversion.test_zero_copy_dictionariesc                 C   s2   t jdgdd}t|jdd}t|| d S )Nr0  r   r   Tr  r5   r^   rE   rI   ro   rp   rw   rd   rX   r   r   r$   test_zero_copy_timestamp
  s    z/TestZeroCopyConversion.test_zero_copy_timestampc                 C   s2   t jdgdd}t|jdd}t|| d S )Nr   r   r   Tr  r?  r@  r   r   r$   test_zero_copy_duration
  s    z.TestZeroCopyConversion.test_zero_copy_durationc              	   C   s(   t tj |jdd W 5 Q R X d S )NTr  )r   r   rE   rd  rI   rw   rd   r   r   r$   check_zero_copy_failure
  s    z.TestZeroCopyConversion.check_zero_copy_failurec                 C   s   |  tdddg d S )NABCrD  rE   r^   r  r   r   r$   &test_zero_copy_failure_on_object_types
  s    z=TestZeroCopyConversion.test_zero_copy_failure_on_object_typesc                 C   s   |  tddd g d S )Nr   r   rH  r  r   r   r$   *test_zero_copy_failure_with_int_when_nulls
  s    zATestZeroCopyConversion.test_zero_copy_failure_with_int_when_nullsc                 C   s   |  tddd g d S )Nr`  ry   rH  r  r   r   r$   ,test_zero_copy_failure_with_float_when_nulls
  s    zCTestZeroCopyConversion.test_zero_copy_failure_with_float_when_nullsc                 C   s   |  tddg d S )NTFrH  r  r   r   r$   $test_zero_copy_failure_on_bool_types
  s    z;TestZeroCopyConversion.test_zero_copy_failure_on_bool_typesc                 C   s2   t jddgddggt t  d}| | d S )Nr   r&   r   r  r  )rE   r^   r  r/   rD  rC  r   r   r$   $test_zero_copy_failure_on_list_types
  s    $z;TestZeroCopyConversion.test_zero_copy_failure_on_list_typesc                 C   s&   t jdd gdd}| t| d S )Nr   r   r   r5   r^   rD  rE   rC  r   r   r$   .test_zero_copy_failure_on_timestamp_with_nulls
  s    zETestZeroCopyConversion.test_zero_copy_failure_on_timestamp_with_nullsc                 C   s&   t jdd gdd}| t| d S )Nr   r   r   rN  rC  r   r   r$   -test_zero_copy_failure_on_duration_with_nulls
  s    zDTestZeroCopyConversion.test_zero_copy_failure_on_duration_with_nullsN)r@  rA  rB  rC  r=  r>  rA  rB  rD  rI  rJ  rK  rL  rM  rO  rP  r   r   r   r$   r<  
  s   r<  c                  C   s$   t  } t| dd t| ddd d S )NFr?   Tr@   rU   r<   rY   rR   r   r   r$   _non_threaded_conversion
  s    rT  c                  C   s$   t  } t| dd t| ddd d S )NTr?   rQ  rR  rS  r   r   r$   _threaded_conversion
  s    rU  c                   @   s  e Zd ZdZeje feje feje fej	e	 fej
e
 feje feje feje feje feje feje feje feje fejedfejee	 fgZdd Zdd Zdd Zd	d
 Zdd Zdd Zdd Zdd Zdd Zej !dddddgiej"fddddgiej"fdddgiej"fdddgiej#fdddgiej#fdddgiej#fgdd Z$dd Z%dd  Z&d!d" Z'd#d$ Z(d%d& Z)d'd( Z*d)d* Z+d+d, Z,d-d. Z-d/S )0TestConvertMiscz)
    Miscellaneous conversion tests.
    rP  c                 C   s    t dd d d gi}t| d S r  r   r   r   r   r$   test_all_none_objects
  s    z%TestConvertMisc.test_all_none_objectsc                 C   s2   t dd d d gi}|d d|d< t| d S )Nr   r   )r3   r4   r   rY   r   r   r   r$   test_all_none_category
  s    z&TestConvertMisc.test_all_none_categoryc                 C   s.   | j D ]"\}}tjg |d}t||d qd S )Nr   r  )
type_pairsr5   r^   rm   )rw   r   pa_typerd   r   r   r$   test_empty_arrays
  s    z!TestConvertMisc.test_empty_arraysc                 C   s
   t   d S r  )rT  r  r   r   r$   test_non_threaded_conversion  s    z,TestConvertMisc.test_non_threaded_conversionc                 C   s0   t d}z|t W 5 |  |  X d S )Nr&   )mpZPoolclosejoinapplyrU  )rw   poolr   r   r$   %test_threaded_conversion_multiprocess  s
    
z5TestConvertMisc.test_threaded_conversion_multiprocessc                 C   s   d}dd ddt jg}dddddg}d	d d
dt jg}t|| tj|| ddgdt|| t|| tj|| dddgdd|| || || || || d
}t|}t| |D ]}t||  qd S )Nr   r   r  rO  r   r   r   r   rP  ra  rQ  ry  Trz  r   )
Zcat_stringsZcat_strings_with_naZcat_intsZ
cat_binaryZcat_strings_orderedr  r  rL  Zstrings2Zstrings3)r5   rR  r3   rt  r4   rY   rm   )rw   rM  r[  r\  Zv3r2  rR   r	  r   r   r$   test_category  s2    
 
zTestConvertMisc.test_categoryc                 C   sP   dd }t jdddgddgdt jdddgddgddg}|D ]}|| q>d S )	Nc                 S   s.   t | }| }tt|t|  d S r  )rE   r^   rI   rP   r`   r3   r_   )r  rd   rX   r   r   r$   _check)  s    
zBTestConvertMisc.test_category_implicit_from_pandas.<locals>._checkr   r   r   ry  Trc  )r3   rt  )rw   re  r2  rd   r   r   r$   "test_category_implicit_from_pandas'  s    z2TestConvertMisc.test_category_implicit_from_pandasc                 C   s    t dt g i}t| d S )Ncat)r3   r4   rt  rY   r   r   r   r$   test_empty_category6  s    z#TestConvertMisc.test_empty_categoryc                 C   s   t  dft  dffD ]\}}t g t t  |}| }tjg t	j
g |dd}tt|t| t d|i}| }td|i}t|| qd S )Nr   r/   r   ry  r   )rE   r0  r/   r  
dictionaryr,   rI   r3   rt  r5   r^   rP   r`   r_   rW   r4   rQ   )rw   rZ  r   r   rX   rS   rW   r   r   r$   test_category_zero_chunks;  s     z)TestConvertMisc.test_category_zero_chunkszdata,error_typer   r   rz   Try   c              	   C   s8   t |}d}tj||d tj| W 5 Q R X d S )N/Conversion failed for column a with type objectr  )r3   r4   r   r   rE   rG   rH   )rw   r   Z
error_typerR   r-  r   r   r$   test_mixed_types_failsH  s    
z&TestConvertMisc.test_mixed_types_failsc              
   C   sH  g }dddg}d\}}t j|| d }dddd	d
dddddg
}|D ]}||| qH|t jdd t|| D td	||  t jdddg| td	|| }d |d< || |t j
d|| dd	||  |dktd d df }	|D ]8}
tj|
|d}|d }t| t| t||	d q
d S )Nr   r   r   )r   rs   r   ri  rj  rk  rl  rm  rn  ro  rp  rF  rG  c                 S   s   g | ]}t d qS rP  r   r"   r/  r   r   r$   r%   g  s     z<TestConvertMisc.test_strided_data_import.<locals>.<listcomp>r   TFr   r   r   r   r   r  )r5   r7   r8   rk   rJ  r   r^   r9   r   Zreshaper6   r2   r3   r4   rY   rm   )rw   casesr   NKZrandom_numbersZnumeric_dtypes	type_nameZboolean_objectsZstrided_maskZcaserR   r~  r   r   r$   test_strided_data_importY  sF    
  
 
 
z(TestConvertMisc.test_strided_data_importc                 C   sX   dd }|t jd gd td |t jtjgd td |t jd tjd gtd d S )Nc                 S   sT   t | }t|t jstt|dks*t|jdks8t|D ]}|t jks<tq<d S )Nrs   )rE   r^   r   Z	NullArrayrK   r   ri   r  )ra   r  itemr   r   r$   _check_series  s    
z5TestConvertMisc.test_all_nones.<locals>._check_seriesrs   r   )r3   r_   r   r5   rR  )rw   rv  r   r   r$   test_all_nones  s    zTestConvertMisc.test_all_nonesc              	   C   s   t ddddddgfdtjdd	dd
dgtjdfddd	dd
dgfg}t|}ttdt	 tdt	 g}t
|||ddg |d d S )Nr   r   r   r&   rs   r   r   ir   rP  r   r   )r=   rS   rT   )r   r5   r^   r.   r3   r4   rE   r=   ru   r/   rY   )rw   r   rR   Zpartial_schemar   r   r$   test_partial_schema  s    

z#TestConvertMisc.test_partial_schemac                 C   s   t i }t|d d t|d dd t jt g d}t||dd t||ddd t ji dddgd	}t|dd t|ddd
 d S )Nr   T)r>   rU   r   Fr   r   r&   r   )rU   r>   )r3   r4   rY   r   )rw   rR   rS   r   r   r   r$    test_table_batch_empty_dataframe  s    
z0TestConvertMisc.test_table_batch_empty_dataframec              	   C   s   t jg t  d}ttjg td}t|	 ttjg tjd t jg t 
 d}t|	 | t jg t t  d}t|	 | t jg t t dt  gd}t|	 | d S )Nr  r   r   )rE   r^   r/   r3   r_   r5   r   rP   r`   rI   r0  r  r  ru   )rw   rd   Zempty_objectsr   r   r$   test_convert_empty_table  s    
"z(TestConvertMisc.test_convert_empty_tablec                 C   s   t dt jfdt jfg}t jddg|d}|jdks:ttj|d t d}| dd	gksdttj|d t d}| d
dgkstdS )zt
        ARROW-2172: converting from a Numpy array with a stride that's
        not a multiple of itemsize.
        r#   r  )r  r   )r&   r   )r   r  r  r|  r   r&   N)	r5   r   r.   r-   r^   r  rK   rE   r]  )rw   r   r   rd   r   r   r$   test_non_natural_stride  s    z'TestConvertMisc.test_non_natural_stridec                 C   sV   t jddt jdddd }tj|t d}tjdd	d
dgt d}|| d S )Nr   rP  r   r   r   r&   r  ry   r{   r}   g      @)r5   r6   r0   rE   r^   r1   rJ   )rw   r}  Zpa_arrrS   r   r   r$   #test_array_from_strided_numpy_array  s    z3TestConvertMisc.test_array_from_strided_numpy_arrayc              	   C   s   t tdtdddd}ttdt tdt	 g}t
t tjj||d W 5 Q R X tjj||d	d
}|djt	 kstd S )Nabcr   r   rs   )rE  rF  rE  rF  r  Fr[  )r3   r4   r   r5   ZlinspacerE   r=   ru   r0  r.   r   r   r   rG   rH   r,  r[   rK   r^  r   r   r$   test_safe_unsafe_casts  s    z&TestConvertMisc.test_safe_unsafe_castsc                 C   s~   z"t dt jdtjdgi}W n0 tk
rR   t dt dtjdgi}Y nX tjt	dd t
j| W 5 Q R X d S )Nr   r   rs   zSparse pandas datar  )r3   r4   r2  ZSparseArrayr5   rR  AttributeErrorr   r   r  rE   rG   rH   r   r   r   r$   test_error_sparse  s    ""z!TestConvertMisc.test_error_sparseN).r@  rA  rB  rC  r5   r,   rE   r-   r.   r/   r(   r)   r*   r+   rI  r0   r1   r&  r0  rS  r  rY  rW  rX  r[  r\  rb  rd  rf  rh  rj  r   rC  rD  r  rd  rl  rt  rw  ry  rz  r{  r}  r~  r  r  r   r   r   r$   rV  
  s\   	
(rV  c                  C   sT   t ddd dg} tjj| t dd}tjddd dgt d}||sPtd S )Nr   r&   r   Tr[   r   r  )	r3   r_   rE   r  rH   r.   r^   rJ   rK   )rl   rd   rS   r   r   r$   *test_safe_cast_from_float_with_nans_to_int  s    r  c                  C   s   t jt jddddttjddgtddg} t jddd}||	d	|	d
|d d d 	d	d
dddgd t ddgd 
djddgd tjdtjjddddt jddddt jddddd}t j|| dS )Nr  r   r8  r&   r   r  r   rP  r  r  r   TFr   r   r:   Z2013Mr   r   r   )r   r   r   )r   r   r&   rs   r   r   r   r   r   r  rP  r   )r3   r   r   r   repeatr5   Ztiler^   r   r>  r   r_   rl   r7   r8   rQ  period_rangeinterval_ranger4   )r   r.  r   r   r   r$   _fully_loaded_dataframe_example  s$    


r  r   rP  r   c                 C   s   t dt tj  kr"t dk r0n n
td tj| d}tj|}tj|	 }|
|sft|j
|jsxt|jj|jjkstd S )Nr   r   r  r   )r   r3   r  r   r  r4   rE   rG   rH   rI   rJ   rK   r=   r   )r   rR   table1table2r   r   r$   !test_roundtrip_with_bytes_unicode	  s    &
r  c                 C   s>   | d }| d } | d }| d } | d }| d }t ||||S )Nr  r  )r   )valrA  secondsminutesr  r   r   r$   _pytime_from_micros  s    r  c                 C   s$   | j d | jd  | jd  | j S )Nl    $'- i r  )hourminuter)  r  )Zpytimer   r   r$   r  #  s    r  c               	   C   sR   G dd d} t d|  |  gi}d}tjt|d tj| W 5 Q R X d S )Nc                   @   s   e Zd ZdS )z6test_convert_unsupported_type_error_message.<locals>.ANr  r   r   r   r$   rE  .  s   rE  r   rk  r  )r3   r4   r   r   r   rE   rG   rH   )rE  rR   r-  r   r   r$   +test_convert_unsupported_type_error_message*  s
    r  c                 C   s,   |   }tj|| jdd}|| s(td S )NTr  )rI   rE   r^   r[   rJ   rK   )rd   ra   r   r   r   r$   test_array_to_pandas_roundtrip<  s    r  c                 C   s   dd t | D }|| S )Nc                 S   s   g | ]}t d qS rm  )r   ro  r   r   r$   r%   H  s     z+_generate_dedup_example.<locals>.<listcomp>)r9   )nuniquerM  unique_valuesr   r   r$   _generate_dedup_exampleG  s    r  c                 C   s   t dd | D |kstd S )Nc                 S   s   h | ]}t |qS r   r  r!   r   r   r$   	<setcomp>M  s     z"_assert_nunique.<locals>.<setcomp>)r   rK   )r  rS   r   r   r$   _assert_nuniqueL  s    r  c                  C   sp   d} d}t | |}tj|t dtj|t dt||gfD ](}t| |  t|jddt| qBd S )Nr   rP  r  Fdeduplicate_objects)	r  rE   r^   rS  r  r  r  rI   r   )r  rM  rl   rd   r   r   r$   .test_to_pandas_deduplicate_strings_array_typesP  s    
r  c                  C   sz   d} d}t | |}t|}tj|gdg}tj|g}||fD ]0}t| d |  t|jddd t	| qDd S )Nr   rP  r   Fr  )
r  rE   r^   rF   r   rG   r   r  rI   r   )r  rM  rl   rd   r3  r4  r  r   r   r$   .test_to_pandas_deduplicate_strings_table_types\  s    

r  c                  C   sn   d} d}t tjjdd| d}d || d < t|| }t|jdd|  t|jdd	d
| d | d  d S )Nr   rP  i i ʚ;r:   r&   Tr  F)r  r  r   )r   r5   r7   rQ  rE   r^   r  rI   )r  rM  r  rd   r   r   r$   .test_to_pandas_deduplicate_integers_as_objectsk  s    r  c            	      C   s   d} d}t t| }ddddifddddifdd	i fdd
i fg}|D ]V\}}}tj|| |d}||}t|jf ||  t|jf ddi|t| qDd S )Nr   rP  r.   r  r  Tr/   r  r$  r"  r  r  F)r   r9   rE   r^   r  r  rI   r   )	r  rM  r  rp  Zraw_typeZ
array_typeZpandas_optionsZraw_arrZ
casted_arrr   r   r$   $test_to_pandas_deduplicate_date_time|  s&    
r  c               	   C   sv   t dddgtjddgd} ttjdt dd	tjd
t dd	g}t	
t tjj| |d W 5 Q R X d S )N333333?g @r   r0  r   r   r   FrY  r   r  )r3   r4   r5   rR  rE   r=   ru   r1   r  r   r   r   rG   rH   )rR   r=   r   r   r$   /test_table_from_pandas_checks_field_nullability  s    
r  c               
   C   s   t tdddddgfddddgddgd d gfdd d d	d
gfg} | dddg }tdt fdtt fdt fg}tdt fdt fdtt fg}tjj	| dd}tjj	|dd}|j
|st|j
|std S )N	partitionr   r   r2  r&   rs   r   r  r   ffffff
@Fr   )r3   r4   r   rE   r=   r/   r  r1   rG   rH   rJ   rK   )df1r   schema1schema2r  r  r   r   r$   6test_table_from_pandas_keeps_column_order_of_dataframe  s(    



r  c               
   C   s   t tdddddgfddddgddgd d gfdd d d	d
gfg} tdt fdtt fdt fg}| | jdk }| | jdk dddg }tj	j
||dd}tj	j
||dd}|j|st|j|jstd S )Nr  r   r   r2  r&   rs   r   r  r   r  Fr  )r3   r4   r   rE   r=   r1   r  r.   r  rG   rH   rJ   rK   )rR   r=   r  r   r  r  r   r   r$   3test_table_from_pandas_keeps_column_order_of_schema  s     

r  c               
   C   s   t tdddddgfddddgddgd d gfdd d d	d
gfg} dddg}tdtt fdt fdt fg}ddg}tdt fdt fg}tjj	| |dd}tjj	| |dd}|j
|st|j
|std S )Nr  r   r   r2  r&   rs   r   r  r   r  F)r   r>   )r3   r4   r   rE   r=   r  r/   r1   rG   rH   rJ   rK   )rR   Zcolumns1r  Zcolumns2r  r  r  r   r   r$   ;test_table_from_pandas_columns_argument_only_does_filtering  s(    




r  c               
   C   s   t tdddddgfddddgddgd d gfdd d d	d
gfg} tdt fdtt fdt fg}ddg}t	t
 tjj| ||d W 5 Q R X d S )Nr  r   r   r2  r&   rs   r   r  r   r  )r=   r   )r3   r4   r   rE   r=   r.   r  r1   r   r   r   rG   rH   )rR   r=   r   r   r   r$   @test_table_from_pandas_columns_and_schema_are_mutually_exclusive  s    

r  c                  C   s~   t dddddgi} ttjdt ddg}tj| }|jdjdksTt	tjj| |d	}|jdjdkszt	d S )
Nr   r   r&   rs   r   FrY  Tr  )
r3   r4   rE   r=   ru   r/   rG   rH   rZ  rK   rR   r=   rW   r   r   r$   /test_table_from_pandas_keeps_schema_nullability   s    r  c               	   C   s  t dddgdddgd} tdt fd	t fd
t fg}tjtdd tj	j
| |d W 5 Q R X d
| j_t| |d|d tjtdd tj	j
| |dd W 5 Q R X tjtdd tj	j
| |d d W 5 Q R X t jdddgd
d| _t| |d |d t| |d|d td
t fdt fd	t fg}t| |d |d t| |d|d tdt fd	t fg}|  }|jdd}t| |d ||d t| |d||d t jjdddgddgd| _tdt fdt fdt fd	t fg}t| |d|d t| |d |d tdt fdt fd	t fg}|  }|jddd}t| |d||d t| |d ||d d S )Nr   r&   rs   r"  r#  r$  r   r   r   r   zname 'index' present in ther  r  T)r=   r>   rT   z'preserve_index=False' wasFr  z=name 'index' is present in the schema, but it is a RangeIndexr   r\   r%  )r=   r>   rT   rS   )r   r   )r   r&   )r   r   Zlevel1Zlevel2r   )r3   r4   rE   r=   r/   r1   r   r   KeyErrorrG   rH   r   r]   rY   r   r   rk   r&  r   from_tuplesr0  )rR   r=   rS   r   r   r$   +test_table_from_pandas_schema_index_columns  s    







  






  r  c                  C   s   t dddgdddgd} tdt fd	t fd
t fg}tjj| dd}tjj| d|d}|j	 
|sztt jdddgdddgddddgd} tj| }tjj| |d}|j	 
|std S )Nr   r&   rs   r"  r#  r$  r   r   r   r   Tr   )r>   r=   r   r   r  )r3   r4   rE   r=   r/   r1   SchemarH   rG   r  rJ   rK   )rR   rT   r=   rW   r   r   r$   :test_table_from_pandas_schema_index_columns__unnamed_indexh  s    


&r  c                  C   sH   t  } tj| ddi}tjj| |d}|jj	ddksDt
d S )NmetaTruer  s   metas   True)r3   r4   rE   r  rH   r  rG   r=   r   getrK   r  r   r   r$   2test_table_from_pandas_schema_with_custom_metadata}  s    r  c               	   C   s*  t t jddddtjdd} ttjdt	 ddtjd	tj
d
ddddg}tjj| |d}|j|sxt|jjd d }|d dkst|d d kst|jjd d }|d d	kst|d ddikst| }ddi}tt jtdkrd|d	< | dd	g |}t|| d S )Nz2020-01-01T00:00:00ZHr&   )r   r   )r   r  r  TrY  r   ra   r:  r;  Fr  r   r   r]   r   r   r   r0   r   zdatetime64[s, UTC])r3   r4   r   r5   r7   r8   rE   r=   ru   r0   r<  rG   rH   rJ   rK   r   rI   r   r  r   rP   rQ   )rR   r=   rW   Zmetadata_floatZmetadata_datetimerX   Zcoerce_cols_to_typesrS   r   r   r$   2test_table_from_pandas_schema_field_order_metadata  s,    
r  c                  C   s~   t tjdddddgddtjdddddgddtjdd	d
d ddgdddddgd} tj| }|	 }t
| | d S )Nr   r&   rs   r   r   r/   r   r*   r   r  r  rO  FT)r.  r/  c3c4Zc5)r3   r4   r5   r^   r7   r8   rE   rF   rH   rI   rP   rQ   )r   r   rX   r   r   r$   test_recordbatch_from_to_pandas  s    
r  c                  C   s   t tjdddgddtjdddgddd	d d
gddd gd} t tjddgddtjddgddd	d	gddgd}tj| }tj|}tj||g}|	 }t 
| |gjd	d}t|| d S )Nr   r&   r*   r   ry   rz   r{   r1   TFr   r  )r.  r/  r  r  rs   r   r|   r}   r  rO  r%  )r3   r4   r5   r^   rE   rF   rH   rG   r   rI   r  r&  rP   rQ   )r  r  Zbatch1Zbatch2rW   rX   r   r   r   r$   test_recordbatchlist_to_pandas  s$    r  c                  C   sp   t jt ddddggdgd} t jt ddddggdgd}| d  jdksVt|d  jdksltd S )Nr   r&   rs   r   Za0r   r   )rE   Zrecord_batchr^   rW   rI   r]   rK   )r3  r   r   r   r$   *test_recordbatch_table_pass_name_to_pandas  s      r  r[   rS   r  r2   r,   r-   r.   r/   r(   r)   r*   r+   rI  r0   r1   r   r  r   )lengthr   zlist[list[int16]]r  rs   r  r  r   r  r:  r   ra   r   c                 C   s   t | |kstd S r  )r   rK   r  r   r   r$   test_logical_type  s    r  c                  C   s   d} t jtj| tjdtjjdd| dtjd}t	
  t  }| }t  || d  ksdtd }t	
  t  |kstt tj| tjd}t  }| }t  |kstd S )N'  r   r   r&   r:   r  r   )rE   r^   r5   r6   r/   r7   rQ  r   rb  r  r  r  rI   rK   )rq  rd   prior_allocationr#   r   r   r$   test_array_uses_memory_pool  s    r  c                  C   sd   t t tjdtjdgdg} t| dd t  }|  }|d j	j
jsPtt  |ks`td S )Nr  r   r  Tsplit_blocks)rE   rW   r^   r5   r6   r/   !_check_to_pandas_memory_unchangedr  rI   rl   flagsZ	writeablerK   )r   r  rX   r   r   r$   test_singleton_blocks_zero_copy  s    "r  c                 K   s(   t  }| jf |}t  |ks$td S r  )rE   r  rI   rK   )r  kwargsr  r#   r   r   r$   r  &  s    r  c                  C   s   t t jdddddgddt jdddddgddt jdddddgd	dt jdddddgd
dt jdddddgddt jdddddgddt jdddddgddt jdddddgddgdd tdD } t| d t| dd d S )Nr   r&   rs   r   r   ri  r  rk  rl  rF  rG  c                 S   s   g | ]}d  |qS zf{}formatro  r   r   r$   r%   9  s     z/test_to_pandas_split_blocks.<locals>.<listcomp>r   Tr  )rE   rW   r^   r9   _check_blocks_createdr  )r   r   r   r$   test_to_pandas_split_blocks.  s    	
r  c                 C   s"   t tjt dk r| jS | jS d S )Nz1.1.0)r   r3   r  _dataZ_mgrrS  r   r   r$   _get_mgr?  s    r  c                 C   s&   | j dd}tt|j|ks"td S )NTr  )rI   r   r  blocksrK   )r   numberr#   r   r   r$   r  F  s    r  c                     s:   d  fdd} |  }t |ddd |  }t |dd d S )N2   c                      s(   t dd t D dd t D S )Nc                 S   s(   g | ] }t tjd ddd qS )r  Nr&   )rE   r^   r5   r7   r8   ro  r   r   r$   r%   O  s   zEtest_to_pandas_self_destruct.<locals>._make_table.<locals>.<listcomp>c                 S   s   g | ]}d  |qS r  r  ro  r   r   r$   r%   S  s     )rE   rW   r9   r   rr  r   r$   _make_tableN  s
    
z1test_to_pandas_self_destruct.<locals>._make_tableT)r  self_destruct)r  )r  )r  r   r   r  r$   test_to_pandas_self_destructK  s    r  c                  C   s~   d} t tj| tjd}t |||gdddg}t  }| }t  |d|  d  ks^td }t	
  t  |ksztd S )Nr  r   r  r  rE  rs   r   )rE   r^   r5   r6   r/   rW   r  rI   rK   r  r  )rq  rd   r   r  r#   r   r   r$   test_table_uses_memory_pool]  s    r  c                  C   sl   t ddig} |  }|jtdks,t|d }t|}t||ksPt~t||d kshtd S )Nr   r   r   r   )rE   r^   rI   r   r5   rK   sysgetrefcount)rd   r}  r  refcountr   r   r$   test_object_leak_in_numpy_arraym  s    
r  c                  C   s   t ddig} t | gdg}| d }|jtdks@t|d }t|}t||ksdt~t||d ks|td S )Nr   r   r  r   r   )	rE   r^   rW   rI   r   r5   rK   r  r  )rd   rW   r~  r  r  r   r   r$   test_object_leak_in_dataframey  s    
r  c                  C   s   ddgddgg} t  }t j| d |d}t j| t |d}tj| d tjd}ttj| d tjdtj| d tjdg}|j|kst|	t |st|	t |std S )	Nr  g333333@g      "@g      E@r   r  r   r   )
rE   r0   r^   r  r5   r3   r_   r[   rK   rJ   )r   r   r  r  rv  rw  r   r   r$   test_array_from_py_float32  s    r  c            
   	   C   s  t  } t| g}|jdjd}tjddd}tj	j
||d}|j|ksTttj	j
|tdd}|d  |d  kst|d  |d  ksttjdgddtd	}tjd
gddtd}|jtd	ksttd}tt || W 5 Q R X |j|dd}||s6tttdtdtdg}	tjdddgtdd}tt tj|	tdd W 5 Q R X tt tj	j
|	tdd W 5 Q R X tj	j
|	tddd}||sttj|	tddd}||std S )Nz	tzlocal()r   r  r;  r  r   i r/   r  {   ra   Fr   r   rP  r  r  )r   nowr3   r_   r=  r>  Z
tz_convertrE   r<  r  rH   r[   rK   rg  to_pydatetimer^   r  r   r   r   rJ   r5  )
r  ra   Zs_nycZ
us_with_tzrd   r  rS   targetrX   r  r   r   r$   test_cast_timestamp_unit  s6    
" r  c                  C   sf   t j } |  }tj|gtjdddd}tj||gddg}|	 }t|}|
|sbtd S )Nr  r   r;  r  r   stop)r3   r5  r  r  rE   r^   r<  r  r   rI   rJ   rK   )tsts_dtrd   r  rX   r   r   r   r$   (test_nested_with_timestamp_tz_round_trip  s    

r  c            
      C   s  t j } |  }dD ]v}|dkr.dd }ndd }tj| gt|d}tj| gtj|ddd}tj||gd	d
g}tj||gd	d
g}|	 }t
|d d	 tst|d d	 jd kstt
|d d
 tst|d d
 jd kst|	 }t
|d d	 tst|d d	 jd k	s*t|d d	 tj}	||	jd d||ks\tt
|d d
 tstt|d d
 jd k	sttd|i	 }t
|d d d	 tst|d d d	 jd kstt
|d d d
 tst|d d d
 jd ksttd|i	 }t
|d d d	 ts<t|d d d	 jd k	sXtt
|d d d
 tstt|d d d
 jd k	stqd S )Nr+  )ra   r  c                 S   s   | j ddS )Nr   r  r!  r#   r   r   r$   truncate  rg  z/test_nested_with_timestamp_tz.<locals>.truncatec                 S   s   | S r  r   r  r   r   r$   r    rg  r  r   r;  r   r  r   r  r   )r3   r5  r  r  rE   r^   r<  r  r   rI   r   r   rK   r  
astimezoner   r  r  rW   )
r  r  r  r  rd   r  Zarr3Zarr4rX   Zutc_dtr   r   r$   test_nested_with_timestamp_tz  s@    


r  c               
   C   sB  t dddgd} t jdddgtd}t ddd	dddg}d
D ]}| |}tj||}tjj|||d}|d dkrt	t
 |  W 5 Q R X q@| }tjj||d}tt|t| | }	|	  dkst|}
|d dkr
||dd  }
tjjt |d|
|d}tt|	t| q@d S )Nr   r   r&   r   r  r  r   FT)r(   r,   r)   r-   r*   r.   r+   r/   r  ury  r   )r5   r  r^   r   r   rE   r   r   r   r   r  rI   r3   rt  
from_codesrP   r`   r_   rh   rj   rK   where)Zsrc_indicesri  rf   
index_typer   d1d2Zpandas1Z
ex_pandas1Zpandas2Zsigned_indicesZ
ex_pandas2r   r   r$   test_dictionary_with_pandas  s2    
r  c                    sd   |d k	r$|t jjdt|| d }n t j fddt| D td}|dkr`d |t j| |k < |S )Nr   r:   c                    s   g | ]}t  qS r   rn  ro  	item_sizer   r$   r%   7  s     z"random_strings.<locals>.<listcomp>r   )r5   r7   rQ  r   r^   r9   r   rand)r  r  Zpct_nullri  rX   r   r  r$   random_strings3  s    r  c                  C   s  t jd tjtdddd} tjtdddd}tjtdddd}tjt jjd	t	| d
dd| }tjt jjd	t	|d
dd|}tjt jjd	t	|d
dd|}tjt jjd	t	|d
ddt j
d
dk d}tj||}t| ||g}t||||g}	t|d|d|d|dg}
|	 }|
 }|jj| k s`t|d}d || < t|| d S )Ni90  r       r0  r  r  r  rP  r   r  rk  r  r"  r  r    )r5   r7   r  rE   r^   r  r   r   rQ  r   r  Zconcat_arraysr  r  rI   rg  rz  r&  rK   r   rh   rP   r`   )r  r  Zd3r  r  r  rk  r  expected_dictr   Za_denserX   Zresult_denseZexpected_denser   r   r$   "test_variable_dictionary_to_pandas@  sD     
r  c                  C   sp   t dddddg } t jdddg| }| }ttjdddgt	dtjddgt	dg}t
|| d S )Nr   r   r   rs   r   r   )rE   r^   r  r  r   rI   r3   r_   r5   r   rP   r`   )r  rd   rX   rS   r   r   r$   (test_dictionary_encoded_nested_to_pandasl  s    r  c                  C   s   t dddg} tt t }t| }| dddgksDt|j	
|sTtt ddd dg} t| }| ddd dgkst|j	
|sttj| tddddgd}| ddd d gkst|j	
|std S )Nr   r   FTr  )r3   rt  rE   ri  r,   r0  r^   r]  rK   r[   rJ   r5   )rg  r  rX   r   r   r$   test_dictionary_from_pandasy  s    

r  c               	   C   sZ  t jtjddgddtjddgtd} tjt t	 d}tj| |d}|j
|s^t| ddgksrttjt t d}ttj tj| |d}W 5 Q R X tjt t	 d	d
}d}tjt|d tj| |d}W 5 Q R X | ddgksttjt t	 d}tj| |tdd	gd}|j
|sJt| dd gks`tt g } tjt t	 d}tj| |d}|j
|st| g ksttjt t d}tj| |d}|j
|st| g kstt ddg} tj| t	 d}tjddgt	 d}||s@t| ddgksVtd S )Nr   r   r,   r   r   r   )r  
value_typer  T)r  r  r   z4The 'ordered' flag of the passed categorical values r  Fr  )r3   rt  r  r5   r^   r   rE   ri  r-   r0  r[   rJ   rK   r]  r,   r/   r   r   rd  r   )rg  typrX   r-  rS   r   r   r$   *test_dictionary_from_pandas_specified_type  sL       
r  c                  C   s   t tjt dk rtd tjddddgidd} | d} t| d j	j
}t| d j	jj}t|tjstttj| d }tj||}||kstd S )	Nz1.3.0z:PyArrow backed string data type introduced in pandas 1.3.0r#   r   r  zstring[pyarrow]r   r   )r   r3   r  r   r  r4   r   rE   r^   rg  codesrz  rl   r   r  rK   rH   r   r   )rR   r   ri  rS   rX   r   r   r$   :test_convert_categories_to_array_with_string_pyarrow_dtype  s    

r   c                  C   s2  t dt jddd gddi} t| }tddd gt }|d d|sVt	t
dt fg}tj| |d}tddd gt }|d d|st	t| d }||st	tj| d t d}||st	t| d j}||st	tj| d jt d}||s.t	d S )	Nr   r   r&   Int64r   r   r  r  )r3   r4   r_   rE   rW   r^   r/   r  rJ   rK   r=   r1   rl   )rR   rX   rS   r=   rw  r   r   r$   test_array_protocol  s     
r  c                   @   s   e Zd Zdd Zdd ZdS )DummyExtensionTypec                 C   s   t j| t   d S r  rE   PyExtensionType__init__r/   r  r   r   r$   r    s    zDummyExtensionType.__init__c                 C   s   t dfS Nr   )r  r  r   r   r$   
__reduce__  s    zDummyExtensionType.__reduce__N)r@  rA  rB  r  r  r   r   r   r$   r    s   r  c                 C   s(   t jdddgt  d}t jt |S )Nr   r&   rs   r  )rE   r^   r/   ExtensionArrayfrom_storager  )rw   r[   storager   r   r$   PandasArray__arrow_array__  s    r  c                 C   s   t jdddgt  d}t jt |}| jtjj	dt
dd | jtjjdt
dd tjddd	d
jtddjfD ]}t |}||stt t|}||stt t|}||stt td|idd}||sttqtd S )Nr   r&   rs   r  Z__arrow_array__F)Zraising
2012-01-01Dr  r   r   r   )rE   r^   r/   r	  r
  r  setattrr3   r2  ZPeriodArrayr  ZIntervalArrayr  r  rJ   rK   r_   r   rW   r4   r,  r  )monkeypatchr  rS   rd   rX   r   r   r$   *test_array_protocol_pandas_extension_types  s*      
 r  c                 C   s   t |tjr|}n
|d}| }tj|d dd|j|jt|  }|d }|d k	rtj	
t t|d |g}t|}ntjt|td}tjj| | dd}|S )Nr   r   r/   r   F)rk   )r   rE   r  r  buffersr5   Z
frombufferoffsetr   ZBooleanArrayZfrom_buffersrb  r7  r  r2   r3   r2  ZIntegerArrayrk   )rw   r^   rd   Zbuflistr   Zbitmaskrf   Zint_arrr   r   r$   _Int64Dtype__from_arrow__  s&    
   r  c                 C   s  dd l m  m} tdddgtjdddgddddd	gd
}t|}| }t	t
|jd |jrltt
|jd jjtdkstt	t
|jd |jstt|| tdtjddd gddi}t|}| }t	t
|jd |jstt|| ttjtdk r2| tjjjjd n| tjjjjd | }tt
|jdksftt	t
|jd |jrtd S )Nr   r   r&   rs   r   r  r   r   r   r   r   r   r/   r   	1.3.0.dev__from_arrow__)pandas.core.internalscore	internalsr3   r4   r^   rE   rW   rI   r   r  r  ExtensionBlockrK   rl   r   r5   rP   rQ   r   r  delattrr2  integer_IntegerDtypeNumericDtyper   )r  _intrR   rW   rX   r   r  r   r   r$   test_convert_to_extension_array4  s:    
 

 
 r!  c                   @   s$   e Zd Zdd Zdd Zdd ZdS )MyCustomIntegerTypec                 C   s   t j| t   d S r  r  r  r   r   r$   r  \  s    zMyCustomIntegerType.__init__c                 C   s   t dfS r  )r"  r  r   r   r$   r  _  s    zMyCustomIntegerType.__reduce__c                 C   s   t  S r  )r3   
Int64Dtyper  r   r   r$   r  b  s    z#MyCustomIntegerType.to_pandas_dtypeN)r@  rA  rB  r  r  r  r   r   r   r$   r"  Z  s   r"  c              	   C   s|  dd l m  m} tddddgt }tjt |}t	d|i}|
 }tt|jd |jsjttjddddgdd}t|| |
 }tt|jd |jsttdtjddddgddi}t|| ttjtd	k r| tjjjjd
 n| tjjjjd
 |
 }tt|jd |jr<ttddddg}t|| tt |
  W 5 Q R X d S )Nr   r   r&   rs   r   r   r  r   r  r  ) r  r  r  rE   r^   r/   r	  r
  r"  rW   rI   r   r  r  r  rK   r3   r_   rP   r`   r4   rQ   r   r  r  r2  r  r  r  r   r   r   )r  r   r  rd   rW   rX   rS   r   r   r$   /test_conversion_extensiontype_to_extensionarrayf  s6     
 
 r$  c               	   C   s>  t dt dddgt  i} |  }|d jtdks@t| jt  t	 ij
d}t|d jtj	sptt dt dddgt  i} | jt  t	 ij
d}|d jtdkstt tdtjdddgdd	i} |  }t|d jtj	st| jt  td
ij
d}t|d jtjs:td S )Nr   r   r&   rs   r/   types_mapperr.   r  r   r  )rE   rW   r^   r/   rI   r   r5   rK   r3   r#  r  r   r.   r4   ZPeriodDtype)rW   rX   r   r   r$   'test_to_pandas_extension_dtypes_mapping  s      $r'  c                  C   s   t tjt dk rtd tjddddjtddjfD ]X} t| 	 }t
| }t|| td	| id	 	 }tj
| d	d
}t|| q<d S )Nz1.1z'ExtensionDtype to_pandas method missingr  rs   r  r  r   r   r~  r\   )r   r3   r  r   r  r  r^   r  rE   rI   r_   rP   r`   rW   )rd   rX   rS   r   r   r$   test_array_to_pandas  s    

r(  c                  C   sT   t jt jdddd} t| }| jt jdddddddddgdd	k d S )
Nr   rs   )r   endr   r   )r  r  r&   r   r   )r3   r4   r  rE   rW   rI   r   r   )rR   rW   r   r   r$   5test_roundtrip_empty_table_with_extension_dtype_index  s    
r*  r   r   r   c                 C   s~   t tjt dk rtd tjddgddgddgd	tt d
	| }|
 }t|}|jtjd}t|| d S )Nz1.5.0zArrowDtype missingr   r&   rs   r   r   r   r  r   r%  )r   r3   r  r   r  r4   Z
ArrowDtyperE   r/   r   rk   rW   rI   rP   rQ   )r   rR   rS   rW   rX   r   r   r$   !test_to_pandas_types_mapper_index  s    

r+  c                  C   s   t tjt dk rtd tdddgt } t t ij	}| j
|d}|jt ksbtt d ij	}| j
|d}|jtdkstt t ij	}| j
|d}|jtdkstd S Nz1.2.0z$Float64Dtype extension dtype missingr   r&   rs   r%  r/   )r   r3   r  r   r  rE   r^   r/   r#  r  rI   r   rK   r5   r1   Float64Dtyper   r&  rX   r   r   r$   !test_array_to_pandas_types_mapper  s    
r/  c                  C   s   t tjt dk rtd ttdddgt g} t	| tj
sJtt t ij}| j|d}|jt ksztt d ij}| j|d}|jtdkstt t ij}| j|d}|jtdkstd S r,  )r   r3   r  r   r  rE   r  r^   r/   r   rY  rK   r#  r  rI   r   r5   r1   r-  r.  r   r   r$   )test_chunked_array_to_pandas_types_mapper  s    
r0  c                  C   s  ddd dg} ddddg}t j| dd}t j|dd}t jdd	d
dgdd}d}d}tjd| itjddd	ddd}t jj||gddgd}|dt	dgd d ddddidgddddd dddddd dgddi}|
 }	t|	| tjd| itjddd	ddd}
t jj||gd|gd}|dt	|gd d ddddidgddddd dd|ddd dgddi}|
 }t||
 tjd| itjddd	d dd}t jj||gd|gd}|dt	|gd d ddddidgddddd dd |ddd dgddi}|
 }t|| tjd| itjddd	dd|gd}t jj|||gdd|gd}|dt	d|gd d ddddidgddddd dddddd dd |ddd dgddi}|
 }t|| tjd| itjddd	d d|gd}t jj|||gd||gd}|dt	||gd d ddddidgddddd dd |ddd dd |ddd dgddi}|
 }t|| d S )Nr   r  r  r   r   r  r  r   r&   r   r   r/   r   Z__index_level_1__r   rO  r   r   r      pandasr   r   r   r   )r]   r   r   r   r   z0.23.4)r   r   r   r6  )rE   r^   r3   r4   r   rG   r   replace_schema_metadatajsondumpsrI   rP   rQ   )a_valuesb_valuesa_arrowb_arrowZrng_index_arrowZ
gen_name_0Z
gen_name_1e1r  Zr1e2r  Zr2Ze3t3Zr3Ze4Zt4Zr4Ze5Zt5Zr5r   r   r$   )test_metadata_compat_range_index_pre_0_12  s     
  
  
    r<  c                  C   s   ddddg} ddddg}t j| d	d
}t j|dd
}tj| |dtjdddddd}t ||d}|dtd d d dddgd dd	d	dd ddddgddddddgddi}|	 }t
|| d S )Nr   r&   rs   r   r   r   r   r  r/   r  r  r   r   r   rO  r   r   r1  r   zmixed-integer)r   r   r]   r   r   )r   r]   r   r   r   r9   )kindr]   r   r   r  z0.25.0)r   r   r   r6  )rE   r^   r3   r4   r   rW   r2  r3  r4  rI   rP   rQ   )r5  r6  r7  r8  rS   rW   rX   r   r   r$   'test_metadata_compat_missing_field_name  sX     r>  c                  C   sL   t d} ttjtjdd| dd}|jj}|d d d dksHt	d S )	Nr   r   r   r\   r   r   r]   6)
r5   r/   rE   rW   r3   r4   r   r=   r   rK   r]   rW   r   r   r   r$   .test_metadata_index_name_not_json_serializable  s    
rA  c                  C   sF   d} t tjtjdd| dd}|jj}|d d d dksBtd S )Nr   r   r   r\   r   r   r]   )rE   rW   r3   r4   r   r=   r   rK   r@  r   r   r$   -test_metadata_index_name_is_json_serializable  s    rB  c               
   C   s   t tddtddtddtddgtddtddtddtddgd	} | jj| jjftd
tdfkst| S )Nz0001-01-01 00:00r  z2012-05-02 12:35z2012-05-03 15:42z3000-05-03 15:42z1991-01-01 00:00r9  z2050-05-03 15:42)
dateTimeMs
dateTimeNsr  zM8[ns])r3   r4   r5   r  rC  r   rD  rK   rS  r   r   r$   make_df_with_timestamps  s"    







 
rE  z)ignore:Parquet format '2.0':FutureWarningc                 C   sP   t  }tj|}| d }tj||dd t|}|jdd}t	|| d S )Nztimestamps_from_pandas.parquetz2.0)versionTZtimestamp_as_object)
rE  rE   rG   rH   pqZwrite_tableZ
read_tablerI   rP   rQ   )tempdirrR   rW   filenamerX   r   r   r   r$    test_timestamp_as_object_parquet  s    
rK  c                  C   s.   t  } tj| }|jdd}t| | d S )NTrG  )rE  rE   rG   rH   rI   rP   rQ   )rR   rW   r   r   r   r$   %test_timestamp_as_object_out_of_range  s    rL  
resolutionr   r   r=  i  i  c                 C   s   t j|gt j| |dd}t d|i}|jdd|jddd fD ]r}|jtksVtt|d t	sht|r|d j
d k	s~t|d j
|}n|d j
d kst|}|d |ksDtqDd S )Nr;  r  r   TrG  r   )rE   r^   r<  rW   rI   r   r   rK   r   r   r  fromutc)rM  r   r=  rd   rW   rX   rS   r   r   r$   'test_timestamp_as_object_non_nanosecond  s    
rO  c               	   C   sl   t d} dd l}| d}||ddddd}td	t|gi}|jd
d}t||ksht	d S )Nr  r   x   i  r   r{  r  9   Ztimestamp_colTrG  )
r   r  r   r  ZlocalizerE   rW   r^   rI   rK   )r  r   r   r=  rW   rX   r   r   r$   %test_timestamp_as_object_fixed_offset7  s    

rR  c                   C   s   t d d S )Nzpandas_threaded_import.py)r
   r   r   r   r$   test_threaded_pandas_importC  s    rS  c                  C   s^   t d} t t j| ksttdt  gi}tj|}|	  t t j| ksZtd S )NZm8r   )
r5   r   r7  rK   r3   r4   rE   rG   rH   rI   )rS   rR   r   r   r   r$   $test_does_not_mutate_timedelta_dtypeG  s    
rT  c                  C   sf   ddl m}  d| dddig}| ddd	d
g}t||d}| }|d d  |d ksbtd S )Nr   )r   timedelta_1r{  r   )r  rA  rs   (   r  )r  r  r  )rU  timedelta_2rW  )r   r   rE   rW   rI   Zto_pytimedeltarK   )r   rU  rW  rW   rR   r   r   r$   %test_does_not_mutate_timedelta_nestedS  s    rX  c                  C   sT  t t dt t t  t t dt  gg} t jdggd dddifgddd ifdddifgggg| d	}t	
ddggd dddifgddd ifdddifgggi}t	
dd
d igd dddiidd iddidggi}| }|jdd}t|| t|| t jj|| d	}||s0tt jj|| d	}||sPtd S )Nr   r   )r   Nr&   s   abcrs   r   s   defr  r   r6  strictZmaps_as_pydicts)rE   r=   ru   r  r  r,   r  rS  rW   r3   r4   rI   rP   rQ   rG   rH   rJ   rK   )r=   rW   Zexpected_default_dfZexpected_as_pydicts_dfZ
default_dfZas_pydicts_dfZtable_default_roundtripZtable_as_pydicts_roundtripr   r   r$   ,test_roundtrip_nested_map_table_with_pydicts`  sJ    "r[  c               	      s  t ddg} t ddddg}t g t  }t ddgd	d
ggt t  }t g dddgddggt t  }t g t t  }t jddg| |}t jdddg||}t jddg||}t t jddg|dt jddg|ddt jddg|dg}	|		 }
t
dddgfdd	d
gfggdddgfggg g}|	j	dd}t
ddgd	d
gdgdddgigg g}|	dd}|	 }t
dddgfggg g}|j	dd}t
dddgigg g}t D tddt t|
| t|| t|| t|| W 5 Q R X t t t  t t   t
jdd fdd}||
|	 |||	 ||| ||| dS )z=
    Slightly more robust test with chunking and slicing
    r   r  r  rO  r  r  r   r   r   r  Nr  r  r  r   r&   rs   r   r   rY  rZ  r   r  rA   rB   )r  returnc                    s(   t t jj|  d}||s$td S r  )rE   r  r  rH   rJ   rK   )r  r   Zarray_roundtripr  r   r$   assert_roundtrip  s    zMtest_roundtrip_nested_map_array_with_pydicts_sliced.<locals>.assert_roundtrip)rE   r^   r0  r  r  r   r  r  r  rI   r3   r_   rL   rM   rN   rO   rP   r`   r  )Zkeys_1Zkeys_2Zkeys_3Zitems_1Zitems_2Zitems_3Zmap_chunk_1Zmap_chunk_2Zmap_chunk_3r  series_defaultexpected_series_defaultseries_pydictsexpected_series_pydictsZslicedZseries_default_slicedZexpected_series_default_slicedZseries_pydicts_slicedZexpected_series_pydicts_slicedr_  r   r^  r$   3test_roundtrip_nested_map_array_with_pydicts_sliced  sx    
 


rd  c               	   C   s:  t dddg} t ddgddgddggt t  }d	d
g}t j|| |}t t  t t  }tt j	j
 |jdd W 5 Q R X |jdd}tddgddgdg}|t jj||drt| }tdddgfdddgfdddgfgg}|t jj||dstt|t|ks4tt||D ]n\}	}
t|	t|
ks\tt|	 |
 D ]:\}}|d	 |d	 kstt|d |d sntqnq>t|t|kstt||D ]f\}	}
t|	t|
kstt|	|
D ]:\}}|d	 |d	 kstt|d |d stqqd S )Nr   r  r   r   r   r  r   r   r   rs   rY  rZ  lossyr\  r  r   )rE   r^   r  r0  r  r   r  r   r   r  ZArrowExceptionrI   r3   r_   rJ   r  rH   rK   r   rT  r  r5   array_equal)r1  r  r  mapsr  rb  rc  r`  ra  row1row2tup1tup2r   r   r$   4test_roundtrip_map_array_with_pydicts_duplicate_keys  s@     "rl  c            
   	   C   sZ  t ddgddgg dgd dgddggt t  } t d	d
ddddg}dddg}t j|| |}tt |j	dd W 5 Q R X |	 }t
ddgd	fddgd
fgg dfdgdfd dgdfddgdfgg}t|t|kstt||D ]d\}}t|t|kstt||D ]:\}}	t|d |	d s:t|d |	d kstqqd S )Nr   r   r   r  r  r  r  rE  r   r  r  rO  r  r  r   r&   r   re  rZ  r   )rE   r^   r  r0  r  r   r   r   r  rI   r3   r_   r   rK   rT  r5   rf  )
r1  r  r  rg  r  ra  rh  ri  rj  rk  r   r   r$   %test_unhashable_map_keys_with_pydicts  s(     
$rm  c                  C   s^   t jt jdddddd} tdt| i}|d}| }|jdksNt	t
||  d S )N2012r&   r   r!  Zdatetime_columnr\   )r3   r_   r   rE   rW   r^   r,  rI   r]   rK   rP   r`   )r  rW   Z	table_colrX   r   r   r$   )test_table_column_conversion_for_datetime/  s    
ro  c                  C   s8   t t jdddd} t| }| }t||  d S )Nrn  r&   r   r!  )r3   r_   r   rE   r^   rI   rP   r`   )r  rd   rX   r   r   r$   "test_array_conversion_for_datetime>  s    
rp  )r   )NFNTNFF)NN)NNN)N)r   N)N)r  r  r3  multiprocessingr]  r  rL   collectionsr   r   r   r   r   r   Z
hypothesisrE  Zhypothesis.strategiesZ
strategiesrG  Znumpyr5   Znumpy.testingr  ro   r   Zpyarrow.pandas_compatr   r	   Zpyarrow.tests.utilr
   r   r   Zpyarrow.tests.strategiestestsrI  Zpyarrow.vendored.versionr   ZpyarrowrE   r   rH  ImportErrorZpandasr3   Zpandas.testingrP   Zpandas_examplesr   r   rC  Z
pytestmarkr<   rY   re   rm   rq   rr   rD  rD  r  r  r  rK  r  r  r  r<  rT  rU  rV  r  r  r  r  r  r  rF  r2  Zpandas_compatible_typesr  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r	  rb  r,   r-   r.   r/   r(   r)   r*   r+   rI  r0   r1   r  r  rS  r0  r  r  r<  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r   r  r  r  r  r  r  r!  r"  r$  r'  r(  r*  r+  r/  r0  r<  r>  rA  rB  rE  rN   rK  rL  rO  rR  rS  rT  rX  r[  rd  rl  rm  ro  rp  r   r   r   r$   <module>   s        
 
   w  '

      X[    " a<  	



Z%

















14$,5	&&	

 *3
	 -M4
!