U
    f/eH                     @   s  d Z ddlZddlZddlZddlmZ ddlZddlZddlZddlZddl	m
Z
 ddlZddlZddlmZmZmZ ddlZddlZddlZddlmZmZmZmZ ddlm  mZ ddlZddlm Z m!Z!m"Z" ddl#m$Z% ddl&m'Z'm(Z( e Z)ej*ej+d	gZ,ej-d
ddd Z.dzddZ/dd Z0dd Z1dd Z2dd Z3dd Z4dd Z5dd Z6dd Z7eej89ej8:e;d d!d"d#Z<ej-e<d$d%d! Z=d&d' Z>d(d) Z?d*d+ Z@ej+Ad,ejBe?d-d.ejBejCd/d.ejBejejCejDd0d1d.ejBejejCd2d0d3d.ejBejejCd4d0d5ej+jEe d6d7d8gej+d9d:d; ZFd<d= ZGd>d? ZHd@dA ZIej-dBdC ZJG dDdE dEZKG dFdG dGZLej+AdHdIgdJe dKdLdMgfdNejMNdKdLdMgdOdPdQggfgdRdS ZOdTdU ZPej+AdVdWdXdYgdZd[ ZQeRd\d]d^ ZSG d_d` d`ejTZUdadb ZVdcdd ZWdedf ZXdgdh ZYej+Adidjdkdiedldmdmdngdodp ZZdqdr Z[dsdt Z\dudv Z]ej+Adwej^ejDgdxdy Z_dS ){a<  
manage legacy pickle tests

How to add pickle tests:

1. Install pandas version intended to output the pickle.

2. Execute "generate_legacy_storage_files.py" to create the pickle.
$ python generate_legacy_storage_files.py <output_dir> pickle

3. Move the created pickle to "data/legacy_pickle/<version>" directory.
    N)partial)Path)catch_warningsfilterwarningssimplefilter)PY38get_lzma_fileimport_lzmais_platform_little_endian)IndexSeriesperiod_range)DayMonthEndz1ignore:Timestamp.freq is deprecated:FutureWarningmodule)Zscopec               
   C   s>   ddl m}  t " tddtd |  W  5 Q R  S Q R X d S )Nr   create_pickle_dataignorez The 'freq' argument in Timestamp)category)Z-pandas.tests.io.generate_legacy_storage_filesr   r   r   FutureWarningr    r   ?/tmp/pip-unpacked-wheel-tiezk1ph/pandas/tests/io/test_pickle.pycurrent_pickle_dataA   s      r   c                 C   s   t |trt||  d S |dr6tj}|| | n`|dkrv|tjkrX| tjksttq| |ksdt| j	|j	kstn t
td| dtj}|| | d S )NZsp_	timestampZassert_Z_equal)
isinstancer   tmassert_index_equal
startswithZassert_equalpdZNaTAssertionErrorfreqgetattrZassert_almost_equal)resultexpectedtypversion
comparatorr   r   r   compare_elementQ   s    


r'   c           
      C   sr   t |} t }|  D ]T\}}| D ]B\}}| | | }d| d| }	||	|d }	|	|||| q(q| S )NZcompare__r'   )r   read_pickleglobalsitemsget)
dataZvfr%   mr$   dvdtr"   r#   r&   r   r   r   compared   s    
r1   c                 C   s   t | | | jj|jjks t| jjjr.tt | dk|dk | jj}|td tdksbt|tjdd }t	|tjst|tjdddkst|tjdd }t	|tjst|tjdddkstd S )Nr         )hours)daysr4   )nanoseconds)r5   r6   )
r   assert_series_equalindexr    r   	normalizer   r   Z	Timedeltar   )r"   r#   r$   r%   r    resr   r   r   compare_series_tsv   s    r;   c                 C   s   t | | d S Nr   r7   r"   r#   r$   r%   r   r   r   compare_series_dt_tz   s    r?   c                 C   s   t | | d S r<   r=   r>   r   r   r   compare_series_cat   s    r@   c                 C   s   t | | d S r<   r   assert_frame_equalr>   r   r   r   compare_frame_dt_mixed_tzs   s    rC   c                 C   s   t | | d S r<   rA   r>   r   r   r   compare_frame_cat_onecol   s    rD   c                 C   s   t | ||| d S r<   )rD   r>   r   r   r   compare_frame_cat_and_float   s    rE   c                 C   sV   t | | t| jtst| jt ks,t| jdks:tt | d|d d S )NMr3   )r   r   r   r    r   r   Zfreqstrshiftr>   r   r   r   compare_index_period   s
    rH   r-   legacy_pickle*z*.pickle)paramsc                 C   s
   || j S r<   )param)requestdatapathr   r   r   rI      s    c              	   C   sR   t  std tjtj|}tdd td t	| || W 5 Q R X d S )Nz"known failure on non-little endianT)recordr   )
r
   pytestskipospathbasenamedirnamer   r   r1   )r   rI   r%   r   r   r   test_pickles   s    
rV   c              	   C   s*   t |d}tj| |dd W 5 Q R X d S NwbprotocolopenpickledumpobjrS   fhr   r   r   python_pickler   s    rc   c              
   C   s6   t | d"}|d t|W  5 Q R  S Q R X d S )Nrbr   )r]   seekr^   load)rS   rb   r   r   r   python_unpickler   s    
rg   pickle_writerpython)idZpandas_proto_defaultrZ   Zpandas_proto_highest   Zpandas_proto_4   Zpandas_proto_5zprotocol 5 not supported)reason)rj   Zmarksz5ignore:The 'freq' argument in Timestamp:FutureWarningc                 C   s   | }|  D ]\}}|  D ]\}}tjtfD ]}t }||| t|}	t|	|| t|}	t|	|| t	|dd}
||| |

d W 5 Q R X t	|dd}
t|
}	|

d W 5 Q R X t|	|| W 5 Q R X q.qqd S )NrX   )moder   rd   )r+   r   	to_picklerc   r   ensure_cleanr)   r'   rg   r]   re   )r   rh   r-   r$   r/   r0   r#   writerrS   r"   handler   r   r   test_round_trip_current   s"    




rs   c                  C   s(   t  } t | jtj}t | | d S r<   )r   makeDataFrameround_trip_pathlibro   r   r)   rB   dfr"   r   r   r   test_pickle_path_pathlib   s    rx   c                  C   s(   t  } t | jtj}t | | d S r<   )r   rt   Zround_trip_localpathro   r   r)   rB   rv   r   r   r   test_pickle_path_localpath   s    ry   c              	   C   st   t t( tdt tj| dddddd W 5 Q R X t t( tdt tj| dddddd W 5 Q R X d	S )
ap  

    Generated with

    >>> df = pd.DataFrame({"A": [1, 2, 3, 4], "B": [0, 0, 1, 1]}).to_sparse()
    >>> df.to_pickle("pandas/tests/io/data/pickle/sparseframe-0.20.3.pickle.gz",
    ...              compression="gzip")

    >>> s = df['B']
    >>> s.to_pickle("pandas/tests/io/data/pickle/sparseseries-0.20.3.pickle.gz",
    ...             compression="gzip")
    r   ior-   r^   zsparseseries-0.20.3.pickle.gzgzipcompressionzsparseframe-0.20.3.pickle.gzN)r   Zassert_produces_warningr   r   DeprecationWarningr   r)   )rN   r   r   r   test_legacy_sparse_warning  s    

r   c                   C   s   dt d dS )N__
   z	__.pickle)r   Zrandsr   r   r   r   get_random_path"  s    r   c                	   @   s   e Zd ZddddddZdd Zd	d
 Zejdddddgdd Z	ejddddddgdd Z
dd Zejdddddddgdd ZdS )TestCompressionz.nonez.gzz.bz2z.zipz.xz)Nr{   bz2zipxzc              
   C   s   |d krt || d S |dkr.t|d}n||dkrDt|d}nf|dkrtj|dtjd}|	|t
j| W 5 Q R X n*|dkrtt|d}nd| }t||dkrt|d$}| |	|  W 5 Q R X W 5 Q R X d S )	Nr{   wr   r   r|   r   zUnrecognized compression type: rd   )shutilcopyfiler{   r]   r   BZ2FilezipfileZipFileZIP_DEFLATEDwriterR   rS   rT   r   lzma
ValueErrorread)selfZsrc_path	dest_pathr}   fmsgrb   r   r   r   compress_file1  s"     
zTestCompression.compress_filec                 C   s   |}|d }|d }t |}t |t}t  }|j||d t j||d*}	t|d}
|
|	  W 5 Q R X W 5 Q R X tj	|d d}t 
|| W 5 Q R X W 5 Q R X d S )N.compressed.rawr|   rX   )r   rp   rt   ro   decompress_filer]   r   r   r   r)   rB   )r   r}   r   basepath1path2p1p2rw   r   rb   df2r   r   r   test_write_explicitG  s    "z#TestCompression.test_write_explicitr}    NonebadZ7zc              
   C   sJ   t jtdd2 t|}t }|j||d W 5 Q R X W 5 Q R X d S )NzUnrecognized compression type)matchr|   )rP   Zraisesr   r   rp   rt   ro   )r   r}   r   rS   rw   r   r   r   test_write_explicit_bad\  s    z'TestCompression.test_write_explicit_badextz.no_compressc                 C   s   |}|| }|d }d }| j D ]}| j | |kr|} q:qt|}t|p}	t }
|
| tj||d*}t|	d}||  W 5 Q R X W 5 Q R X t	j
|	d d}t|
| W 5 Q R X W 5 Q R X d S )Nr   r|   rX   )_compression_to_extensionr   rp   rt   ro   r   r]   r   r   r   r)   rB   )r   r   r   r   r   r   r}   cr   r   rw   r   rb   r   r   r   r   test_write_inferc  s     

"z TestCompression.test_write_inferc           
   
   C   s   |}|d }|d }t |\}t |F}t  }|j|d d | j|||d tj||d}	t ||	 W 5 Q R X W 5 Q R X d S )Nr   r   r|   )r   rp   rt   ro   r   r   r)   rB   )
r   r}   r   r   r   r   r   r   rw   r   r   r   r   test_read_explicit~  s    z"TestCompression.test_read_explicitc              
   C   s   |}|d }|| }d }| j D ]}| j | |kr|} q:qt|X}t|B}	t }
|
j|d d | j||	|d t|	}t|
| W 5 Q R X W 5 Q R X d S )Nr   r|   )	r   r   rp   rt   ro   r   r   r)   rB   )r   r   r   r   r   r   r}   r   r   r   rw   r   r   r   r   test_read_infer  s    

zTestCompression.test_read_inferN)__name__
__module____qualname__r   r   r   rP   markparametrizer   r   r   r   r   r   r   r   r   '  s   

r   c                   @   s*   e Zd Zejdddddgdd ZdS )	TestProtocolr[   rY   r   r2   r3   c              	   C   sF   t |2}t  }|j||d t|}t || W 5 Q R X d S )NrZ   r   rp   rt   ro   r   r)   rB   )r   r[   r   rS   rw   r   r   r   r   	test_read  s
    
zTestProtocol.test_readN)r   r   r   rP   r   r   r   r   r   r   r   r     s   r   pickle_fileexcolsztest_py27.pklabr   ztest_mi_py27.pklABCc                 C   s*   | ddd|}t |}t|j| d S )Nrz   r-   r^   )r   r)   r   r   columns)rN   r   r   rS   rw   r   r   r   test_unicode_decode_error  s    
r   c               
   C   sl   t  Z} t  }t| d}|| W 5 Q R X t| d}t|}W 5 Q R X t || W 5 Q R X d S )NrX   rd   )r   rp   rt   r]   ro   r   r)   rB   )rS   rw   rb   r"   r   r   r   test_pickle_buffer_roundtrip  s    
r   mockurlzhttp://url.comzftp://test.comzhttp://gzip.comc              	      sp   dd }G dd d t  H fdd}t  }|| | d| t|}t || W 5 Q R X d S )Nc              	   S   s*   t |d}tj| |dd W 5 Q R X d S rW   r\   r`   r   r   r   rc     s    z3test_pickle_generalurl_read.<locals>.python_picklerc                   @   s4   e Zd Zdd Zdd Zdd Zdd Zd	d
 ZdS )z5test_pickle_generalurl_read.<locals>.MockReadResponsec                 S   s.   t |d| _d|kr ddi| _n
dd i| _d S )Nrd   r{   zContent-Encoding)r]   fileheaders)r   rS   r   r   r   __init__  s    z>test_pickle_generalurl_read.<locals>.MockReadResponse.__init__c                 S   s   | S r<   r   r   r   r   r   	__enter__  s    z?test_pickle_generalurl_read.<locals>.MockReadResponse.__enter__c                 W   s   |    d S r<   )close)r   argsr   r   r   __exit__  s    z>test_pickle_generalurl_read.<locals>.MockReadResponse.__exit__c                 S   s
   | j  S r<   )r   r   r   r   r   r   r     s    z:test_pickle_generalurl_read.<locals>.MockReadResponse.readc                 S   s
   | j  S r<   )r   r   r   r   r   r   r     s    z;test_pickle_generalurl_read.<locals>.MockReadResponse.closeN)r   r   r   r   r   r   r   r   r   r   r   r   MockReadResponse  s
   r   c                     s    S r<   r   )r   kwargsr   rS   r   r   mock_urlopen_read   s    z6test_pickle_generalurl_read.<locals>.mock_urlopen_readzurllib.request.urlopen)r   rp   rt   setattrr   r)   rB   )Zmonkeypatchr   rc   r   rw   r"   r   r   r   test_pickle_generalurl_read  s    


r   Zfsspecc               	   C   sD   t  2 d} t  }||  t| }t || W 5 Q R X d S )Nzmemory://afiler   )r   rw   r"   r   r   r   test_pickle_fsspec_roundtrip
  s    


r   c                   @   s   e Zd Zdd ZdS )MyTzc                 C   s   d S r<   r   r   r   r   r   r     s    zMyTz.__init__N)r   r   r   r   r   r   r   r   r     s   r   c                  C   sD   t tdt f} t| }t|d | d  t|d ts@td S )N)Zdtyper   r2   )r   objectr   r   round_trip_pickler7   r   r   )r#   r"   r   r   r   test_read_pickle_with_subclass  s    
r   c              	   C   s   t  }t   }|j|| d t| }W 5 Q R X t }|j|| d |d |	 |ksn| dksnt
tj|| d}|d t || dS )ze
    Read/write from binary file-objects w/wo compression.

    GH 26237, GH 29054, and GH 29570
    r|   r   )r{   r   N)r   rt   rp   ro   r   
read_bytesrz   BytesIOre   getvaluer   r   r)   rB   )r}   rw   rS   	referencebufferZread_dfr   r   r   %test_pickle_binary_object_compression"  s    


r   c                 C   s8   | }|}dd }|| ||j  || ||j  d S )Nc                 S   s   t | }t | | d S r<   )r   r   rB   )frame	unpickledr   r   r   _test_roundtripD  s    
zDtest_pickle_dataframe_with_multilevel_index.<locals>._test_roundtrip)T)Z/multiindex_year_month_day_dataframe_random_dataZ multiindex_dataframe_random_dataZymdr   r   r   r   r   +test_pickle_dataframe_with_multilevel_index=  s    
r   c                  C   sB   t dddd} ttjt| | }t|}|jj	dks>t
d S )Nz1/1/2011z1/1/2012rF   )r    )r   r   nprandomZrandnlenr   r   r8   r    r   )ZprngtsZnew_tsr   r   r   "test_pickle_timeseries_periodindexN  s    
r   namei	  g     H@i     )r2   r3   c                 C   s$   t t j| d}|j| ks td S )N)r   )r   r   ZmakeTimeSeriesr   r   )r   r   r   r   r   test_pickle_preserve_nameV  s    r   c                 C   s   t | }t ||  d S r<   r   r   r7   )Zdatetime_seriesZunp_tsr   r   r   test_pickle_datetimes_  s    
r   c                 C   s   t | }t ||  d S r<   r   )Zstring_seriesZ
unp_seriesr   r   r   test_pickle_stringsd  s    
r   c                  C   sf   t tddjdg } t| }|jjd jdks:t	|jjd j
dksPt	t|dg |  d S )Nabcr   r   r2   )r2   T)r   listZastypeZilocr   r   Z_mgrblocksndimr   shaper7   )Zserr:   r   r   r    test_pickle_preserves_block_ndimi  s
    
r   r[   c                 C   s@   t td}tt|j| |dtt j|d}t|| d S )Ni )r[   r}   r|   )	r   Z	DataFrameranger   ru   r   ro   r)   rB   )r[   r}   rw   r"   r   r   r   %test_pickle_big_dataframe_compressionu  s    r   )N)`__doc__r   datetime	functoolsr   globr{   rz   rR   pathlibr   r^   r   warningsr   r   r   r   Znumpyr   rP   Zpandas.compatr   r   r	   r
   Zpandas.util._test_decoratorsutilZ_test_decoratorstdZpandasr   r   r   r   Zpandas._testingZ_testingr   Zpandas.tseries.offsetsr   r   r   Z&skip_array_manager_not_yet_implementedr   Z
pytestmarkZfixturer   r'   r1   r;   r?   r@   rC   rD   rE   rH   rS   joinrU   __file__filesrI   rV   rc   rg   r   rL   ro   HIGHEST_PROTOCOLZskipifrs   rx   ry   r   r   r   r   Z
MultiIndexZfrom_arraysr   r   r   Z
skip_if_nor   tzinfor   r   r   r   r   r   r   r   r   DEFAULT_PROTOCOLr   r   r   r   r   <module>   s   







!
 



 
%
		 
