U
    /eei                  	   @   sP  d dl Z d dlZd dlmZ d dlZd dlZd dlZd dl	m
Z
 d dlZd dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d d	lmZmZmZ d
d Zejjee
dkdddd Z dd Z!dd Z"dd Z#dd Z$ejjee
dkddejj%dd Z&dd Z'ejjee
dkddej(dddd gej(d!d"d#d$gd%d& Z)d'd( Z*d)d* Z+ej(d+ej,d,d-d.d/gd"d0d1d#gd2d3d4d5d6gd7ej-fej.d"d0d1d#gd,d8ej/fgd9d: Z0d;d< Z1ejjee
dkddd=d> Z2d?d@ Z3dAdB Z4dCdD Z5dEdF Z6dGdH Z7dIdJ Z8dKdL Z9dMdN Z:dOdP Z;dQdR Z<dS )S    Nsleep)Version)_PY_VERSION)tm)optimize_dataframe_getitem)	assert_eq)DataFrameIOLayer)dependency_depthtmpdirtmpfilec               	   C   sb  t d tjddddgdddd	gd
ddddgd} t| d}td2}||d t|d}t	
| |d d   W 5 Q R X td6}|j|d t|d}t	| j|d d   W 5 Q R X t| d}td2}||d t|d}t	
| |d d   W 5 Q R X td>}|j|ddd}|  t|d}t	
| |d d   W 5 Q R X d S )Ntablesabcd            xy      ?       @      @      @indexh5/dataFcompute)pytestimportorskippd	DataFrameddfrom_pandasr   to_hdfread_hdfr   assert_frame_equalr   assert_series_equalr"   )dfr   fnoutr r1   D/tmp/pip-unpacked-wheel-dbjnr7gq/dask/dataframe/io/tests/test_hdf.pytest_to_hdf   s0    
 


 

r3   z3.11z@segfaults due to https://github.com/PyTables/PyTables/issues/977)reasonc            
      C   s  t d tjddddgdddd	gd
ddddgd} t| d}tjddddddddddddddddgdddd	ddddd d!d"d#d$d%d&d'gd
ddddd(d)d*d+d,d-d.d/d0d1d2d3gd}t|d'}td4(}||d5 t|d5}t	| | W 5 Q R X td4(}||d5 t|d5}t	|| W 5 Q R X td4t}|j|d6d7d8 d9 t|d6}t	| | t|d:}t
|| jd d  t|d;}t
|| jdd   W 5 Q R X td4>}t|}||d5 W 5 Q R X t|d5}t	|| W 5 Q R X td4v}||d5 t|d5d<g }t|j| d=}d>d? |jD d@ }|j| }	t|	tsdt|	jd<gksvtW 5 Q R X d S )ANr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   efghijklmnop            	   
                           @      @      @       @      "@      $@      &@      (@      *@      ,@      .@      0@r   /data*/data_*c                 S   s   d| d  S Nr   r   r1   r9   r1   r1   r2   <lambda>v       z,test_to_hdf_multiple_nodes.<locals>.<lambda>Zname_functionz/data_az/data_aar   )keysc                 S   s   g | ]}| d r|qS )zread-hdf)
startswith).0keyr1   r1   r2   
<listcomp>   s     
 z.test_to_hdf_multiple_nodes.<locals>.<listcomp>r   )r#   r$   r%   r&   r'   r(   r   r)   r*   r   r   r+   ilocHDFStorer   daskZ__dask_keys__Zlayers
isinstancer	   AssertionErrorcolumns)
r-   r   df16r   r.   r/   hdfZdskreadZsubgraphr1   r1   r2   test_to_hdf_multiple_nodes3   s    
 
")



 


rn   c            
      C   s  t d tjddddgdddd	gd
ddddgd} t| d}tjddddddddddddddddgdddd	ddddd d!d"d#d$d%d&d'gd
ddddd(d)d*d+d,d-d.d/d0d1d2d3gd}t|d'}t 6}tj	|d4}|
|d5 t|d5}t| | W 5 Q R X t 6}tj	|d4}|
|d5 t|d5}t|| W 5 Q R X t ^}tj	|d6}tj	|d7}|
|d5 |
|d5 t||gd5}tt|| g| W 5 Q R X t }tj	|d4}|j
|d5d8d9 d: t|d5}t| | ttj	|d;d5}t|| jd d  ttj	|d<d5}t|| jdd   W 5 Q R X td=>}t|}	|
|	d> W 5 Q R X t|d>}t| | W 5 Q R X d S )?Nr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r5   r6   r7   r8   r9   r:   r;   r<   r=   r>   r?   r@   rA   rB   rC   rD   rE   rF   rG   rH   rI   rJ   rK   rL   rM   rN   rO   rP   rQ   rR   rS   rT   rU   rV   rW   rX   	data_*.h5r    z	data_1.h5z	data_2.h5c                 S   s   d| d  S r[   r1   r\   r1   r1   r2   r]      r^   z,test_to_hdf_multiple_files.<locals>.<lambda>r_   z	data_a.h5z
data_aa.h5r   rY   )r#   r$   r%   r&   r'   r(   r   ospathjoinr)   r*   r   concatr   r+   re   r   rf   )
r-   r   rk   r   dnr.   r/   Zfn1Zfn2rl   r1   r1   r2   test_to_hdf_multiple_files   s    
 
")
 
ru   c               	   C   s   t d tjddddgdddd	gd
ddddgd} t| d}tdB}||d |j|ddd t|d}t	t
| | g| W 5 Q R X t| d}td8}||d |j|ddd t|d}t	| | W 5 Q R X t| d}tdB}||d |j|ddd t|d}t	t
| | g| W 5 Q R X t| d}td8}||d |j|ddd t|d}t	| | W 5 Q R X t| d}tdP}||d ||d |j|dddd t|d}t	t
| | g| W 5 Q R X d S )Nr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   z/data2rY   modewz/data1Frw   append)r#   r$   r%   r&   r'   r(   r   r)   r*   r   rs   )r-   r   r.   r/   r1   r1   r2    test_to_hdf_modes_multiple_nodes   sH    
 





r{   c               	   C   s  t d tjddddgdddd	gd
ddddgd} t| d}t Z}tj	|d}|
tj	|dd |j
|ddd t|d}tt| | g| W 5 Q R X t| d}t Z}tj	|d}|
tj	|dd |j
|ddd t|d}tt| | g| W 5 Q R X t| d}t P}tj	|d}|
tj	|dd |j
|ddd t|d}t| | W 5 Q R X t| d}t \}tj	|d}|
tj	|dd |j
|dddd t|d}tt| | g| W 5 Q R X d S )Nr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   data*Zdata2r    rv   rY   Zdata1rx   Fry   )r#   r$   r%   r&   r'   r(   r   rp   rq   rr   r)   r*   r   rs   )r-   r   rt   r.   r/   r1   r1   r2    test_to_hdf_modes_multiple_files  sB    
 
r}   c                  C   s@  t d tjdddddddd	d
dddddddgddddddddddddddd d!gd"d#d$d%d&d'd(d)d*d+d,d-d.d/d0d1d2gd3} t| d!}t 6}tj	|d4}|j
|d5d6d7}t|jdkstW 5 Q R X t (}|j
|d8d6d7}t|jdkstW 5 Q R X t 0}|j
|d5d6d7}t|jd|j ks2tW 5 Q R X d9S ):zNtesting dask link levels is correct by calculating the depth of the dask graphr   r   r   r   r   r5   r6   r7   r8   r9   r:   r;   r<   r=   r>   r?   r@   r   r   r   r   rA   rB   rC   rD   rE   rF   rG   rH   rI   rJ   rK   rL   r   r   r   r   r   rM   rN   rO   rP   rQ   rR   rS   rT   rU   rV   rW   rX   r   r|   r    Fr!   rY   N)r#   r$   r%   r&   r'   r(   r   rp   rq   rr   r)   r
   rg   ri   r   npartitions)rk   r   rt   r.   r   r1   r1   r2   test_to_hdf_link_optimizationsI  sd    
")r   c                  C   s,  t d tjdddddddd	d
dddddddgddddddddddddddd d!gd"d#d$d%d&d'd(d)d*d+d,d-d.d/d0d1d2gd3} t| d!}d4d5 }t 8}|j|d|d6}||d7 t	|d7}t
| | W 5 Q R X t F}tj|d8}|j|d|d6}||d9 t	|d9}t
| | W 5 Q R X d S ):Nr   r   r   r   r   r5   r6   r7   r8   r9   r:   r;   r<   r=   r>   r?   r@   r   r   r   r   rA   rB   rC   rD   rE   rF   rG   rH   rI   rJ   rK   rL   r   r   r   r   r   rM   rN   rO   rP   rQ   rR   rS   rT   rU   rV   rW   rX   r   c                 S   s$   | d dk r t dd| d    | S )Nr   rF   g?r   r\   r1   r1   r2   delayed_nop  s    z,test_to_hdf_lock_delays.<locals>.delayed_nop)ZaxismetarY   r|   r    )r#   r$   r%   r&   r'   r(   r   applyr)   r*   r   r   rp   rq   rr   )rk   r   r   r.   r/   rt   r1   r1   r2   test_to_hdf_lock_delays  sh    
")r   c                  C   s   t d tjddddgdddd	gd
ddddgd} t| d}t 6}t t  t	j
|d}||d W 5 Q R X W 5 Q R X t >}t|(}t t ||d W 5 Q R X W 5 Q R X W 5 Q R X d S )Nr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   ro   rZ   z	/data_*_*)r#   r$   r%   r&   r'   r(   r   raises
ValueErrorrp   rq   rr   r)   r   rf   )r-   r   rt   r.   rl   r1   r1   r2   test_to_hdf_exceptions  s    
 
 r   	schedulersyncthreadsZ	processesr~   r   r   rF   c                 C   sJ  t d tjdddddddd	d
dddddddgddddddddddddddd d!gd"d#d$d%d&d'd(d)d*d+d,d-d.d/d0d1d2gd3}tj||d4}td5,}|j|d6| d7 t|d6}t	|| W 5 Q R X t
 :}tj|d8}|j|d6| d7 t|d6}t	|| W 5 Q R X td5,}|j|d9| d7 t|d9}t	|| W 5 Q R X d S ):Nr   r   r   r   r   r5   r6   r7   r8   r9   r:   r;   r<   r=   r>   r?   r@   r   r   r   r   rA   rB   rC   rD   rE   rF   rG   rH   rI   rJ   rK   rL   r   r   r   r   r   rM   rN   rO   rP   rQ   rR   rS   rT   rU   rV   rW   rX   r   r~   r   r    r   ro   rY   )r#   r$   r%   r&   r'   r(   r   r)   r*   r   r   rp   rq   rr   )r   r~   r-   r   r.   r/   rt   r1   r1   r2   test_to_hdf_schedulers  sj    
")

r   c               	   C   s   t d tdddgi} tj| dd}td0}|j|dd	d
d t|d}t	
| | W 5 Q R X td0}|j|ddd
d t|d}t	
| | W 5 Q R X d S )Nr   Ar   Zaaaar   r   r   Zfoo4tabler   )formatZmin_itemsizet)r#   r$   r%   r&   r'   r(   r   r)   r*   r   r+   )r-   ddfr.   Zdf2r1   r1   r2   test_to_hdf_kwargs2  s    


r   c                  C   s   t d tjdddddddd	d
dddddddgddddddddddddddd d!gd"d#d$d%d&d'd(d)d*d+d,d-d.d/d0d1d2gd3} t| d!}td40}t jtd5d6 |j	|d7t
d8 W 5 Q R X W 5 Q R X t "}tj|d9}|j|t
d8 W 5 Q R X d S ):Nr   r   r   r   r   r5   r6   r7   r8   r9   r:   r;   r<   r=   r>   r?   r@   r   r   r   r   rA   rB   rC   rD   rE   rF   rG   rH   rI   rJ   rK   rL   r   r   r   r   r   rM   rN   rO   rP   rQ   rR   rS   rT   rU   rV   rW   rX   r   r   z2To preserve order between partitions name_functionmatchrY   r_   z
data_*.csv)r#   r$   r%   r&   r'   r(   r   ZwarnsUserWarningr)   strr   rp   rq   rr   Zto_csv)rk   r   r.   rt   r1   r1   r2   test_to_fmt_warnsB  sb    
")
 $r   zdata, comparer   r   r   r   r   r   r   r   r   r   r   r   )namec                 C   s  t d tdd}| |d ztj|dddd ds<tW n2 tk
rp } zd	t|ks`tW 5 d }~X Y nX W 5 Q R X td}| j|dd
d tj|dddd}|j	dkst||
 |  |tj|dddddd
 tj|dddd ttj|dddjttj|dddjks$tW 5 Q R X tdP}|  }|j|dd
d tj|ddddd}|j	dkstt||
 | W 5 Q R X d S )Nr   r   r    datar   r0   	chunksizerw   Fzformat='table'r   r   r   r   )r   startstoprw   r   r   rv   T)r   sorted_indexrw   )r#   r$   r   r)   r'   r*   ri   	TypeErrorr   r~   r"   r%   sortedrg   Z
sort_index)r   comparer.   r5   r   Zsorted_datar1   r1   r2   test_read_hdf{  s4    

,

r   c               
   C   s   t d tjddddgdddd	gd
ddddgd} tdB}| j|ddd tj|dd tj|dddd W 5 Q R X W 5 Q R X dS )zXTest that we can read from a file that's already opened elsewhere in
    read-only mode.r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r    r   r   r0   rv   r   N)	r#   r$   r%   r&   r   r)   rf   r'   r*   r-   r.   r1   r1   r2   test_read_hdf_multiply_open  s    
 

r   c                  C   s   t d tjdddddddd	d
dddddddgddddddddddddddd d!gd"d#d$d%d&d'd(d)d*d+d,d-d.d/d0d1d2gd3} t| d!}td4L}||d5 tj|d5d6d7}|j	|j	kst
|j|jkst
t|| W 5 Q R X d S )8Nr   r   r   r   r   r5   r6   r7   r8   r9   r:   r;   r<   r=   r>   r?   r@   r   r   r   r   rA   rB   rC   rD   rE   rF   rG   rH   rI   rJ   rK   rL   r   r   r   r   r   rM   rN   rO   rP   rQ   rR   rS   rT   rU   rV   rW   rX   r   r   rY   T)r   )r#   r$   r%   r&   r'   r(   r   r)   r*   r~   ri   Z	divisionsr   )r-   r   r.   r0   r1   r1   r2   test_read_hdf_multiple  s\    
")
r   c               
   C   s   t d tjddddgdddd	gd
ddddgd} td}| j|ddd t jtdd tj	|ddd W 5 Q R X t jtdd tj	|ddd W 5 Q R X t jtdd tj	|ddd W 5 Q R X W 5 Q R X d S )Nr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r    r   r   znumber of rowsr   rF   )r   zis above or equal to)r   zpositive integerr   )
r#   r$   r%   r&   r   r)   r   r   r'   r*   r   r1   r1   r2   test_read_hdf_start_stop_values  s    
 

r   c               
   C   s  t d tjddddgdddd	gd
ddddgd} t }| jtj|dddd | jtj|dddd | jtj|dddd t	j
jddZ tjtj|dddd}|jdkstt| |  tjtj|dddddd}tjtj|ddddd}t| | tjtj|dddd}|jd	ksLtt| t| gd  tjtj|dddd}|jd	kstt| t| gd  tjtj|dddd}|jdkstt| t| gd  W 5 Q R X W 5 Q R X d S )Nr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   one.h5z	/foo/datar   r   two.h5z	/bar/datar   r   z/*/datar   )r   r   r   r   z*.h5rB   )r#   r$   r%   r&   r   r)   rp   rq   rr   rg   configsetr'   r*   r~   ri   r   r+   r"   rs   )r-   tdirresexpectedr1   r1   r2   test_hdf_globbing  sH    
 

       r   c               
   C   s   t d tjddddgdddd	gd
ddddgd} t }| jd d jtj	|dddd | jdd  jtj	|dddd t
jjdd> tj	|dtj	|dg}t|d}t| |  W 5 Q R X W 5 Q R X d S )Nr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   	dataframer   r   r   r   r   )r#   r$   r%   r&   r   re   r)   rp   rq   rr   rg   r   r   r'   r*   r   r+   r"   )r-   r   Zinput_filesr   r1   r1   r2   test_hdf_file_list(  s    
 
$$r   c               	   C   s~   t d tjddddgdddd	gd
ddddgd} td6}t|}| j|ddd t	|d}t
||  W 5 Q R X d S )Nr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   )r#   r$   r%   r&   r   pathlibPathr)   r'   r*   r   )r-   r.   rq   r   r1   r1   r2   test_read_hdf_pattern_pathlike8  s    
 


r   c               	   C   s   t d tjddddgdddd	gd
ddddgd} tj| dd}td2}t|}|	|d t
|d}t|| W 5 Q R X d S )Nr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r    )r#   r$   r%   r&   r'   r(   r   r   r   r)   r*   r   )r-   r   r.   rq   r   r1   r1   r2   test_to_hdf_path_pathlikeE  s    
 


r   c               
   C   s   t d tdd} d}ttj|d}tj| dd}|	d| W 5 Q R X t
j| ddd	}t||ksrtW 5 Q R X d S )
Nr   r   (   r   rx   rv   z/xr   r   )r#   r$   r   r%   r&   nprandomrandnrf   rz   r'   r*   lenri   )r.   Nr-   storer   r1   r1   r2   test_read_hdf_doesnt_segfaultS  s    

r   c                  C   st   t d tjddddgdddd	gd
ddddgd} tj| dd}|ddddgks\tt	d t	d d S )Nr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   z	foo*.hdf5rc   z	foo0.hdf5z	foo1.hdf5)
r#   r$   r%   r&   r'   r(   r)   ri   rp   remove)r-   r   r1   r1   r2   test_hdf_filenames_  s    
 

r   c                	   C   sn   t t tdd W 5 Q R X t t tddgd W 5 Q R X t t tg d W 5 Q R X d S )NZnonexistant_store_X34HJKz/tmpZnonexistant_store_UY56YH)r#   r   IOErrorr'   r*   r   r1   r1   r1   r2   test_hdf_path_exceptionsj  s    r   c            	   
      sZ  t d dd l G  fddd j} G  fddd j}G  fddd j}td	} j|d
d}|dd}||d| d}|j}d|d< |	  ||d|d}|j}d|d< |	  ||d|d}|j}d|d< |	  W 5 Q R X t
tjdd}|j|ddd
d t|d t|d t|d t|d W 5 Q R X d S ) Nr   r   c                       s   e Zd Z  ZdS )z'test_hdf_nonpandas_keys.<locals>.Table1N)__name__
__module____qualname__
Float32Colvalue1r1   r   r1   r2   Table1  s   r   c                       s   e Zd Z  ZdS )z'test_hdf_nonpandas_keys.<locals>.Table2N)r   r   r   r   value2r1   r   r1   r2   Table2  s   r   c                       s   e Zd Z  ZdS )z'test_hdf_nonpandas_keys.<locals>.Table3N)r   r   r   r   value3r1   r   r1   r2   Table3  s   r   r   r   rv   /groupZtable1zTable 1r   r   Ztable2zTable 2r   Ztable3zTable 3r   rF   r   z/barr   )r   rw   z/group/table1z/group/table2z/group/table3)r#   r$   r   ZIsDescriptionr   	open_fileZcreate_groupZcreate_tablerowrz   r%   r&   r   r   r   r)   r'   r*   )	r   r   r   rq   Zh5filer   r   r   barr1   r   r2   test_hdf_nonpandas_keysy  s4    

r   c              	   C   sn   t d ddlm} tjg g dg d}|j| d ddd	d
 t jt|d t	
| d d W 5 Q R X d S )Nr   r   )dont_use_fixed_error_message)r   Br   zdata.h5Zfixedr-   rx   )r   rc   rw   r   )r#   r$   Zdask.dataframe.io.hdfr   r%   r&   r)   r   r   r'   r*   )Ztmp_pathr   r-   r1   r1   r2   test_hdf_empty_dataframe  s    
r   )=rp   r   timer   Znumpyr   Zpandasr%   r#   Zpackaging.versionr   rg   Zdask.dataframer   r'   Zdask.compatibilityr   Zdask.dataframe._compatr   Zdask.dataframe.optimizer   Zdask.dataframe.utilsr   Zdask.layersr	   Z
dask.utilsr
   r   r   r3   markZskipifrn   ru   r{   r}   r   Zslowr   r   Zparametrizer   r   r   r&   r+   ZSeriesr,   r   r   r   r   r   r   r   r   r   r   r   r   r   r1   r1   r1   r2   <module>   s   

Z]0+D
E
A9

#

5%*