U
    /e                  	   @   s  d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dl	m
Z d dlmZ d dlmZ d dlmZ ed edZdZdd	gZejjd
d Zejdddd Zejdddgejjdd Zejjdd Zejjeej edk ddejdddgejdddddggdd  Z!ejjeej edk ddejdd!d"dd#gd$d% Z"ejjd&d' Z#ejjeej edk ddejjd(d) Z$ejjeej edk ddd*d+ Z%dS ),    N)parse)optimize_dataframe_getitem)	assert_eqzpyarrow.orcZpyarrowzhttps://www.googleapis.com/download/storage/v1/b/anaconda-public-data/o/orc%2FTestOrcFile.testDate1900.orc?generation=1522611448751555&alt=mediatimedatec                  C   s>   t d tt} t| jddhks*tt| dks:td S )Nrequestsr   r   p )	pytestimportorskipddread_orcurlsetcolumnsAssertionErrorlend r   D/tmp/pip-unpacked-wheel-dbjnr7gq/dask/dataframe/io/tests/test_orc.pytest_orc_with_backend   s    

r   module)Zscopec               
   #   sz   t d} | tj}t   fdddD }|D ]$}t|d}|| W 5 Q R X q4z
|V  W 5 t	j
 dd X d S )Nr   c                    s   g | ]}t j |qS r   )ospathjoin).0fnr   r   r   
<listcomp>(   s     zorc_files.<locals>.<listcomp>)z	test1.orcz	test2.orcwbT)ignore_errors)r	   r
   getr   contenttempfilemkdtempopenwriteshutilrmtree)r   datafilesr   fr   r   r   	orc_files#   s    

r+   split_stripes      c              	      s   | d }t j||d}t|dks&t|jd| ks8tt j|ddgd}t|t |t dd	 tjt	d
d t j|dd
gd W 5 Q R X |t   fddt
 jD }t  |}dd |j D d }t|j| jttkstd S )Nr   )r,   r      r   r   r   FZcheck_indexZnonexist)matchc                    s   g | ]} j |fqS r   )_name)r   iZd3r   r   r   B   s     z#test_orc_single.<locals>.<listcomp>c                 S   s   g | ]}| d r|qS )z	read-orc-)
startswith)r   kr   r   r   r   D   s     
 )r   r   r   r   npartitionsr   r   r	   Zraises
ValueErrorranger   Z__dask_graph__Zlayerskeysr   )r+   r,   r   r   d2r;   graphkeyr   r5   r   test_orc_single2   s    r?   c                 C   sv   t | d }t | }t|t t ||gt dd t tj| d d }t|t t ||gt dd d S )Nr   Fr1   z/*.orc)r   r   r   r   concatr   r   dirname)r+   r   r<   r   r   r   test_orc_multipleH   s
    
 rB   z4.0.0z.PyArrow>=4.0.0 required for ORC write support.)reasonindexi32r   i64r*   c              	      s   t | }ttjdtjdtjdtjdtjdtjdtjj	dddgdd
dd} rf| }tj|d	d
}|r| fdd|D  }|j|t d tj| |d}t||t d d S )Ni  ZdtypeZhelloZyoZpeoplesizeO)rE   rF   r*   Zbhelloi  )	chunksizec                    s   g | ]}| kr|qS r   r   )r   crD   r   r   r   g   s      z&test_orc_roundtrip.<locals>.<listcomp>Zwrite_index)rD   r   r1   )strpd	DataFramenparangeZint32Zint64float64randomchoiceZastypeZ	set_indexr   from_pandasto_orcboolr   r   )tmpdirrD   r   tmpr(   dfdf2r   rM   r   test_orc_roundtripQ   s$    

r^   TF   c                 C   s   t | }ttjdtjdtjjdddgddd}tj	|dd	}|j
|d
d tj||dd}|r|j|jt| kstn|j|jkstt||d
d d S )Nd   rG   catZdogZmouserH   )abr/   r8   FrN   Tr,   Zaggregate_filesr1   )rO   rP   rQ   rR   rS   rT   rU   rV   r   rW   rX   r   r8   intr   r   )rZ   r,   r[   r(   r\   r]   r   r   r   "test_orc_roundtrip_aggregate_filesq   s    rg   c                 C   sL   t j| d d ddd}|jdks&tt|jd jt|jd ksHtd S )Nr.      Tre   r   )r   r   r8   r   r   Z
partitionsrD   )r+   r]   r   r   r   test_orc_aggregate_files_offset   s    ri   c                 C   s<   t | }|jdst|j|dd}|jds8td S )Nzread-orcFcomputezto-orc)r   r   r3   r6   r   rX   )r+   tmp_pathr\   outr   r   r   test_orc_names   s    
rn   c                 C   s   t jtjddddddgd}tj|dd}tj	| d	}|
| tttj	|d
dksfttj	| d}|j
|dd}|  tttj	|d
dkstd S )Nr`   r_   rb   rc   rL   r   r0   rd   Zeager_orc_dataset*Zdelayed_orc_datasetFrj   )rP   rQ   rR   rU   Zrandnr   rW   r   r   r   rX   r   globr   rk   )rl   r\   ZddfZ
eager_pathZdelayed_pathZdatasetr   r   r   test_to_orc_delayed   s     
 rq   )&rp   r   r&   r"   ZnumpyrR   ZpandasrP   r	   Zpackaging.versionr   parse_versionZdask.dataframeZ	dataframer   Zdask.dataframe.optimizer   Zdask.dataframe.utilsr   r
   par   r   marknetworkr   Zfixturer+   Zparametrizer?   rB   Zskipif__version__r^   rg   ri   rn   rq   r   r   r   r   <module>   sd   







