U
    /e,                     @   sF  d dl Z d dlZd dlZd dlZd dlmZ d dlmZ d dlZd dl	Z	d dl
mZ d dlmZ d dlmZ d dlmZmZ d dlmZ d d	lmZ d d
lmZ d dlmZ eeddZdddZddddZdd Zdd Zdd Zdd Z dd Z!dd  Z"d!d" Z#e	j$%d#d$d%gd&d' Z&d(d) Z'd*d+ Z(d,d- Z)e	j$j*ej+d.kd/d0d1d2 Z,d3d4 Z-d5d6 Z.d7d8 eD d9d8 eD  Z/e	j$%d:e/d;d< Z0d=d> Z1e	j$%d?d@dAgdBdC Z2e	j$%dDdEdFge	j$%dGe3edHdI Z4dJdK Z5dLdM Z6e	j$j7dNdO Z8e	j$%dPde9fdQe j9fgdRdS Z:dTdU Z;dVdW Z<dXdY Z=dS )Z    N)partial)sleep)compr)
open_files)LocalFileSystem)concatvalmap)compute)
read_bytes)compress)	filetextssync)Z	schedulers   {"amount": 100, "name": "Alice"}
{"amount": 200, "name": "Bob"}
{"amount": 300, "name": "Charlie"}
{"amount": 400, "name": "Dennis"}
s   {"amount": 500, "name": "Alice"}
{"amount": 600, "name": "Bob"}
{"amount": 700, "name": "Charlie"}
{"amount": 800, "name": "Dennis"}
).test.accounts.1.jsonz.test.accounts.2.jsons   a,b
1,2
s   a,b
3,4
s   a,b
5,6
)z.test.fakedata.1.csvz.test.fakedata.2.csvzsubdir/.test.fakedata.2.csvc                 C   s   t tj|  S N)pathlibPathospathabspathas_uri)r    r   ?/tmp/pip-unpacked-wheel-dbjnr7gq/dask/bytes/tests/test_local.pyto_uri-   s    r   c                	   C   s*   t t tddddh W 5 Q R X d S )Nzsets/are.csvzunordered/so/they.csvzshould/not/be.csvzallowed.csv)pytestraises	TypeErrorr
   r   r   r   r   test_unordered_urlpath_errors1   s    r   c               	   C   s   t tdd td\} }t| ts(t| d d tttd  d d ksPt| ds^tt|tt	fsptt|d tt	fstt
|d d dstttt|ttksttt| }t|tt kstW 5 Q R X d S )Nbmode.test.accounts.*   r      
dask)r   filesr
   
isinstancebytesAssertionErrorsortedendswithlisttuplehasattrsummaplenr	   r   setvalues)sampler1   resultsr   r   r   test_read_bytes?   s    (r4   c               	   C   s|   t tddf tdddd\} }| ds.ttdddd\} }| dsNttdddd\} }| dsntW 5 Q R X d S )	Nr   r   r    P   r"   )r2   	delimiterr      )r   r$   r
   r)   r'   r2   r1   r   r   r    test_read_bytes_sample_delimiterO   s    r9   c               	   C   s<   t tdd& tddd\} }t| dks.tW 5 Q R X d S )Nr   r   r    z40 Br2   (   )r   r$   r
   r/   r'   r8   r   r   r   test_parse_sample_bytesY   s    r<   c               	   C   s8   t tdd" tddd\} }| dks*tW 5 Q R X d S )Nr   r   r   Fr:   )r   r$   r
   r'   )r2   _r   r   r   test_read_bytes_no_sample_   s    r>   c               	   C   sF   t tdd0 tdd d\} }ttt|ttks8tW 5 Q R X d S )Nr   r   r    	blocksize)r   r$   r
   r-   r.   r/   r'   r8   r   r   r   test_read_bytes_blocksize_nonee   s    rA   r@   g      @z5 Bc              	   C   sp   t tddZ td| d\}}tt| }d|d}dt d}t|t|ksbt	W 5 Q R X d S )Nr   r   .test.account*r?       r"   )
r   r$   r
   r	   r   joinsplitr1   r0   r'   )r@   r2   valsr3   ourlines	testlinesr   r   r   test_read_bytes_blocksize_typesk   s    rI   c                
   C   s>   t tdd( tt tddd W 5 Q R X W 5 Q R X d S )Nr   r   rB   g      @r?   )r   r$   r   r   r   r
   r   r   r   r   $test_read_bytes_blocksize_float_errsu   s    rJ   c               	   C   sH   t tdd2 tddd\} } }dd |D t ks:tW 5 Q R X d S )Nr   r   r    T)Zinclude_pathc                 S   s   h | ]}t j|d  qS    )r   r   rE   ).0r   r   r   r   	<setcomp>~   s     z/test_read_bytes_include_path.<locals>.<setcomp>)r   r$   r
   keysr'   )r=   pathsr   r   r   test_read_bytes_include_path{   s    rQ   c               	   C   sR   t tdd< tdd } t| d d\}}ttt|ttksDtW 5 Q R X d S )Nr   r   z.test.accounts.*r?   )r   r$   r   r
   r-   r.   r/   r'   urlr2   r1   r   r   r   test_with_urls   s    rU   win32z!pathlib and moto clash on windows)reasonc               	   C   s|   t tdd: td} t| d d\}}ttt|ttksBtW 5 Q R X t	
t td} t| d d W 5 Q R X d S )Nr   r   z./.test.accounts.*r?   zfile://.test.accounts.*)r   r$   r   r   r
   r-   r.   r/   r'   r   r   OSErrorrS   r   r   r   test_with_paths   s    
$
rY   c               	      s   t tdd dD ] td d\} }ttt| fddt D ksNttt	| }t
dd	 |D t
d
d	 t D kstd|d}dt d}t|t|kstqW 5 Q R X d S )Nr   r   )r!      -   i  rB   r?   c                    s   g | ]}t t|  d qS rK   )maxr/   rM   vbsr   r   
<listcomp>   s    z)test_read_bytes_block.<locals>.<listcomp>c                 s   s   | ]}t |V  qd S r   r/   rM   rr   r   r   	<genexpr>   s     z(test_read_bytes_block.<locals>.<genexpr>c                 s   s   | ]}t |V  qd S r   rb   r]   r   r   r   re      s     rC   r"   )r   r$   r
   r*   r.   r/   r1   r'   r	   r   r-   rD   rE   r0   )r2   rF   r3   rG   rH   r   r_   r   test_read_bytes_block   s    ,rf   c               	   C   sb  t tddJ dD ]<} td| dd\}}td| dd\}}dd	 t|D d
d	 t|D ksfttt| }dd	 |D }tdd |D std|d}ddd t	tD d}||kstd}td| |d\}}tt| }dd	 |D }t
dd |D t|d ks$td|}	ddd t	tD }
|	|
kstqW 5 Q R X d S )Nr   r   )r!   rZ   r[   z1.5 kBz.test.accounts*r"   )r@   r6   s   fooc                 S   s   g | ]
}|j qS r   key)rM   ar   r   r   ra      s     z-test_read_bytes_delimited.<locals>.<listcomp>c                 S   s   g | ]
}|j qS r   rg   rM   r   r   r   r   ra      s     c                 S   s   g | ]}|r|qS r   r   rc   r   r   r   ra      s      c                 s   s   | ]}| d V  qdS )r"   Nr)   rc   r   r   r   re      s     z,test_read_bytes_delimited.<locals>.<genexpr>rC   c                 s   s   | ]}t | V  qd S r   r$   rM   kr   r   r   re      s        }c                 S   s   g | ]}|r|qS r   r   rc   r   r   r   ra      s      c                 s   s   | ]}| d V  qdS )ro   Nrk   rc   r   r   r   re      s     r7   c                 s   s   | ]}t | V  qd S r   rl   r]   r   r   r   re      s     )r   r$   r
   r   r'   r	   allrD   rE   r(   r-   r/   )r`   r=   r1   Zvalues2r3   resrG   rH   dZourstestr   r   r   test_read_bytes_delimited   s&    
($
rt   c                 C   s   g | ]}|d fqS r   r   rM   fmtr   r   r   ra      s     ra   c                 C   s   g | ]}|d fqS )
   r   ru   r   r   r   ra      s     zfmt,blocksizec              
   C   s   | t krtd tt |  t}t|dd | rj|rjtt td|d| d W 5 Q R X W 5 Q R  d S td|d| d\}}|d d tt	td  d d kst
|dst
tt| }d	|d	d
d t	tD kst
W 5 Q R X d S )N!compression function not providedr   r   z.test.accounts.*.jsonr"   )r@   r6   compressionr!   r   rC   c                 S   s   g | ]}t | qS r   rl   rm   r   r   r   ra      s     z$test_compression.<locals>.<listcomp>)r   r   skipr   r$   r   r   
ValueErrorr
   r(   r'   r)   r	   r   rD   )rv   r@   files2r2   r1   r3   r   r   r   test_compression   s.    

(r}   c                  C   sx   t tddb td} t| ttks*tt| ttD ]0\}}|}| }|t| ks^tW 5 Q R X q8W 5 Q R X d S Nr   r   r    )r   r$   r   r/   r'   zipr(   read)myfilesZ	lazy_fileZ	data_filefxr   r   r   test_open_files   s    r   encodingzutf-8asciic                    s   t tddv tdd d}t|ttks0tg }|D ]"}|}||  W 5 Q R X q8t| fddttD ks~tW 5 Q R X d S )Nr   r   r    rt)r   r   c                    s   g | ]}t |  qS r   )r$   decoderm   r   r   r   ra      s     z-test_open_files_text_mode.<locals>.<listcomp>)	r   r$   r   r/   r'   appendr   r*   r(   )r   r   datafiler   r   r   r   test_open_files_text_mode   s    r   r   r   rbrv   c                 C   s   |t krtd tt | t}t|ddx td| |d}g }|D ]"}|}||  W 5 Q R X qDdd t	tD }| dkrd	d |D }t
||kstW 5 Q R X d S )
Nrx   r   r   r    )r   ry   c                 S   s   g | ]}t | qS r   rl   rm   r   r   r   ra      s     z/test_open_files_compression.<locals>.<listcomp>r   c                 S   s   g | ]}|  qS r   )r   rj   r   r   r   ra     s     )r   r   rz   r   r$   r   r   r   r   r(   r*   r'   )r   rv   r|   r   r   r   r   Zsolr   r   r   test_open_files_compression   s    
r   c                  C   sP   t tdd: ttfD ]*} tt | ddd\}}W 5 Q R X qW 5 Q R X d S )Nr   r   r    z	not-found)ry   )r   r$   r
   r   r   r   r{   )funcr2   r1   r   r   r   test_bad_compression  s    r   c               	   C   s.   d} t jttf| d t|  W 5 Q R X d S )Nz
not-a-file)match)r   r   FileNotFoundErrorrX   r
   )fnr   r   r   test_not_found  s    r   c                  C   s   t tdd td\} }td\} }tt|}tt|}dd |D dd |D ks^ttd tD ]$}t|d}|d	 W 5 Q R X qjtd\} }tt|}d
d |D dd |D kstW 5 Q R X d S )Nr   r   r    c                 S   s   g | ]
}|j qS r   _keyrM   Zaar   r   r   ra     s     ztest_names.<locals>.<listcomp>c                 S   s   g | ]
}|j qS r   r   )rM   Zbbr   r   r   ra     s     rL   ab   xc                 S   s   g | ]
}|j qS r   r   r   r   r   r   ra   $  s     c                 S   s   g | ]
}|j qS r   r   )rM   ccr   r   r   ra   $  s     )	r   r$   r
   r*   r   r'   r   openwrite)r=   ri   r   r   r   cr   r   r   
test_names  s     r   compression_openergzipc              
   C   s   |\}}t | } t| dd|d}t|dks0tdd |D dhksHt|D ]}|}|d W 5 Q R X qLtt| }|ddgkst|tj	| |d	 d
}|
 }W 5 Q R X |dkstd S )Nr7   wb)numr   ry   c                 S   s   h | ]
}|j qS r   r   )rM   r   r   r   r   rN   -  s     z(test_open_files_write.<locals>.<setcomp>s   000z0.partz1.partr   r   )strr   r/   r'   r   r(   r   listdirr   rD   r   )tmpdirr   ry   openerr$   Zfilr   rr   r   r   r   test_open_files_write'  s    r   c                 C   s   t | } ttdd td}tt|}t||D ]d\}}|j|jksPt	t
|jt
|jksht	|*}|}| | kst	W 5 Q R X W 5 Q R X q8W 5 Q R X d S r~   )r   r   r$   r   cloudpickleloadsdumpsr   r   r'   typefsr   )r   r   Zmyfiles2r   f2Zf_openZf2_openr   r   r   test_pickability_of_lazy_files9  s    r   c              	   C   sl   t | d }tj|dd}|d W 5 Q R X t|ddd}|d }td	d
 |D s^tW 5 Q R X d S )Nzmyfile.txt.gzr   r   s   hello
worldr   r   )ry   r   r   c                 s   s   | ]}t |tV  qd S r   )r%   r   )rM   liner   r   r   re   O  s     z'test_py2_local_bytes.<locals>.<genexpr>)r   r   r   r   r   rp   r'   )r   r   r   r$   r   r   r   test_py2_local_bytesG  s    
r   c              	   C   s   t | } t }t|  tdd}|d W 5 Q R X t d}t|dksVt	d|d ksft	d|d ksvt	t }t| ||d d}|
 }W 5 Q R X |dkst	d S )	NtmpwhirR   rL   /r   rd   )r   r   getcwdchdirr   r   r   globr/   r'   r   )r   herer   outr   rq   r   r   r   test_abs_pathsR  s    

r   )>r   r   r   sys	functoolsr   timer   r   r   Zfsspec.compressionr   Zfsspec.corer   Zfsspec.implementations.localr   Ztlzr   r   r#   r	   Zdask.bytes.corer
   Zdask.bytes.utilsr   Z
dask.utilsr   r$   Z	csv_filesr   r   r4   r9   r<   r>   rA   markZparametrizerI   rJ   rQ   rU   ZskipifplatformrY   rf   rt   Zfmt_bsr}   r   r   r*   r   r   r   Zslowr   r   r   r   r   r   r   r   r   r   <module>   sr   

	





