U
    /e                     @   s   d dl mZ d dlZd dlmZ d dlmZ d dlmZm	Z	 d dl
mZ d dlmZ d dlmZ eed	d
ZdddZddd eeD Zdd eD dg ZddgZdd eD Zejdedd Zdd Zdd Zdd Zdd  ZdS )!    )partialN)compr)concat)computeconfig)	read_text)utils)	filetextssync	schedulerz{"amount": 100, "name": "Alice"}
{"amount": 200, "name": "Bob"}
{"amount": 300, "name": "Charlie"}
{"amount": 400, "name": "Dennis"}
z{"amount": 500, "name": "Alice"}
{"amount": 600, "name": "Bob"}
{"amount": 700, "name": "Charlie"}
{"amount": 800, "name": "Dennis"}
)z.test.accounts.1.jsonz.test.accounts.2.json c                 C   s   g | ]}t | qS  )files).0vr   r   </tmp/pip-unpacked-wheel-dbjnr7gq/dask/bag/tests/test_text.py
<listcomp>   s     r   c                 C   s   g | ]}|d fqS Nr   )r   fmtr   r   r   r   !   s     )Nz10 Basciizutf-8c                 C   s0   g | ](\}}t D ]}d D ]}||||fqqqS ))TF)	encodings)r   r   bsencodinginclude_pathr   r   r   r   $   s     zfmt,bs,encoding,include_pathc              	      sj  | t jkrtd|   t j|    fddt D }t|dd td| |d}t|\}d	|t
ksvtttt| ||d	}|r|d
n|}t|\}d	|t
kst|r$t|d\}ttdd t D }	t|t|	kstt||	D ]\}
}|
|stqtd| |dd}t| }d	dd |D t
ks\tW 5 Q R X d S )Nz%compress function not provided for %sc                    s    i | ]\}}| | qS r   )encoder   kr   compressr   r   r   
<dictcomp>1   s      z"test_read_text.<locals>.<dictcomp>bmodez.test.accounts.*.json)compression	blocksizer   r   )r$   r%   r   r   r      c                 S   s    g | ]\}}|g| d  qS )
)countr   r   r   r   r   F   s     z"test_read_text.<locals>.<listcomp>F)r$   r%   r   
collectionc                 s   s   | ]}|D ]
}|V  q
qd S r   r   )r   blockliner   r   r   	<genexpr>T   s       z!test_read_text.<locals>.<genexpr>)r   r   pytestskipr   itemsr	   r   r   joinexpectedAssertionErrorsortedZplucklistr   lenzipendswith)r   r   r   r   Zfiles2r!   LopathsZexpected_pathspathZexpected_pathblocksr   r   r   test_read_text,   sP    

   

r=   c              	   C   sb   d}| d }t |d}|d||g W 5 Q R X t|dd}|d  }t|dks^td S )	Ns   abcdézdata.txtwb   
F)r)   r      )openwriter0   r   r   r5   r2   )Ztmp_pathdatafnfresultr   r   r   $test_read_text_unicode_no_collectionW   s    rG   c                  C   s*  dd t dD } t|  tddi tt$ tddd}t|j	d	d
d}W 5 Q R X |dksrt
d|  dkst
dtt" tdddd}|j	d	d
d}W 5 Q R X tt| d
 }t|}t|dkst
tt|  d
 }t|}t|dkst
W 5 Q R X W 5 Q R X d S )Nc                 S   s   i | ]}|d ddqS )02z.txtzline from {:02}r   )r   nr   r   r   r    d   s     
 z,test_files_per_partition.<locals>.<dictcomp>   r   zsingle-threadedz*.txt
   )files_per_partitiond   r&   )Znpartitionsz-10 files should be grouped into one partitionzAll 20 lines should be readT)rL   r   )ranger	   r   setr-   ZwarnsUserWarningr   r5   Ztaker2   r(   r   tupler6   )Zfiles3r!   lpZp_pathsZp_unique_pathsZb_pathsZb_unique_pathsr   r   r   test_files_per_partitionc   s"    rT   c               
   C   sp   t ddiddV tt tddd  W 5 Q R X tdddd} | jd	d
} | ddgksbtW 5 Q R X d S )Nz	.test.foos
   Jos
Alicer!   r"   r   )r   ignore)r   errorsr
   r   zJos
ZAlice)r	   r-   ZraisesUnicodeDecodeErrorr   r   r2   )rF   r   r   r   test_errors~   s    rX   c               	   C   s   d} t d| i tddd  dks.ttdddd  dksLttddd }|d d	kslt|d
 ds~ttdddd }|d d	kst|d
 dstW 5 Q R X d S )Nz*abc
def
123
$$$$
dog
cat
fish


$$$$helloz.test.delim.txtz$$$$)linedelimiter   r@   )rY   r%   Zhellor   )r	   r   r(   r   r2   r7   )Zlongstrvalsr   r   r   test_complex_delimiter   s    r]   )	functoolsr   r-   Zfsspec.compressionr   Ztlzr   Zdaskr   r   Zdask.bag.textr   Z
dask.bytesr   Z
dask.utilsr	   r   r0   r3   r1   Zfmt_bsr   Zfmt_bs_enc_pathmarkZparametrizer=   rG   rT   rX   r]   r   r   r   r   <module>   s.   
*
