U
    k/e1                     @   s   d dl mZ d dlmZ d dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
Zd dlZd dlZd dlmZmZmZ dd Zdd
dZdd Zdd Zdd ZG dd dZG dd dee	jZG dd dee	jZdS )    )OrderedDict)DecimalN)	read_jsonReadOptionsParseOptionsc                  c   s2   t j} | E d H  | D ]}| D ]}|| V  qqd S N)stringascii_lowercase)lettersfirstsecond r   ;/tmp/pip-unpacked-wheel-seu8352k/pyarrow/tests/test_json.pygenerate_col_names"   s
    
r      
   
c                 C   s   t jdjdd| |fd}ttt | }g }|jD ],}t	dd t
||D }|t| q8|| }dd |D }	tj|	|}
||
fS )N*   r   i  )sizec                 S   s   g | ]\}}|t |fqS r   )int).0kvr   r   r   
<listcomp>0   s     z$make_random_json.<locals>.<listcomp>c                 S   s   g | ]}t j|t  d qS ))type)paarrayint64)r   colr   r   r   r   3   s     )nprandomZRandomStaterandintlist	itertoolsislicer   Tr   zipappendjsondumpsjoinencoder   ZTableZfrom_arrays)num_colsnum_rowslinesepZarrZ	col_nameslinesrowZjson_objdatacolumnsexpectedr   r   r   make_random_json+   s    
r4   c                 K   sH   | f |}t t j|t jd}| D ]\}}t|||ks(tq(d S )N)protocol)pickleloadsr)   HIGHEST_PROTOCOLitemsgetattrAssertionError)clsZattr_valuesoptsZnew_optsnamevaluer   r   r   check_options_class_pickling8   s    

r@   c                  C   s   t } |  }|jdkstd|_|jdks,t|jdks:td|_|jdksNt| ddd}|jdksht|jdksvtt| ddd d S )Nr   i90  TFi  )
block_sizeuse_threads)r   rA   r;   rB   r@   )r<   r=   r   r   r   test_read_options@   s    rC   c               	   C   s   t } |  }|jdkst|jd ks&td|_|jdks:tttdt g}||_|j|ksft|jdksttdD ]}||_|j|ksxtqxt	
t d|_W 5 Q R X t| |ddd d S )	NFTfooinfer)ignoreerrorrE   zinvalid-valuerF   )explicit_schemanewlines_in_valuesunexpected_field_behavior)r   rI   r;   rH   r   schemafieldZint32rJ   pytestraises
ValueErrorr@   )r<   r=   rK   r?   r   r   r   test_parse_optionsT   s&    rP   c                   @   s   e Zd Zdd Zdd Zdd Zdd Zd	d
 Zdd Zdd Z	dd Z
dd Zdd Zdd Zdd Zdd Zdd Zdd ZdS ) BaseTestJSONReadc                 K   s   | j t|f|S r   )r   r   Z	py_buffer)selfbkwargsr   r   r   
read_bytesp   s    zBaseTestJSONRead.read_bytesc                 C   s.   |j t|kstdd |jD |ks*td S )Nc                 S   s   g | ]
}|j qS r   )r>   )r   cr   r   r   r   u   s     z0BaseTestJSONRead.check_names.<locals>.<listcomp>)num_columnslenr;   r2   )rR   tablenamesr   r   r   check_namess   s    zBaseTestJSONRead.check_namesc              	   C   sh   d}dgdgd}t |}| |}| |ks6tt | }tt	 | | W 5 Q R X d S )Ns   {"a": 1, "b": 2}
   r   )arS   )
ioBytesIOr   	to_pydictr;   StringIOdecoderM   rN   	TypeError)rR   r1   Zexpected_dataZbiorY   sior   r   r   test_file_objectw   s    

z!BaseTestJSONRead.test_file_objectc                 C   s   d}t  }t }||d fD ]}dD ]z}||_d|_tjtdd | j|||d W 5 Q R X tdd	D ]4}||_| j|||d}|	 d
dddgiksht
qhq$qd S )Ns   {"a": 1}
{"a": 2}
{"a": 3}   
FT   ztry to increase block sizematchread_optionsparse_options	      r]   r\   r      )r   r   rI   rA   rM   rN   rO   rU   ranger`   r;   )rR   rowsrl   rm   r1   rI   rA   rY   r   r   r   test_block_sizes   s&    z!BaseTestJSONRead.test_block_sizesc                 C   s6   d}|  |}| ddgddgddgdks2td S )	Ns/   {"a": 1,"b": 2, "c": 3}
{"a": 4,"b": 5, "c": 6}r\   rh   r      rp      r]   rS   rV   )rU   r`   r;   )rR   rr   rY   r   r   r   test_no_newline_at_end   s    
z'BaseTestJSONRead.test_no_newline_at_endc                 C   sl   d}|  |}tdt fdt fdt fg}|j|ksDt| ddgddgd	d
gdkshtd S )Ns0   {"a": 1,"b": 2, "c": 3}
{"a": 4,"b": 5, "c": 6}
r]   rS   rV   r\   rh   r   rt   rp   ru   rv   )rU   r   rK   r   r;   r`   rR   rr   rY   rK   r   r   r   test_simple_ints   s    


z!BaseTestJSONRead.test_simple_intsc                 C   s|   d}|  |}tdt fdt fdt fdt fg}|j|ksNt| ddgdd	gd
dgddgdksxtd S )NsQ   {"a": 1,"b": 2, "c": "3", "d": false}
{"a": 4.0, "b": -5, "c": "foo", "d": true}
r]   rS   rV   d      ?g      @r   3rD   FT)r]   rS   rV   rz   )	rU   r   rK   float64r   r   bool_r;   r`   rx   r   r   r   test_simple_varied   s    



z#BaseTestJSONRead.test_simple_variedc              	   C   s   d}|  |}tdt fdt fdt fdt fdt fg}|j|ksXt|	 dd dgd	d
d gd ddgd d d gd ddgdkstd S )Ns   {"a": 1, "b": 2, "c": null, "d": null, "e": null}
{"a": null, "b": -5, "c": "foo", "d": null, "e": true}
{"a": 4.5, "b": null, "c": "nan", "d": null,"e": false}
r]   rS   rV   rz   er{   g      @r   r|   rD   nanTF)r]   rS   rV   rz   r   )
rU   r   rK   r~   r   r   nullr   r;   r`   rx   r   r   r   test_simple_nulls   s     




z"BaseTestJSONRead.test_simple_nullsc                 C   sP   d}|  |}tdtt fg}|j|ks6t| dg giksLtd S )N	   {"a": []}r]   )rU   r   rK   list_r   r;   r`   rx   r   r   r   test_empty_lists   s
    
z!BaseTestJSONRead.test_empty_listsc                 C   sF   d}|  |}tg }|j|ks&t|jdks4t|jdksBtd S )Ns   {}
{}
r   r   )rU   r   rK   r;   rW   r-   rx   r   r   r   test_empty_rows   s    

z BaseTestJSONRead.test_empty_rowsc              
   C   s   d}t t|d}dd dgfdd g gfdd g dgggfdd i gfd	d d
d id
ddiigffD ]F\}}| j|| |d}d|i}| |kst|djdks\tq\d S )Ns"   {                               }
)rA   s   {"a": 0}r   r   s   {"a": []}
{"a": [[1]]}r\   s	   {"a": {}}s    {"a": {}}
{"a": {"b": {"c": 1}}}rS   rV   )rl   r]   )r   rX   rU   r`   r;   columnZ
num_chunks)rR   Z	first_rowrl   Z	next_rowsZexpected_pylistrY   r3   r   r   r   test_reconcile_accross_blocks   s     



z.BaseTestJSONRead.test_reconcile_accross_blocksc                 C   s   d}dt dt dt dd gi}tjtjfD ]P}td|ddfg}t|d}| j||d	}|j|kslt| |ks,tq,d S )
Ns'   {"a": 1}
{"a": 1.45}
{"a": -23.456}
{}
r]   1z1.45z-23.456rn   rh   rH   rm   )	r   r   Z
decimal128Z
decimal256rK   r   rU   r;   r`   )rR   rr   r3   Ztype_factoryrK   r=   rY   r   r   r   test_explicit_schema_decimal   s     
z-BaseTestJSONRead.test_explicit_schema_decimalc              	   C   s   d}t dt  fg}t|d}| j||d}|jt dt  fdt  fgksXt| ddgdd	gd
ksvtt|dd}| j||d}|jt dt  fgkst| dddgikstt|dd}tj	t j
dd | j||d W 5 Q R X d S )Ns2   {"foo": "bar", "num": 0}
{"foo": "baz", "num": 1}
rD   r   r   nums   bars   bazr   r\   )rD   r   rF   )rH   rJ   rG   z"JSON parse error: unexpected fieldri   )r   rK   binaryr   rU   r   r;   r`   rM   rN   ZArrowInvalid)rR   rr   rK   r=   rY   r   r   r   .test_explicit_schema_with_unexpected_behaviour  s@    




 
z?BaseTestJSONRead.test_explicit_schema_with_unexpected_behaviourc                 C   sP   t ddd\}}| |}|j|jks*t||s8t| | ksLtd S )Nr   r   r,   r-   )r4   rU   rK   r;   equalsr`   )rR   r1   r3   rY   r   r   r   test_small_random_json+  s
    
z'BaseTestJSONRead.test_small_random_jsonc           	      C   s   t ddd\}}t }t }||dfD ]b}dD ]X}||_dD ]H}||_| j|||d}|j|jksjt|	|s@|
 |
 ks@tq@q2q*d S )Nr   d   r   s   
rg   )      %   rk   )r4   r   r   rstriprI   rA   rU   rK   r;   r   r`   )	rR   Z	data_baser3   rl   rm   r1   rI   rA   rY   r   r   r   test_stress_block_sizes2  s    
z(BaseTestJSONRead.test_stress_block_sizesN)__name__
__module____qualname__rU   r[   re   rs   rw   ry   r   r   r   r   r   r   r   r   r   r   r   r   r   rQ   n   s   	%rQ   c                   @   s   e Zd Zdd ZdS )TestSerialJSONReadc                 O   s.   | dt }d|_t||}|jdd |S )Nrl   FTfull
setdefaultr   rB   r   validaterR   argsrT   rl   rY   r   r   r   r   G  s
    
zTestSerialJSONRead.read_jsonNr   r   r   r   r   r   r   r   r   E  s   r   c                   @   s   e Zd Zdd ZdS )TestParallelJSONReadc                 O   s.   | dt }d|_t||}|jdd |S )Nrl   Tr   r   r   r   r   r   r   Q  s
    
zTestParallelJSONRead.read_jsonNr   r   r   r   r   r   O  s   r   )r   r   r   )collectionsr   decimalr   r^   r#   r(   r6   r   ZunittestZnumpyr   rM   Zpyarrowr   Zpyarrow.jsonr   r   r   r   r4   r@   rC   rP   rQ   ZTestCaser   r   r   r   r   r   <module>   s(   	
 X
