U
    /e&
                     @  s   d dl mZ d dlmZmZ er(d dlZdddddd	Zdd
dddZdd
dddZ	dddddZ
d
ddddZdd
dddZdS )    )annotations)TYPE_CHECKINGBinaryIONzlist[pa.Table]r   None)shardsfilereturnc              
   C  s<   ddl }| D ]*}|j||j}|| W 5 Q R X qdS )z
    Write multiple shard tables to the file

    Note: This function appends to the file and dumps each table as an individual stream.
    This results in multiple end-of-stream signals in the file.

    See Also
    --------
    load_partition
    r   N)pyarrowipc
new_streamschemawrite_table)r   r   patablewriter r   >/tmp/pip-unpacked-wheel-g426oqom/distributed/shuffle/_arrow.pydump_shards	   s    r   zpa.Table)r   r   c                 C  sl   ddl }ddl}|  }| d|j |  }| | g }|  |k rb|| }||  q<||S )a  Load partition data written to file back out into a single table

    Example
    -------
    >>> tables = [pa.Table.from_pandas(df), pa.Table.from_pandas(df2)]  # doctest: +SKIP
    >>> with open("myfile", mode="wb") as f:  # doctest: +SKIP
    ...     for table in tables:  # doctest: +SKIP
    ...         dump_shards(tables, f)  # doctest: +SKIP

    >>> with open("myfile", mode="rb") as f:  # doctest: +SKIP
    ...     t = load_partition(f)  # doctest: +SKIP

    See Also
    --------
    dump_shards
    r   N)	osr	   tellseekSEEK_ENDRecordBatchStreamReaderappendread_allconcat_tables)r   r   r   posendr   srr   r   r   load_partition   s    

r   zlist[bytes])datar   c                 C  s   ddl }|dd | D S )z>Convert a list of arrow buffers and a schema to an Arrow Tabler   Nc                 s  s   | ]}t |V  qd S )N)deserialize_table).0bufferr   r   r   	<genexpr>?   s     z+list_of_buffers_to_table.<locals>.<genexpr>)r	   r   )r    r   r   r   r   list_of_buffers_to_table;   s    r%   bytesz	pa.Schemac                 C  sL   ddl }ddl}| }||  |d ||}| }|  |jS )zDeserialize an arrow schema

    Examples
    --------
    >>> b = schema.serialize()  # doctest: +SKIP
    >>> deserialize_schema(b)  # doctest: +SKIP

    See also
    --------
    pa.Schema.serialize
    r   N)	ior	   BytesIOwriter   r   r   closer   )r    r'   r   Zbior   r   r   r   r   deserialize_schemaB   s    


r+   )r   r   c              	   C  sF   dd l }dd l}| }|j|| j}||  W 5 Q R X | S Nr   )r'   r	   r(   r
   r   r   r   getvalue)r   r'   r   streamr   r   r   r   serialize_table[   s    r/   )r#   r   c              
   C  s:   dd l }|j|| }| W  5 Q R  S Q R X d S r,   )r	   r
   Zopen_streamZ	py_bufferr   )r#   r   readerr   r   r   r!   f   s    r!   )
__future__r   typingr   r   r	   r   r   r   r%   r+   r/   r!   r   r   r   r   <module>   s    