U
    /e0                      @   s   d dl Z d dlZd dlZd dlZd dlZd dlmZ d dlmZm	Z	 d dl
mZ d dlmZmZmZmZ dd ZdaG dd	 d	eZed
dddddddddg	G dd dejZG dd deZdS )    N)suppress)cached_propertywraps)AbstractFileSystem)"get_package_version_without_importinfer_storage_optionsmirror_fromtokenizec                    s   t   fdd}|S )Nc               
      sj   z | |W S  t k
rd } z:|js& |j^}} t|trRd|krRttj||n W 5 d }~X Y nX d S )Nzdoes not exist)OSErrorargs
isinstancestrFileNotFoundErrorerrnoENOENT)r   kwargs	exceptionmessagefunc @/tmp/pip-unpacked-wheel-a8rmfik6/fsspec/implementations/arrow.pywrapper   s    
z wrap_exceptions.<locals>.wrapper)r   )r   r   r   r   r   wrap_exceptions   s    r   c                       s   e Zd ZdZdZ fddZedd Zedd Z	e
d	d
 Zd-ddZdd Zdd Zdd Zedd Zedd ZeZedd Zed.ddZed/dd Zed0d!d"Zed1d#d$Zed%d& Zed'd( Zd2 fd)d*	Z fd+d,Z  ZS )3ArrowFSWrapperzwFSSpec-compatible wrapper of pyarrow.fs.FileSystem.

    Parameters
    ----------
    fs : pyarrow.fs.FileSystem

    /c                    s    t da|| _t jf | d S )NZpyarrow)r   PYARROW_VERSIONfssuper__init__)selfr   r   	__class__r   r   r   2   s    zArrowFSWrapper.__init__c                 C   s   | j jS N)r   	type_namer    r   r   r   protocol8   s    zArrowFSWrapper.protocolc                 C   s   dt | jj| jj S )NZhdfs_)r	   r   hostportr%   r   r   r   fsid<   s    zArrowFSWrapper.fsidc                 C   s*   t |}|d }|dr&|dd  }|S )Npathz//   )r   
startswith)clsr*   opsr   r   r   _strip_protocol@   s
    
zArrowFSWrapper._strip_protocolFc                    sN     |}ddlm}  fdd j||D }|r<|S dd |D S d S )Nr   )FileSelectorc                    s   g | ]}  |qS r   )_make_entry.0entryr%   r   r   
<listcomp>M   s   z%ArrowFSWrapper.ls.<locals>.<listcomp>c                 S   s   g | ]}|d  qS )namer   r2   r   r   r   r5   T   s     )r/   
pyarrow.fsr0   r   get_file_info)r    r*   Zdetailr   r0   entriesr   r%   r   lsI   s    

zArrowFSWrapper.lsc                 K   s$   |  |}| j|g\}| |S r#   )r/   r   r8   r1   )r    r*   r   infor   r   r   r;   V   s    
zArrowFSWrapper.infoc                 C   s8   |  |}z| | W n tk
r.   Y dS X dS d S )NFT)r/   r;   r   r    r*   r   r   r   exists[   s    
zArrowFSWrapper.existsc                 C   sn   ddl m} |j|jkrd}n<|j|jkr0d}n*|j|jkrVttjt	
tj|jnd}|j|j||jdS )Nr   )FileType	directoryfileother)r6   sizetypemtime)r7   r>   rC   Z	DirectoryZFileZNotFoundr   r   r   osstrerrorr*   rB   rD   )r    r;   r>   kindr   r   r   r1   d   s    zArrowFSWrapper._make_entryc                 K   s   |  |d}|  |d}| |d}| dtd }z6| |d}t|| W 5 Q R X | j	|| W n6 t
k
r   tt | j| W 5 Q R X  Y nX W 5 Q R X d S )Nr   rbz.tmp.   wb)r/   rstrip_opensecretsZ	token_hexopenshutilcopyfileobjr   moveBaseExceptionr   r   delete_file)r    path1path2r   ZlstreamZ	tmp_fnameZrstreamr   r   r   cp_filew   s    
zArrowFSWrapper.cp_filec                 K   s2   |  |d}|  |d}| j|| d S )Nr   )r/   rK   r   rQ   )r    rT   rU   r   r   r   r   mv   s    zArrowFSWrapper.mvc                 C   s   |  |}| j| d S r#   )r/   r   rS   r<   r   r   r   rm_file   s    
zArrowFSWrapper.rm_fileNc                 C   sF   |  |d}| |r6|r,| j| qBtdn| j| d S )Nr   z0Can't delete directories without recursive=False)r/   rK   isdirr   
delete_dir
ValueErrorrS   )r    r*   	recursiveZmaxdepthr   r   r   rm   s    

zArrowFSWrapper.rmrH   Tc           	      K   s   |dkr |r| j j}qR| j j}n2|dkr2| j j}n |dkrD| j j}ntd|i }|dksb|sttdd dkrd |d< ||f|}t	| ||||f|S )	NrH   rJ   abz'unsupported mode for Arrow filesystem: .r      compression)
r   Zopen_input_fileZopen_input_streamZopen_output_streamZopen_append_streamr[   intr   split	ArrowFile)	r    r*   mode
block_sizeseekabler   method_kwargsstreamr   r   r   rL      s    



zArrowFSWrapper._openc                 K   s2   |  |}|r| j|dd n| jj|dd d S )NT)exist_okFr\   )r/   makedirsr   
create_dir)r    r*   Zcreate_parentsr   r   r   r   mkdir   s    
zArrowFSWrapper.mkdirc                 C   s   |  |}| jj|dd d S )NTrl   )r/   r   rn   )r    r*   rk   r   r   r   rm      s    
zArrowFSWrapper.makedirsc                 C   s   |  |}| j| d S r#   )r/   r   rZ   r<   r   r   r   rmdir   s    
zArrowFSWrapper.rmdirc                 C   s   |  |}| j|jS r#   )r/   r   r8   rD   r<   r   r   r   modified   s    
zArrowFSWrapper.modifiedc                    s&   |dk|d< t  j|fd d d|S )N)Nr   rg   )startend)r   cat_file)r    r*   rr   rs   r   r!   r   r   rt      s    zArrowFSWrapper.cat_filec                    s   d|d< t  j||f| d S )NFrg   )r   get_file)r    ZrpathZlpathr   r!   r   r   ru      s    zArrowFSWrapper.get_file)F)FN)rH   NT)T)F)NN)__name__
__module____qualname____doc__Zroot_markerr   propertyr&   r   r)   classmethodr/   r:   r;   r=   r1   r   rV   rW   Zmv_filerX   r]   rL   ro   rm   rp   rq   rt   ru   __classcell__r   r   r!   r   r   '   sD   



	





r   rj   readseektellwritereadablewritablecloserB   rg   c                   @   s&   e Zd ZdddZdd Zdd ZdS )	rd   Nc                 K   s.   || _ || _|| _|| _| | _| _|| _d S r#   )r*   re   r   rj   	blocksizerf   r   )r    r   rj   r*   re   rf   r   r   r   r   r      s    zArrowFile.__init__c                 C   s   | S r#   r   r%   r   r   r   	__enter__   s    zArrowFile.__enter__c                 G   s   |   S r#   )r   )r    r   r   r   r   __exit__   s    zArrowFile.__exit__)N)rv   rw   rx   r   r   r   r   r   r   r   rd      s   

rd   c                       s2   e Zd ZdZdZd
 fdd	Zedd	 Z  ZS )HadoopFileSystemz]A wrapper on top of the pyarrow.fs.HadoopFileSystem
    to connect it's interface with fsspecZhdfsdefaultr   Nc           	         s8   ddl m} ||||||d}t jf d|i| dS )a  

        Parameters
        ----------
        host: str
            Hostname, IP or "default" to try to read from Hadoop config
        port: int
            Port to connect on, or default from Hadoop config if 0
        user: str or None
            If given, connect as this username
        kerb_ticket: str or None
            If given, use this ticket for authentication
        extra_conf: None or dict
            Passed on to HadoopFileSystem
        r   )r   )r'   r(   userkerb_ticket
extra_confr   N)r7   r   r   r   )	r    r'   r(   r   r   r   r   r   r   r!   r   r   r      s    zHadoopFileSystem.__init__c                 C   sX   t | }i }|dd r$|d |d< |dd r<|d |d< |dd rT|d |d< |S )Nr'   usernamer   r(   )r   get)r*   r.   outr   r   r   _get_kwargs_from_urls  s    z&HadoopFileSystem._get_kwargs_from_urls)r   r   NNN)	rv   rw   rx   ry   r&   r   staticmethodr   r|   r   r   r!   r   r      s        #r   )r   iorE   rM   rO   
contextlibr   	functoolsr   r   Zfsspec.specr   Zfsspec.utilsr   r   r   r	   r   r   r   IOBaserd   r   r   r   r   r   <module>   s6    0