U f/ez @sxdZddlmZddlZddlZddlmZddlZddlZddl m Z m Z m Z m Z mZddlZddlZddlmZmZmZmZmZddlmZmZmZmZmZddlZddlZddl m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&dd l'm(Z(m)Z)dd l*m+Z+dd l,m-Z-e)Z.e/eeeZ0e01d ej2Gd ddZ3ej2GdddZ4ddddZ5dddddZ6ddddZ7dQdddddd Z8d!d"Z9d#dd$d%d&Z:dRd#d)d*d)d+dd,d-d.Z;d)d)d/d0d1Zd#d;d;d<d=d>Z?dSd#d)d;d*ddd;d+dd@ dAdBZ@GdCdDdDejAe ZBGdEdFdFejCZDdGdd)d)d;ddHdIdJdKZEd#dddLdMZFd#d)ddNdOdPZGdS)TzCommon IO api utilities) annotationsN)abc)BufferedIOBaseBytesIO RawIOBaseStringIO TextIOWrapper)IOAnyAnyStrMappingcast)urljoinurlparse uses_netloc uses_params uses_relative)BufferCompressionDictCompressionOptions FileOrBufferFilePathOrBufferStorageOptions) get_lzma_file import_lzma)import_optional_dependency) is_file_likec@s>eZdZUdZded<ded<ded<ded<d Zd ed <d S) IOArgsa) Return value of io/common.py:_get_filepath_or_buffer. Note (copy&past from io/parsers): filepath_or_buffer can be Union[FilePathOrBuffer, s3fs.S3File, gcsfs.GCSFile] though mypy handling of conditional imports is difficult. See https://github.com/python/mypy/issues/1297 rfilepath_or_bufferstrencodingmoder compressionFbool should_closeN)__name__ __module__ __qualname____doc____annotations__r%r+r+4/tmp/pip-unpacked-wheel-tiezk1ph/pandas/io/common.pyr:s  rc@szeZdZUdZded<ded<ejedZded<d Z d ed <d Z d ed <d dddZ ddddZ dd dddZ dS) IOHandlesau Return value of io/common.py:get_handle Can be used as a context manager. This is used to easily close created buffers and to handle corner cases when TextIOWrapper is inserted. handle: The file handle to be used. created_handles: All file handles that are created by get_handle is_wrapped: Whether a TextIOWrapper needs to be detached. rhandlerr#)default_factoryz list[Buffer]created_handlesFr$ is_wrappedis_mmapNonereturnc Csz|jr8t|jtst|j|j|j|jz|jD] }| q@Wnt t fk rhYnXg|_d|_dS)z Close all created buffers. Note: If a TextIOWrapper was inserted, it is flushed and detached to avoid closing the potentially user-created buffer. FN) r1 isinstancer.rAssertionErrorflushdetachr0removecloseOSError ValueError)selfr.r+r+r,r;as   zIOHandles.closecCs|SNr+r>r+r+r, __enter__uszIOHandles.__enter__r )argsr5cGs |dSr?)r;)r>rBr+r+r,__exit__xszIOHandles.__exit__N)r&r'r(r)r* dataclassesfieldlistr0r1r2r;rArCr+r+r+r,r-Ls    r-r$r4cCst|tsdSt|jtkS)z Check to see if a URL has a valid protocol. Parameters ---------- url : str or unicode Returns ------- isurl : bool If `url` has a valid protocol return True otherwise False. F)r6r parse_urlscheme _VALID_URLSurlr+r+r,is_url|s rLzFileOrBuffer[AnyStr])rr5cCst|trtj|S|S)a] Return the argument with an initial component of ~ or ~user replaced by that user's home directory. Parameters ---------- filepath_or_buffer : object to be converted if possible Returns ------- expanded_filepath_or_buffer : an expanded filepath or the input if not expandable )r6r ospath expanduser)rr+r+r, _expand_users  rPr3cCst|trtddS)NzPassing a bool to header is invalid. Use header=None for no header or header=int or list-like of ints to specify the row(s) making up the column names)r6r$ TypeError)headerr+r+r,validate_header_args rSFzFilePathOrBuffer[AnyStr])rconvert_file_liker5cCs6|st|rttt|St|tjr.|}t|S)a Attempt to convert a path-like object to a string. Parameters ---------- filepath_or_buffer : object to be converted Returns ------- str_filepath_or_buffer : maybe a string version of the object Notes ----- Objects supporting the fspath protocol (python 3.6+) are coerced according to its __fspath__ method. Any other object is passed through unchanged, which includes bytes, strings, buffers, or anything else that's not even path-like. ) rr rr r6rMPathLike __fspath__rP)rrTr+r+r,stringify_paths   rWcOsddl}|jj||S)z` Lazy-import wrapper for stdlib urlopen, as that imports a big chunk of the stdlib. rN)urllib.requestrequesturlopen)rBkwargsurllibr+r+r,rZsrZr)rKr5cCst|tod|ko|d S)zR Returns true if the given URL looks like something fsspec can handle z://)zhttp://zhttps://)r6r startswithrJr+r+r, is_fsspec_urls   r^utf-8rr rr)rr!r#r"storage_optionsr5c Cst|}t|\}}t||}|rHt|drHd|krHtjdtddd}t||d}|dk rl|dd  }d |kr|d kr|d krt|d |t |}d|krd|kr|d7}t |t r>t |r>|pi}ddl}|jj||d}t|2} | jdd} | dkrddi}t| } W5QRXt| ||d|dSt|rht |t sXt|drp|dd}|dr|dd}td} g} z&tdddlm}m}||tg} Wntk rYnXz$| j |fd|i|pi }Wn^t!| k rT|dkr ddi}nt|}d|d<| j |fd|i|pFi }YnXt|||d|dS|rvt"dt |t t#t$j$frtt%|||d|dSt&|sd t'|}t"|t|||d|dS)!a If the filepath_or_buffer is a url, translate and return the buffer. Otherwise passthrough. Parameters ---------- filepath_or_buffer : a url, filepath (str, py.path.local or pathlib.Path), or buffer compression : {{'gzip', 'bz2', 'zip', 'xz', None}}, optional encoding : the encoding to use to decode bytes, default is 'utf-8' mode : str, optional storage_options : dict, optional Extra options that make sense for a particular storage connection, e.g. host, port, username, password, etc., if using a URL that will be parsed by ``fsspec``, e.g., starting "s3://", "gcs://". An error will be raised if providing this argument with a local path or a file-like buffer. See the fsspec and backend storage implementation docs for the set of allowed keys and values .. versionadded:: 1.2.0 ..versionchange:: 1.2.0 Returns the dataclass IOArgs. writebzDcompression has no effect when passing a non-binary object as input.) stacklevelN)method_-w)bz2xz)zutf-16zutf-32z( will not write the byte order mark for tr)headerszContent-EncodinggziprfT)rr!r#r%r"zs3a://zs3://zs3n://fsspecZbotocore) ClientErrorNoCredentialsErrorr"Zanonz?storage_options passed with file object or non-fsspec file pathFz)Invalid file path or buffer object type: )(rWget_compression_methodinfer_compressionhasattrwarningswarnRuntimeWarningdictreplacelowerUnicodeWarningr6r rLrXrYRequestrZrmgetrreadrr^r7r]rZbotocore.exceptionsrprqPermissionError ImportErroropentupler=bytesmmaprPrtype)rr!r#r"racompression_methodZ fsspec_moder\Zreq_inforeqcontent_encodingreaderroZerr_types_to_retry_with_anonrprqZfile_objmsgr+r+r,_get_filepath_or_buffers!               r)rNr5cCsddlm}td||S)z converts an absolute native path to a FILE URL. Parameters ---------- path : a path in native format Returns ------- a valid FILE URL r) pathname2urlzfile:)rXrr)rNrr+r+r,file_path_to_urls rz.gzz.bz2z.zipz.xz)rnrjziprkz"tuple[str | None, CompressionDict])r#r5c Cs`t|trPt|}z|d}WqXtk rL}ztd|W5d}~XYqXXni}|}||fS)a Simplifies a compression argument to a compression method string and a mapping containing additional arguments. Parameters ---------- compression : str or mapping If string, specifies the compression method. If mapping, value at key 'method' specifies compression method. Returns ------- tuple of ({compression method}, Optional[str] {compression arguments}, Dict[str, Any]) Raises ------ ValueError on mapping missing 'method' key rfz.If mapping, compression must have key 'method'N)r6r rxpopKeyErrorr=)r#compression_argsrerrr+r+r,rrs rr str | None)rr#r5cCs|dkr dS|dkrZt|dd}t|ts.dStD]\}}||r6|Sq6dS|tkrf|Sddgtt}d|d|}t|dS)a Get the compression method for filepath_or_buffer. If compression='infer', the inferred compression method is returned. Otherwise, the input compression method is returned unchanged, unless it's invalid, in which case an error is raised. Parameters ---------- filepath_or_buffer : str or file handle File path or object. compression : {'infer', 'gzip', 'bz2', 'zip', 'xz', None} If 'infer' and `filepath_or_buffer` is path-like, then detect compression from the following extensions: '.gz', '.bz2', '.zip', or '.xz' (otherwise no compression). Returns ------- string or None Raises ------ ValueError on invalid compression specified. NZinferT)rTUnrecognized compression type: z Valid compression types are ) rWr6r _compression_to_extensionitemsrzendswithsortedr=)rr# extensionZvalidrr+r+r,rss$   rsT) path_or_bufr"r!r# memory_mapis_texterrorsrar5cCs|pd}t||r"d|kr"|d7}t|tr4|}|dkrTtd|dddt|||||d}|j} t| ||j|j ||j d t k\} }} t| t} t |j } | d }|r|j d d |_ |d kr| rt| tsttjf| |j d | } ntjf| |j d| } n|dkr:tj| fd|j i| } n|dkrt| |j f| } | j dkr| | | } t| dkr| | } n,t| dkrtd|ntd| n.|dkrtt| |j } nd|}t|t| trt| | nNt| trV|jr@d|j kr@t| |j |j|d d} n t| |j } | | d}|r|stt| |j rt| |j|d d} | | t|jtp|j }| |jrt|jtrt| |jt| trtt| | |||j dS)a Get file handle for given path/buffer and mode. Parameters ---------- path_or_buf : str or file handle File path or object. mode : str Mode to open path_or_buf with. encoding : str or None Encoding to use. compression : str or dict, default None If string, specifies compression mode. If dict, value at key 'method' specifies compression mode. Compression mode must be one of {'infer', 'gzip', 'bz2', 'zip', 'xz', None}. If compression mode is 'infer' and `filepath_or_buffer` is path-like, then detect compression from the following extensions: '.gz', '.bz2', '.zip', or '.xz' (otherwise no compression). If dict and compression mode is one of {'zip', 'gzip', 'bz2'}, or inferred as one of the above, other entries passed as additional compression options. .. versionchanged:: 1.0.0 May now be a dict with key 'method' as compression mode and other keys as compression options if compression mode is 'zip'. .. versionchanged:: 1.1.0 Passing compression options as keys in dict is now supported for compression modes 'gzip' and 'bz2' as well as 'zip'. memory_map : bool, default False See parsers._parser_params for more information. is_text : bool, default True Whether the type of the content passed to the file/buffer is string or bytes. This is not the same as `"b" not in mode`. If a string content is passed to a binary file/buffer, a wrapper is inserted. errors : str, default 'strict' Specifies how encoding and decoding errors are to be handled. See the errors argument for :func:`open` for a full list of options. storage_options: StorageOptions = None Passed to _get_filepath_or_buffer .. versionchanged:: 1.2.0 Returns the dataclass IOHandles r_rc) Nstrictignoreryxmlcharrefreplacebackslashreplace namereplacesurrogateescape surrogatepassz%Invalid value for `encoding_errors` (z). Please see z=https://docs.python.org/3/library/codecs.html#error-handlers zfor valid values.)r!r#r"rarfrlrrn)filenamer")fileobjr"rjr"rr`rzZero files found in ZIP file z9Multiple files found in ZIP file. Only one file per ZIP: rkrr!rnewlineF)r.r0r1r2r#)_is_binary_moder6r rzr=rr_maybe_memory_mapr!r"r#rrxrryr7rnGzipFilerjBZ2File _BytesZipFileappendnamelistlenrrlzmarr%reverser-)rr"r!r#rrrraZioargsr.handlesZis_pathrZ zip_namesrr1r+r+r, get_handle s<                   rcs^eZdZdZdddddfdd Zd d Zd d fd d ZfddZeddZ Z S)ra  Wrapper for standard library class ZipFile and allow the returned file-like handle to accept byte strings via `write` method. BytesIO provides attributes of file-like object and ZipFile.writestr writes bytes strings into a member of the archive. Nrr r)filer" archive_namec sB|dd}||_d|_dtji}||tj||f|dS)Nrcrr#)ryrmultiple_write_bufferzipfile ZIP_DEFLATEDupdatesuper__init__)r>rr"rr[Z kwargs_zip __class__r+r,rs    z_BytesZipFile.__init__cCs2|jdkr"t|trtnt|_|j|dSr?)rr6rrrrb)r>datar+r+r,rbs z_BytesZipFile.writer3r4c sP|jdks|jjrdS|jp$|jp$d}|jt||jW5QRXdS)Nr)rclosedrrrwritestrgetvalue)r>rrr+r,r8s z_BytesZipFile.flushcs|tdSr?)r8rr;r@rr+r,r;&sz_BytesZipFile.closecCs |jdkSr?)fpr@r+r+r,r*sz_BytesZipFile.closed)N) r&r'r(r)rrbr8r;propertyr __classcell__r+r+rr,rs  rc@sbeZdZdZddddddd d Zdd d d ZddddZddddddZddddZdS) _MMapWrappera Wrapper for the Python's mmap class so that it can be properly read in by Python's csv.reader class. Parameters ---------- f : file object File object to be mapped onto memory. Must support the 'fileno' method or have an equivalent attribute r_rTr r r$)fr!rdecodecCsn||_||_t||d|_||_i|_dD]"}t||s>q.t|||j|<q.t j | dt j d|_ dS)Nr)seekablereadableZ writeabler)access) r!rcodecsgetincrementaldecoderdecoderr attributesrtgetattrrfilenoZ ACCESS_READ)r>rr!rr attributer+r+r,r<s z_MMapWrapper.__init__)namecs$jkrfddStjS)Ncs jSr?)rr+rr>r+r,Qz*_MMapWrapper.__getattr__..)rrr)r>rr+rr, __getattr__Os z_MMapWrapper.__getattr__r4cCs|Sr?r+r@r+r+r,__iter__Tsz_MMapWrapper.__iter__intz str | bytes)sizer5cCs(|j|}|jr$|j|j|jdS|S)Nr)rr~rr!r)r>rcontentr+r+r,r~Ws z_MMapWrapper.readcCs,|j}|j|}|dkr"t|dS)Nr )rreadlinerr StopIterationlstrip)r>Znewbytesrr+r+r,__next__`s   z_MMapWrapper.__next__N)r_rT)r) r&r'r(r)rrrr~rr+r+r+r,r/s rrz'tuple[FileOrBuffer, bool, list[Buffer]])r.rr!r"rrr5cCsg}|t|dpt|tM}|s*|||fSt|trh|rTd|krTt||||dd}n t||}||z:ttjt||||}|| ||||}Wnt k rd}YnX|||fS)zTry to memory map file/buffer.rrcrrF) rtr6r rrr rrr;r: Exception)r.rr!r"rrrwrappedr+r+r,rqs*          rc CsHd}t|}t|ts|Sztj|}Wnttfk rBYnX|S)zTest whether file exists.F)rWr6r rMrNexistsrQr=)rrr+r+r, file_existss r)r.r"r5cCs\d|ksd|krd|kStjtjtjf}tt||r:dSttf}t||pZdt |d|kS)z+Whether the handle is opened in binary moderlrcFr") r StreamWriter StreamReaderStreamReaderWriter issubclassrrrr6r)r.r"Z text_classesZbinary_classesr+r+r,rsr)F)r_Nr`N)NNFTNN)Hr) __future__rrjr collectionsrrDrniorrrrrrrMtypingr r r r r urllib.parserrrGrrrrurZpandas._typingrrrrrrZ pandas.compatrrZpandas.compat._optionalrZpandas.core.dtypes.commonrrsetrIdiscardZ dataclassrr-rLrPrSrWrZr^rrrrrrsrZipFilerIteratorrrrrr+r+r+r,sl      / " 5#> n9B-