U /e@sddlZddlmZmZddlmZddlZddlZddl m Z ddl m Z zddlmZWnek rtdZYnXddZdd Zdd d Zd dZddZddZdddZeGdddeZdS)N)Protocolruntime_checkable)uuid4)LocalFileSystem)parsecCs|ot|tpt|S)z'Check if an fsspec file-system is local) isinstancer_is_local_fs_pyarrowfsr ;/tmp/pip-unpacked-wheel-dbjnr7gq/dask/dataframe/io/utils.py _is_local_fss r cCs0|r,t|drt|jSt|dr,|jdkSdS)z-Check if a pyarrow-based file-system is localr type_namelocalF)hasattrrr rr r r r rs     rFc Cs0|rddlm}|j}ndd}|jdk o2d|jk}|r|t|jdd}dd |d gD}d d |d gD}ni}i} tt|D]} || } | j |kr| j |krt j gd d j || j j} n || j } n*z|| j} Wntk rYqYnX| | | j <q|r,|D]} d| | <q| S)z4Convert a pyarrow.Schema object to pandas dtype dictr)PYARROW_NULLABLE_DTYPE_MAPPINGcSs|S)N)Zto_pandas_dtype)tr r r ,z%_get_pyarrow_dtypes..Nspandasutf8c Ss&i|]}|d|dd|dqS) field_namenameNZ numpy_typeget.0cr r r 2sz'_get_pyarrow_dtypes..columnsc SsBi|]:}|ddkr|dr|d|dd|dddqS)Z pandas_type)datetimeZ datetimetzmetadatarrNtimezonerrr r r r6s zM8[ns]dtypecategory)Zdask.dataframe.io.parquet.arrowrrr jsonloadsdecoderangelenrpdSeriesdtZ tz_localizer#typeNotImplementedError)Zschema categoriesZuse_nullable_dtypesrZ type_mapperZhas_pandas_metadataZpandas_metadataZpandas_metadata_dtypestzZdtypesifieldZ numpy_dtypecatr r r _get_pyarrow_dtypes%s>         r4csfdd|Dfdd|p"gD}t|dkr:d}n.csg|]}|qSr )popr)datar r ssz%_meta_from_dtypes..rNZ__index_level_0__)names)index)r)rr*Z MultiIndexZ from_arraysZ DataFramerr:)Zto_read_columnsr5Z index_colsZcolumn_index_namesZindexesr;Zdfr )r7r5r _meta_from_dtypes[s     r<cCstjS)z0Simple utility function to get random hex string)rhexr r r r _guidsr>cCs|dkr |S||S)z5Helper function to place an object on a context stackN) enter_context)objstackr r r _set_contextsrBc  sdk rfdd|DS|p$i}|dd}|dkrĈdk rtsttjtdkrĈ|ddpdgt|}dd }|d krtd |d fd dt ||DSdk rfd d|DSfdd|DS)aReturn a list of open-file objects given a list of input-file paths. WARNING: This utility is experimental, and is meant for internal ``dask.dataframe`` use only. Parameters ---------- paths : list(str) Remote or local path of the parquet file fs : fsspec object, optional File-system instance to use for file handling context_stack : contextlib.ExitStack, Optional Context manager to use for open files. open_file_func : callable, optional Callable function to use for file opening. If this argument is specified, ``open_file_func(path, **kwargs)`` will be used to open each file in ``paths``. Default is ``fs.open``. precache_options : dict, optional Dictionary of key-word arguments to use for precaching. If ``precache_options`` contains ``{"method": "parquet"}``, ``fsspec.parquet.open_parquet_file`` will be used for remote storage. **kwargs : Key-word arguments to pass to the appropriate open function Ncsg|]}t|fqSr )rBrpath) context_stackkwargsopen_file_funcr r r8sz%_open_input_files..methodparquetz 2021.11.0 row_groups cache_typepartsz<'parts' `cache_type` required for 'parquet' precaching, got .cs.g|]&\}}ttj|f|dqS))r rJ)rBfsspec_parquetZopen_parquet_file)rrDZrgsrEr rFr r r8s cs g|]}tj|fqSr rBopenrCrOr r r8scsg|]}tt|fqSr rPrC)rErFr r r8s) copyr6r parse_versionfsspec __version__updater) ValueErrorzip) pathsr rErGZprecache_optionsrFZprecacherJrKr )rEr rFrGr _open_input_filess6#       rZc@s,eZdZdZeddZddZddZdS) DataFrameIOFunctionzkDataFrame IO function with projectable columns Enables column projection in ``DataFrameIOLayer``. cCstdS)z$Return the current column projectionNr.)selfr r r rszDataFrameIOFunction.columnscCstdS)zUReturn a new DataFrameIOFunction object with a new column projection Nr\)r]rr r r project_columnssz#DataFrameIOFunction.project_columnscOstdS)z Return a new DataFrame partitionNr\)r]argsrFr r r __call__szDataFrameIOFunction.__call__N)__name__ __module__ __qualname____doc__propertyrr^r`r r r r r[s  r[)F)NNNN)r%typingrruuidrrTZpandasr*Zfsspec.implementations.localrZpackaging.versionrrSZfsspec.parquetrIrN ImportErrorr rr4r<r>rBrZr[r r r r s.      6*  N