U /ea @sUddlmZddlZddlZddlZddlZddlZddlmZm Z ddl m Z ddl m Z ddlmZmZmZddlZddlZddlmZmZddlmZmZdd lmZdd lmZm Z dd l!m"Z"m#Z#m$Z$dd l%m&Z&m'Z'm(Z(dd l)m*Z*ddl+m,Z,m-Z-m.Z.m/Z/m0Z0ej1ej2ej3ej4fZ5de6d<zddl7m8Z9e5e9j:f7Z5Wne;k r\YnXddZdZ?dZ@ededZAedddddZBeddd d!dZBd"dZBe d]d$d%ZCd&ZDd'd(ZEd^d)d*ZFd_d,d-ZGd`d.d/ZHeId+eJd0eKd1eLd2eMd3eMd3eNd3d3d4ZOd5d6ZPd7d8ZQdad9d:ZRd;d<ZSdbd=d>ZTdcd?d@ZUdAdBdCdDZVdddFdGZWdHdIZXdedJdKZYdLdMZZdfdNdOZ[dgdPdQZ\dRdSZ]dhdUdVZ^GdWdXdXe_e`ZadYdZZbd[d\ZcdS)i) annotationsN)IteratorMapping)contextmanager)Number)CallableTypeVaroverload)is_categorical_dtypeis_dtype_equal) get_scheduleris_dask_collection)get_deps)_dtypesmethods) PANDAS_GT_110 PANDAS_GT_120tm) make_meta make_meta_obj meta_nonempty) make_scalar) asciitableis_dataframe_like is_index_likeis_series_liketypenameztuple[type, ...]meta_object_typescCs:t|d|}tjtjtjtjtjtjtjtj f}t ||S)Ndtype) getattrpdZ Int8DtypeZ Int16DtypeZ Int32DtypeZ Int64DtypeZ UInt8DtypeZ UInt16DtypeZ UInt32DtypeZ UInt64Dtype isinstancetrtypesr%8/tmp/pip-unpacked-wheel-dbjnr7gq/dask/dataframe/utils.pyis_integer_na_dtype/s  r'cCs*tsdSt|d|}tjtjf}t||S)NFr)rrr Z Float32DtypeZ Float64Dtyper!r"r%r%r&is_float_na_dtype>s r(ccst|trt|}t|s"|Vnt|}|}|j}t|rJ| }| |}|j d|dVt t|dD] }|j ||||dVqx|j |ddVdS)aLShard a DataFrame by ranges on its index Examples -------- >>> df = pd.DataFrame({'a': [0, 10, 20, 30, 40], 'b': [5, 4 ,3, 2, 1]}) >>> df a b 0 0 5 1 10 4 2 20 3 3 30 2 4 40 1 >>> shards = list(shard_df_on_index(df, [2, 4])) >>> shards[0] a b 0 0 5 1 10 4 >>> shards[1] a b 2 20 3 3 30 2 >>> shards[2] a b 4 40 1 >>> list(shard_df_on_index(df, []))[0] # empty case a b 0 0 5 1 10 4 2 20 3 3 30 2 4 40 1 Nr) r!rlistlennparray sort_indexindexr Z as_orderedZ searchsortedilocrange)df divisionsr0indicesir%r%r&shard_df_on_indexJs'   r7z?meta : pd.DataFrame, pd.Series, dict, iterable, tuple, optionalaAn empty ``pd.DataFrame`` or ``pd.Series`` that matches the dtypes and column names of the output. This metadata is necessary for many algorithms in dask dataframe to work. For ease of use, some alternative inputs are also available. Instead of a ``DataFrame``, a ``dict`` of ``{name: dtype}`` or iterable of ``(name, dtype)`` can be provided (note that the order of the names should match the order of the columns). Instead of a series, a tuple of ``(name, dtype)`` can be used. If not provided, dask will try to infer the metadata. This may lead to unexpected results, so providing ``meta`` is recommended. For more information, see ``dask.dataframe.utils.make_meta``. T)bound)funcreturncCsdSNr%)r:r%r%r&insert_meta_param_descriptionsr=intzCallable[[T], T])padr;cCsdSr<r%)r?r%r%r&r=sc s|sfddS|d}ddd}tjt||dd}d td |}|jrd |jkrn|jd ||_nPd |d d}t d|j\}}| dd\} } d||| |d d|| |_|S)zReplace `$META` in docstring with param description. If pad keyword is provided, will pad description by that number of spaces (default is 8).cs t|fSr<)r=)fkwargsr%r&z/insert_meta_param_description..r r?N)initial_indentsubsequent_indentwidthz{} {} z$METAzParameters %s----------NzParameters\n[ ]*----------z r)z{}{}{} {}{} {}) gettextwrapwrap_META_DESCRIPTIONformat _META_TYPESjoin__doc__replaceresplit) argsrBr@indentbodydescrZparameter_headerfirstlast parametersrestr%rAr&r=s4   Fc csz dVWntk r}zft\}}}dt|}d}|rJ|d7}|d7}||rfd|dndt||}t||W5d}~XYnXdS)zReraise errors in this block to show metadata inference failure. Parameters ---------- funcname : str, optional If provided, will be added to the error message to indicate the name of the method that failed. NzMetadata inference failed{0}. zYou have supplied a custom function and Dask is unable to determine the type of output that that function returns. To resolve this please provide a meta= keyword. The docstring of the Dask function you ran should have more information. zOOriginal error is below: ------------------------ {1} Traceback: --------- {2}z in ``) Exceptionsysexc_inforS traceback format_tbrQrepr ValueError)funcnameZudfeexc_type exc_value exc_tracebacktbmsgr%r%r&raise_on_meta_errors  "rpZ__UNKNOWN_CATEGORIES__cCsHt|d|}t|r t|jjkSt|r}t||sF|rp||j j t ddqF||j g||<qFn*t|tjrt |jrt|s|j g}t|jtjrt|js|j g|_nt|tjrt|s| g}|S)zReplace any unknown categoricals with empty categoricals. Useful for preventing ``UNKNOWN_CATEGORIES`` from leaking into results. categoryT)inplace)r!r Series DataFramecopydtypesanyr0ryrtZremove_categoriesrsset_categoriesr rCategoricalIndex)rxZjust_drop_unknownZcat_maskZcatscr%r%r&strip_unknown_categoriess*       rTcCst|tjtjfr|}t|tjr||jdk}|dkrD||j}n|j|sZt d|D]}||j t g||<q^n$t|tjrt |jr|j t g}|rt|jtjr|j t g|_nt|tjr| t g}|S)aSet categories to be unknown. Parameters ---------- x : DataFrame, Series, Index cols : iterable, optional If x is a DataFrame, set only categoricals in these columns to unknown. By default, all categorical columns are set to unknown categoricals index : bool, optional If True and x is a Series or DataFrame, set the clear known categories in the index as well. rzNz Not all columns are categoricals)r!r r|r}r~rr0locallrhrtrrsr rr)rxcolsr0maskrr%r%r&clear_known_categories s$        rcCsZt|trH|dkrHtjttg|djdd}|dk rDt||_|Stjg|||dS)Nrz)namer)rrr0) r!strr r|Z Categoricalrsr1rr0)rrr0sr%r%r& _empty_series+s   r z 1970-01-01r)Zfoo)bVMmSaUOcCsl|jdkr|dS|jdkr.|tddS|jtkrZt|j}|jdkrV||S|Std|dS)N)r6r@ur)rr)rrzCan't handle dtype: )kindtypecomplex_simple_fake_mappingZastyperv)ror%r%r&_scalar_from_dtype@s     rcCsbt|tjkrt|St|rFt|dr0|jn tt|}t|Stdtt|ddS)NrzCan't handle meta of type '') rrZ_lookupr-isscalarrurrvr)rxrr%r%r&_nonempty_scalarLs  rcsT|rdddhntfddt|s8t|s8t|r@t|rTtdtt||j|jkr~d tt|tt|}nt|rt j |j |j gdd d }fd d | d D}|rd tt|tdddg|}nt|||SnB|j|jr|Sd tt|tddgd|jfd|jfg}td|rDd|nd|fdS)aCheck that the dask metadata matches the result. If metadata matches, ``x`` is passed through unchanged. A nice error is raised if metadata doesn't match. Parameters ---------- x : DataFrame, Series, or Index meta : DataFrame, Series, or Index The expected metadata that ``x`` should match funcname : str, optional The name of the function in which the metadata was specified. If provided, the function name will be included in the error message to be more helpful to users. numeric_equal : bool, optionl If True, integer and floating dtypes compare equal. This is useful due to panda's implicit conversion of integer to floating upon encountering missingness, which is hard to infer statically. r6r@rcst|t|krdSt|tr&|dks8t|tr<|dkr.equal_dtypesz>Expected partition to be DataFrame, Series, or Index, got `%s`z,Expected partition of type `{}` but got `{}`r)T)axissortcs*g|]"\}}}||st|||fqSr%)rg).0colrr)rr%r& s zcheck_meta..rzPartition type: `{}` {}ZColumnFoundZExpectedr`rzMetadata mismatch found%s. %sz in `%s`N)setrrrr rvrr __class__rQr concatrZfillna itertuplesrcheck_matching_columnsrrh)rxmetari numeric_equalerrmsgrZ bad_dtypesr%)rrr& check_metaWsV          rcCsttt|jt|jspt|j|j}t|j|j}|sL|r^d|d|}nd}td|dS)Nz Extra: z Missing: zOrder of columns does not matchzSThe columns in the computed data do not match the columns in the provided metadata )r-Z array_equalZ nan_to_numcolumnsrtolist differencerh)ractualextramissingZ extra_infor%r%r&rsrcCsVt|}|dkr|jj}|r>|d}|d}d|d|}nd}|d|d|S) z&Summarized representation of an Index.Nrr*z, z to r`z: z entries)r,r__name__)idxrnheadtailsummaryr%r%r& index_summarysrcCsddlm}t|dr|}t|dr2||dkrF|j|d}t||jrdt|j kslt t||r|j |j kst |j j |j kst t|t jr|j|j jkst |rt||nt||jrldt|j kst t|t|j t|ks t t|j |rD|j |j ks0t |j |j f|j j |j ksDt |rTt||t|j|||jdnt||jr$dt|j kst t|t|jt jst t|jt|j t|kst t|j |rt|j|jt|j j|j|rt||t|j|||jdn^t||jjrjt|sXt|t jt jfsXt |rt||nd t|d }t ||S|S) Nr__dask_graph__validate schedulerIndexr|) check_names check_dtypesresultr}zUnsupported dask instance z found)dask.dataframeZ dataframerurrZcomputer!rrrAssertionErrorrrqr MultiIndexnamesassert_dask_dtypesr| _check_daskr0r}rrassert_index_equalcoreZScalarr-rZ TimestampZ Timedelta)Zdskrrrrddgraphror%r%r&rsp      "  "   rbool) check_indexc Csz^t|rTt|jjt|j@r>ddtt|jjD|j_|jt |jd}n|}Wnt t t fk rxYnX|r| S|S)NcSsg|] }d|qS)z-overlapped-index-name-%dr%)rr6r%r%r&r sz_maybe_sort..)Zby)rrr0rrr2r,Z sort_valuesrrrv IndexErrorrhr/)rrr%r%r& _maybe_sorts  rsyncc Ks|rtt||dt||dt|drtt|drttt|jd} tt|jd} | | kstt| | ft|t|t ||||d}t ||||d}t|dr| }t|dr| }t |t j t jfr|rt||}t||}|s|jdd}|jdd}t |t j r:tj||f||d|nt |t jrdtj||f||d|nft |t jrtj||fd |i|n>||krdSt|rt|stnt||stdS) Nrr4r)rrr to_pandasT)drop)r check_dtypeexact)assert_divisionsrurr-Zasarrayr4rrassert_sane_keynamesrrr!r r}r|rZ reset_indexrZassert_frame_equalZassert_series_equalrrisnanZallclose) rrrrZcheck_divisionsrZ sort_resultsrrBatZbtr%r%r& assert_eqsv           rcCs\t|dr|j}t|tst|D]&}t|tr8|d}||r"dSq"td|dS)NdaskrTz(given dask graph doesn't contain label: )rurr!rrtuple startswith)rlabelkr%r%r&assert_dask_graphNs   rcCst|dsdSt|jtstt|dds.dSdd}t|t|gd}||j| }t |ddD]H\}}t |rh|| |j|kst|| |j|dkshtqht |dr||d |jd kst||d |jdkstdS) Nr4Zknown_divisionsFcSs:t|r |Sz|jdWStk r4|jYSXdS)Nr)rr0Zget_level_valuesAttributeErrorrwr%r%r&r0cs zassert_divisions..index)r collectionsr*r))rur!r4rrrr rrZ __dask_keys__ enumerater,minmax)ddfrr0rMresultsr6r3r%r%r&rZs    rcCs~t|dsdS|jD]`}t|tr0|d}qt|ttfsBtt|dksRtd|ks^t| dd st|qdS)NrrdrEr) rurkeysr!rrbytesrr,rW isidentifier)rrr%r%r&rws    rcs ddddhg|r"dddhfdd }t|sxt|rxtj|jj|jgd d jd d D]\}}|||s^tq^nt|st |st |r|jj }|j }|||stnnt |jdr|jj }t |dst |stt t|}n|j }|||stnt|jt|kstdS)aCheck that the dask metadata matches the result. If `numeric_equal`, integer and floating dtypes compare equal. This is useful due to the implicit conversion of integer to floating upon encountering missingness, which is hard to infer statically.rrrrr6r@rcs tfddDpkS)Nc3s"|]}j|koj|kVqdSr<)r)rrrr%r& sz8assert_dask_dtypes..eq_dtypes..)rrZ eq_type_setsrr& eq_dtypess z%assert_dask_dtypes..eq_dtypesr))rF)r0rN)appendr rr rrqrrrrrrrur-rr)rresrrrrr%rr&rs*  rcCsLt|j\}}|r.ttt||ksHtnttt||ksHtdSr<)rrrmapr,valuesr)rxreqZ dependenciesZ dependentsr%r%r&assert_max_depssrcCst|ttfsdSt|ddD]8\}}|||dkr@dSt|tr"t|r"dSq"|ddD]}t|trht|rhdSqh|d|dkrdSdS)aRAre the provided divisions valid? Examples -------- >>> valid_divisions([1, 2, 3]) True >>> valid_divisions([3, 2, 1]) False >>> valid_divisions([1, 1, 1]) False >>> valid_divisions([0, 1, 1]) True >>> valid_divisions(123) False >>> valid_divisions([0, float('nan'), 1]) False FNrr)r*T)r!rr+rrmathr)r4r6rxr%r%r&valid_divisionssrraisecCs4|jdd}tjj|s |g}|j|d|d|S)z)Use shallow copy to drop columns in placeF)deepT)rr{errors)r~r apir$Z is_list_liker)r3rrZdf2r%r%r&drop_by_shallow_copys  rc@seZdZdZdS)AttributeNotImplementedErrorz&NotImplementedError and AttributeErrorN)r __module__ __qualname__rTr%r%r%r&rsrcCs~t|r:z |j}Wn&tk r8tt|dYnXt|rH|jSt|rV|jSt |rh| jStt|ddS)zReturn a serial DataFrame constructor Parameters ---------- like : Any series-like, Index-like or dataframe-like object. z( not supported by meta_frame_constructorN) r rqrrvrr _constructorrZ_constructor_expanddimrto_frameliker%r%r&meta_frame_constructors  r cCs~t|r:z |j}Wn&tk r8tt|dYnXt|rH|jSt|rV|jSt |rh| jStt|ddS)zReturn a serial Series constructor Parameters ---------- like : Any series-like, Index-like or dataframe-like object. z) not supported by meta_series_constructorN) r rqrrvrrZ_constructor_slicedrrrrrr%r%r&meta_series_constructors  r )NF)F)NT)N)NT)N)TTNN)TTTTTr)N)T)T)r)d __future__rrrVrcrNrecollections.abcrr contextlibrZnumbersrtypingrrr Znumpyr-Zpandasr Zpandas.api.typesr r Z dask.baser r Z dask.corerrrrZdask.dataframe._compatrrrZdask.dataframe.dispatchrrrZdask.dataframe.extensionsrZ dask.utilsrrrrrr|r}rrr__annotations__Z scipy.sparsesparsespZspmatrix ImportErrorr'r(r7rRrPr8r=rprsryrrrZbool_voidZ datetime64Z timedelta64Zstr_Zunicode_rrrrrrrrrrrrrrrrNotImplementedErrorrrr r r%r%r%r&s       8  "  !    M  = 9  & %