U /e}$@sddlmZddlmZddlZddlmZddlm Z ddl m Z m Z ddl mZddlmZdd lmZmZmZdd lmZmZmZd d Zd dZddZdddZGdddeZdS)) defaultdict)IntegralN) is_scalar) partition_all)compute_as_if_collectiontokenize)methods)Accessor)categorical_dtypecategorical_dtype_dispatchis_categorical_dtype)AttributeNotImplementedErrorclear_known_categorieshas_known_categoriescCs|}|D]N\}}t||r:||j|||<qt|||dd}|||||<q|dk rt|jr|j|}nt|j|dd}|jj|d}|jj|_||_|S)zCategorize a dataframe with given categories df: DataFrame categories: dict mapping column name to iterable of categories F)meta categoriesorderedN)dtype) copyitemsr catset_categoriesr Zastypeindexname)dfrrcolvalsZ cat_dtypeindr>/tmp/pip-unpacked-wheel-dbjnr7gq/dask/dataframe/categorical.py_categorize_blocks$   r cCsvi}|D]8}||}t|r0||jj||<q|||<q|rnt|jr\||jjfS||jfS|dfS)N)r Z _constructorrrdropnadrop_duplicatesr)rcolumnsrresrxrrr_get_categories1s  r&cCstt}g}|D]6}|dD]\}}|||q ||dqdd|D}|ddkrn|dfS||d|ddfS)NrcSs$i|]\}}|tj|ddqS)T)Z ignore_index)rconcatr".0kvrrr Gsz'_get_categories_agg..)rlistrappendr")partsr$Zres_indpr+r,rrr_get_categories_agg@s r2c s|jdkr$tddgjntr2gfddDdk rxtjrdtj ndkrxjjt kt sdkr|S|dkrd}n*|dkr|j }nt |t r|dkrtd t||}d |fd d t|D}d |}|j }d} ||krt|t| } tt|t|D](\} } tfdd| Df|| | f<q2| d}| | d7} q tfddt|Df||df<||jt|j||dff|\} dd | D} |t| S)a0Convert columns of the DataFrame to category dtype. Parameters ---------- columns : list, optional A list of column names to convert to categoricals. By default any column with an object dtype is converted to a categorical, and any unknown categoricals are made known. index : bool, optional Whether to categorize the index. By default, object indices are converted to categorical, and unknown categorical indices are made known. Set True to always categorize the index, False to never. split_every : int, optional Group partitions into groups of this size while performing a tree-reduction. If set to False, no tree-reduction will be used. Default is 16. kwargs Keyword arguments are passed on to compute. Nobjectcategorycs(g|] }t|r t|s|qSr)r r)r*c)rrr ks zcategorize..Fz#split_every must be an integer >= 2zget-categories-chunk-cs"i|]\}}|ft|fqSr)r&)r*ikey)ar#rrrr-szcategorize..zget-categories-agg-rcsg|] }|fqSrrr*r9r;rrr6sr'csg|] }|fqSrrr<r=rrr6scSsi|]\}}||qSr)Z sort_valuesr)rrrr-s)_metar.Z select_dtypesr#rr rrrr3lenZ npartitions isinstancer ValueErrorr enumerateZ __dask_keys__strrranger2updateZdaskr __class__rmap_partitionsr )rr#rZ split_everykwargstokenZdskprefixr+depthbZpart_iZindsrr)r;r#rrr categorizePs^      " " rMc@sdeZdZdZdZdZdZeddZddZ d d Z ed d Z ed dZ eddZ ddZdS)CategoricalAccessora Accessor object for categorical properties of the Series values. Examples -------- >>> s.cat.categories # doctest: +SKIP Notes ----- Attributes that depend only on metadata are eager * categories * ordered Attributes depending on the entire dataset are lazy * codes * ... So `df.a.cat.categories` <=> `df.a._meta.cat.categories` So `df.a.cat.codes` <=> `df.a.map_partitions(lambda x: x.cat.codes)` r)Zadd_categoriesZ as_orderedZ as_unorderedZremove_categoriesZrename_categoriesZreorder_categoriesrrcCs t|jS)z&Whether the categories are fully known)r_seriesselfrrrknownszCategoricalAccessor.knowncKs.|jr |jS|djf|}||jS)aLEnsure the categories in this series are known. If the categories are known, this is a no-op. If unknown, the categories are computed, and a new series with known categories is returned. Parameters ---------- kwargs Keywords to pass on to the call to `compute`. r)rRrO _property_mapuniquecomputervalues)rQrHrrrras_knowns zCategoricalAccessor.as_knowncCs&|js |jS|j}t|j|_|S)z0Ensure the categories in this series are unknown)rRrOrrr>)rQoutrrr as_unknowns   zCategoricalAccessor.as_unknowncCs||jjddS)z3Whether the categories have an ordered relationshiprr)_delegate_propertyrOr>rPrrrrszCategoricalAccessor.orderedcCs$|jsd}t|||jjddS)zZThe categories of this categorical. If categories are unknown, an error is raisedz`df.column.cat.categories` with unknown categories is not supported. Please use `column.cat.as_known()` or `df.categorize()` beforehand to ensure known categoriesrr)rRr rZrOr>rQmsgrrrrs zCategoricalAccessor.categoriescCs|jsd}t||dS)zUThe codes of this categorical. If categories are unknown, an error is raisedz`df.column.cat.codes` with unknown categories is not supported. Please use `column.cat.as_known()` or `df.categorize()` beforehand to ensure known categoriescodes)rRr rSr[rrrr]s zCategoricalAccessor.codesc Cs|j}t|}t|jjtjr6|jj}n |jjj }| |j \}}|dkr^|jS||dk}|j ||j d}|jj|jdddd|i|dd S) z Removes categories which are not used Notes ----- This method requires a full scan of the data to compute the unique values, which can be expensive. N)rrrrnew_categorieszcat-set_categories)rrI)rOr!rTpdZIndexrUr@r>ZCategoricalIndexrZreindexrrrrGZ_delegate_method)rQZpresentZmeta_catrmaskr_rrrrremove_unused_categoriess&    z,CategoricalAccessor.remove_unused_categoriesN)__name__ __module__ __qualname____doc__Z_accessor_nameZ_accessor_methodsZ_accessor_propertiespropertyrRrWrYrrr]rbrrrrrNs     rN)NNN) collectionsrZnumbersrZpandasr`Zpandas.api.typesrZtlzrZ dask.baserrZdask.dataframerZdask.dataframe.accessorr Zdask.dataframe.dispatchr r r Zdask.dataframe.utilsr rrr r&r2rMrNrrrrs       S