U f/e@sddlmZddlmZddlZddlmZmZmZm Z m Z m Z m Z ddl Z ddlZddlmZmZddlmZddlmZmZmZmZmZddlmZdd lmZdd l m!Z!dd l"m#Z#m$Z$m%Z%m&Z&dd l'm(Z(dd l)m*Z*m+Z+ddl,m-Z-m.Z.ddl/m0m1Z2ddl3m4Z4ddl5m6Z6ddl7m8Z8m9Z9ddl:m;Z;ddlZ>m?Z?ddl@mAZAmBZBmCZCddlDmEZEmFZFmGZGmHZHmIZImJZJmKZKmLZLmMZMddlNmOZOmPZPe dddZQGdddeAZRGdddejSeRZSGdddeReBZTdTd d!d"dd#d$d%ZUd&d'ZVdUd(d!d"dd)d*d+ZWdVd,d-d!d.d/d0d1ZXd2d(d!d"d d3d4d5ZYd"d d6d7d8ZZdWd9d"d:d;d<Z[d9d d=d>d?Z\d@dAdBdCZ]dDd dEdFdGZ^d dHd"d dIdJdKZ_dLdLdMdNdOZ`dPd,d"dQdRdSZadS)X) annotations) defaultdictN)AnyCallable DefaultDictHashableSequenceTypeVarcast) internalslib)BlockPlacement) ArrayLikeDtypeDtypeObjShapetype_t)PerformanceWarning)validate_bool_kwarg)infer_dtype_from_scalar)ensure_platform_intis_1d_only_ea_dtypeis_dtype_equal is_list_like)ExtensionDtype) ABCDataFrame ABCSeries) array_equalsisna)NDArrayBackedExtensionArray) SparseDtype)ensure_wrapped_if_datetimelike extract_array)maybe_convert_indices) Float64IndexIndex ensure_index) DataManagerSingleDataManagerinterleaved_dtype) BlockCategoricalBlockDatetimeTZBlockExtensionBlockensure_block_shape extend_blocksget_block_typemaybe_coerce_values new_block) blockwise_alloperate_blockwiseTBaseBlockManager)boundc @sneZdZUdZdZded<ded<ded<ded <d ed <d ed <d ed<dddZedddddddZe ddZ e ddZ ddddddZ d d d!d"Z e Zd d d#d$d%Zd d&d'd(d)d*Ze d d d+d,Zd'd d-d.Ze d&d d/d0Zd1d2Ze d3d d4d5Zd6d d7d8Zddd:d dd;dd?d@ZddddAdBZdd dCdDdEZdd d ddFdGdHZddddIdJZdd d ddKdLdMZdd ddNdOdPZddddQdRZddd d6ddTdUdVZ ddd d d d ddWdXdYZ!dd d ddZd[d\Z"ddd]d]d d dd^d_d`Z#ddddadbZ$d d dcddZ%d'd dedfZ&e d d dgdhZ'e d d didjZ(e d d dkdlZ)ddd ddmdndoZ*ddd ddmdpdqZ+dddd drddsdtduZ,e d d dvdwZ-dddddxdyZ.ddddzd{Z/d'd d|d}Z0ddd&d d d d d dd~ddZ1e2j3d9fdd ddddZ4ddddddZ5ddd d ddddZ6dS)r6a Core internal data structure to implement DataFrame, Series, etc. Manage a bunch of labeled 2D mixed-type ndarrays. Essentially it's a lightweight blocked set of labeled data to be manipulated by the DataFrame public API class Attributes ---------- shape ndim axes values items Methods ------- set_axis(axis, new_labels) copy(deep=True) get_dtypes apply(func, axes, block_filter_fn) get_bool_data get_numeric_data get_slice(slice_like, axis) get(label) iget(loc) take(indexer, axis) reindex_axis(new_labels, axis) reindex_indexer(new_labels, indexer, axis) delete(label) insert(loc, label, value) set(label, value) Parameters ---------- blocks: Sequence of Block axes: Sequence of Index verify_integrity: bool, default True Notes ----- This is *not* a public API class np.ndarray_blknos_blklocstuple[Block, ...]blocks list[Index]axesintndimbool_known_consolidated_is_consolidatedTcCstdSNNotImplementedError)selfr=r?verify_integrityr8r8B/tmp/pip-unpacked-wheel-tiezk1ph/pandas/core/internals/managers.py__init__szBaseBlockManager.__init__z type_t[T] list[Block]r5)clsr=r?returncCstdSrErFrMr=r?r8r8rJ from_blocksszBaseBlockManager.from_blockscCs|jdkr||jS)a  Suppose we want to find the array corresponding to our i'th column. blknos[i] identifies the block from self.blocks that contains this column. blklocs[i] identifies the column of interest within self.blocks[self.blknos[i]] N)r:_rebuild_blknos_and_blklocsrHr8r8rJblknoss zBaseBlockManager.blknoscCs|jdkr||jS)z$ See blknos.__doc__ N)r;rQrRr8r8rJblklocss zBaseBlockManager.blklocsN)rHrNcCs|dkr tgg|jdd}|jdkrtt|ts8t|jd}|jdd}tt dd}|j ||d}|g}ng}t | ||S)z9return an empty BlockManager with the items axis of len 0Nr placement) r%r?rA isinstanceSingleBlockManagerAssertionErrorr=valuesr slicemake_block_same_classtyperP)rHr?blkarrbpnbr=r8r8rJ make_emptys  zBaseBlockManager.make_emptyrNcCsdSNTr8rRr8r8rJ __nonzero__szBaseBlockManager.__nonzero__)axisrNcCs|jdkr|dkrdnd}|S)NrrUrA)rHrgr8r8rJ_normalize_axiss z BaseBlockManager._normalize_axisr%None)rg new_labelsrNcCs|||||j|<dSrE)Z_validate_set_axisr?)rHrgrlr8r8rJset_axiss zBaseBlockManager.set_axiscCst|jdkSNrUlenr=rRr8r8rJis_single_blocksz BaseBlockManager.is_single_blockcCstj|jdtjd}tj|jdtjd}|d|dt|jD],\}}|j}|||j<t t |||j<qJ|dk rt d||_ ||_dS)z4 Update mgr._blknos / mgr._blklocs. rdtypezGaps in blk ref_locsN)npemptyshapeintpfill enumerater=mgr_locsindexerarangerpanyrZr:r;)rH new_blknosZ new_blklocsblknor_rlr8r8rJrQs    z,BaseBlockManager._rebuild_blknos_and_blklocscCs |jdSNrr?rRr8r8rJitemsszBaseBlockManager.itemscCs"tdd|jD}||jS)NcSsg|] }|jqSr8rr.0r_r8r8rJ sz/BaseBlockManager.get_dtypes..)ruarrayr=takerSrHZdtypesr8r8rJ get_dtypesszBaseBlockManager.get_dtypeslist[ArrayLike]cCsdd|jDS)a2 Quick access to the backing arrays of the Blocks. Only for compatibility with ArrayManager for testing convenience. Not to be used in actual code, and return value is not the same as the ArrayManager method (list of 1D arrays vs iterator of 2D ndarrays / 1D EAs). cSsg|] }|jqSr8r[rr8r8rJrsz+BaseBlockManager.arrays..r=rRr8r8rJarrayss zBaseBlockManager.arraysstrcCsht|j}t|jD]4\}}|dkr4|d|7}q|d|d|7}q|jD]}|d|7}qP|S)Nrz Items: z Axis z:  )r^__name__rzr?r=)rHoutputiaxblockr8r8rJ__repr__s  zBaseBlockManager.__repr__Fzlist[str] | None)rH align_keysignore_failuresrNc s$dks t|pg}g}fdd|D}|jD]}|r|D]d\}} t| ttfr| jdkrv| j|jj j |<q| jdd|jj fj |<q@| |jj |<q@z,t |r|j |f} nt ||f} Wn"ttfk r|sYq0YnXt| |}q0|r||St|||jS)a Iterate over the blocks, collect and create a new BlockManager. Parameters ---------- f : str or callable Name of the Block method to apply. align_keys: List[str] or None, default None ignore_failures: bool, default False **kwargs Keywords to pass to `f` Returns ------- BlockManager filtercsi|]}||qSr8r8)rkkwargsr8rJ 1sz*BaseBlockManager.apply..rUN)rZr=rrXrrrAZilocr{r|Z_valuescallableapplygetattr TypeErrorrGr/_combiner^rPr?) rHfrrr result_blocksZ aligned_argsbrobjappliedr8rrJrs0      zBaseBlockManager.apply)rHalignerrorsrNcCs4|rddg}ndg}t|dd}|jd||||dS)NothercondTZ extract_numpywhere)rrrrr"r)rHrrrrrr8r8rJrSs  zBaseBlockManager.wherecCs|jd||dS)Nsetitem)r|valuer)rHr|rr8r8rJrbszBaseBlockManager.setitem)rcCs2|rddg}ndg}t|dd}|jd|||dS)NnewmaskTrputmask)rrrr)rHrrrrr8r8rJres  zBaseBlockManager.putmask)rHnrgrNcCs||}|jd||dS)Ndiff)rrg)rjr)rHrrgr8r8rJrts zBaseBlockManager.diffcKs |jd|S)N interpolate)rrrHrr8r8rJrxszBaseBlockManager.interpolate)rHperiodsrgrNcCs||}|tjkrd}|dkr|jdkr|jdkr|jd}|dkrbdg|tt||}n t|}tt||dg|}|j |j |d|ddd}|S|j d|||d S) NrrhrUrtTF)rg fill_value allow_dups consolidateshift)rrgr) rjr no_defaultrAnblocksrwlistrangeabsreindex_indexerrr)rHrrgrZncolsr|Znperresultr8r8rJr{s&   zBaseBlockManager.shift)rHinplacerNcCs|jd||||dS)Nfillna)rlimitrdowncastr)rHrrrrr8r8rJrszBaseBlockManager.fillnacCs |dS)NrrrRr8r8rJrszBaseBlockManager.downcastraise)rHcopyrrNcCs|jd|||dS)Nastype)rsrrr)rHrsrrr8r8rJrszBaseBlockManager.astype)rHrdatetimenumeric timedeltarNcCs|jd||||dS)Nconvert)rrrrr)rHrrrrr8r8rJrszBaseBlockManager.convert)rHrregexrNcCs*t|dkst||jd||||dS)Nrreplace) to_replacerrr)rurArZr)rHrrrrr8r8rJrszBaseBlockManager.replacez list[Any])rHsrc_list dest_listrrrNcCs*t|d}|jd||||d}||S)zdo a list replacerZ _replace_list)rrrr)rr_consolidate_inplace)rHrrrrbmr8r8rJ replace_lists zBaseBlockManager.replace_listcKs |jd|S)z} Convert values to native types (strings / python objects) that are used in formatting (repr / csv). to_native_types)rrrr8r8rJrsz BaseBlockManager.to_native_typescCs|js||jS)zH Return True if more than one block with the same dtype )rC_consolidate_checkrDrRr8r8rJis_consolidatedsz BaseBlockManager.is_consolidatedcCs0dd|jD}t|tt|k|_d|_dS)NcSsg|]}|jr|jqSr8)_can_consolidatersrr8r8rJrsz7BaseBlockManager._consolidate_check..T)r=rpsetrDrCrr8r8rJrsz#BaseBlockManager._consolidate_checkcCstdd|jDS)Ncss|] }|jVqdSrEZ is_numericrrr8r8rJ sz9BaseBlockManager.is_numeric_mixed_type..)allr=rRr8r8rJis_numeric_mixed_typesz&BaseBlockManager.is_numeric_mixed_typecCstdd|jDS)z>Whether any of the blocks in this manager are extension blockscss|] }|jVqdSrE) is_extensionrr8r8rJrsz7BaseBlockManager.any_extension_types..)r~r=rRr8r8rJany_extension_typessz$BaseBlockManager.any_extension_typescCst|jdkr|jdjSdS)z8return a boolean if we are a single block and are a viewrUrF)rpr=is_viewrRr8r8rJrs zBaseBlockManager.is_view)rHrrNcCsZg}|jD]B}|jtkr$||q |jr |}|D]}|jr6||q6q |||S)z Select blocks that are bool-dtype and columns from object-dtype blocks that are all-bool. Parameters ---------- copy : bool, default False Whether to copy the blocks )r=rsrBappend is_object_splitZis_boolr)rHr new_blocksr_nbsrbr8r8rJ get_bool_datas    zBaseBlockManager.get_bool_datacCs|dd|jD|S)zy Parameters ---------- copy : bool, default False Whether to copy the blocks cSsg|]}|jr|qSr8rrrr8r8rJrsz5BaseBlockManager.get_numeric_data..)rr=)rHrr8r8rJget_numeric_datasz!BaseBlockManager.get_numeric_dataz Index | None)rHr=rindexrNc Cst|dkrb|jdkrZ|dk r2|jdd|g}n|jddg|jdd}||S|Sttdd|D}t ||j d}g}|D],}|j |d}t ||j j|_ ||qt|j}|dk r||d<|j||d<t|||S) z$return a new manager with the blocksrrhNrUcSsg|] }|jjqSr8r{as_arrayrr8r8rJr&sz-BaseBlockManager._combine..deeprt)rprArr?rcrusort concatenater Zget_reverse_indexerrwrr r{r|rrrr^rP) rHr=rrr?r|Z inv_indexerrrr8r8rJrs&      zBaseBlockManager._combinecCs t|jSrErorRr8r8rJr6szBaseBlockManager.nblockscsHr&fddfdd|jD}n t|j}|jdd}||_|S)a> Make deep or shallow copy of BlockManager Parameters ---------- deep : bool or string, default True If False, return shallow copy (do not copy data) If 'all', copy data and a deep copy of the index Returns ------- BlockManager csdkr|jddS|S)NrTr)rview)rrr8rJ copy_funcLsz(BaseBlockManager.copy..copy_funccsg|] }|qSr8r8rr)rr8rJrOsz)BaseBlockManager.copy..rr)r?rr)rHrnew_axesresr8)rrrJr:s  zBaseBlockManager.copycCs4|r |St||j|jdd}d|_||S)zr Join together blocks having same dtype Returns ------- y : BlockManager FrI)rr^r=r?rDr)rHrr8r8rJrWs zBaseBlockManager.consolidatecCs0|s,tt|j|_d|_d|_|dSre)rtuple _consolidater=rDrCrQrRr8r8rJrgs z%BaseBlockManager._consolidate_inplace)rHnew_axisrgrrr only_slicerNc sdkrD||j|kr|s|S|j|d} t|j| _|| j|<| S|rP||sd|j|||jkrvtd|dkr|j|d} nfdd|jD} t|j} || |<t | | | S)a Parameters ---------- new_axis : Index indexer : ndarray of int64 or None axis : int fill_value : object, default None allow_dups : bool, default False copy : bool, default True consolidate: bool, default True Whether to consolidate inplace before reindexing. only_slice : bool, default False Whether to take views, not copies, along columns. pandas-indexer with -1's only. Nr#Requested axis not found in managerr)rrcs*g|]"}|jddk rn|jdqS)rUN)rgr)take_ndrrrr|r8rJrs z4BaseBlockManager.reindex_indexer..) r?rrrZ_validate_can_reindexrA IndexError_slice_take_blocks_ax0r=r^rP) rHrr|rgrrrrrrrrr8rrJrns2      z BaseBlockManager.reindex_indexerslice | np.ndarray)slice_or_indexerrrNcsl|tjk }t||jd|d\}}}|jr|jd|dkrf|dkrHgSttd|}j||dgS|rt|j dkr|r|dkrj }|s|rfddt |D} | Sttd|}j |d||d gS|dkr|j |} |j|} n(tj |j |d |d } tj |j|d |d } g} | } tj| | d D]<\} }| d krT| |j||d q(|j| jsjs|D]0}jdd}tt||d|_| |qrn| |j}tt||d}|rt||}t|trj||d}| |nf|rJt||D]8\}}t||d}t|}j||d}| |qnj |d|d}| |q(| S)a Slice/take blocks along axis=0. Overloaded for SingleBlock Parameters ---------- slice_or_indexer : slice or np.ndarray[int64] fill_value : scalar, default lib.no_default only_slice : bool, default False If True, we always return views on existing arrays, never copies. This is used when called from ops.blockwise.operate_blockwise. Returns ------- new_blocks : list of Block r) allow_fillr\ new_mgr_locsrUNcs,g|]$\}}jt||dt|dqS)rUr)getitem_block_columnsr\r )rrmlr_r8rJrs  z;BaseBlockManager._slice_take_blocks_ax0..)rgrrrt)rrgroup)rWrFr)rgr)r r_preprocess_slice_or_indexerrwrqr=r r\r rArrzrrSrTalgos libinternalsget_blkno_placementsr_make_na_blockrZ_validate_ndimrr{r|maxrpZmaybe_indices_to_slicerXzip)rHrrrrZsl_typeslobjZsllenrar=rSrTr rr{mgr_locZnewblktakermax_lenrbrr Zslcr8r rJrs                z'BaseBlockManager._slice_take_blocks_ax0r r*)rWrNcCsX|dkrtj}t|j}t||d<t|\}}tj||d}||t|||j dS)NrrrrWrA) runanrrwrprrvryr2rA)rHrWr block_shapers block_valuesr8r8rJr1s    zBaseBlockManager._make_na_blockrU)rHrgverifyrNcCslt|tr"tj|j|j|jddn tj|dd}|j|}t |||d}|j | |}|j |||dddS)ac Take items along any axis. indexer : np.ndarray or slice axis : int, default 1 verify : bool, default True Check that all entries are between 0 and len(self) - 1, inclusive. Pass verify=False if this check has been done by the caller. Returns ------- BlockManager Zint64rr)rTF)rr|rgrr) rXr\rur}startstopstepZ asanyarrayrwr#r?rr)rHr|rgrrrlr8r8rJrAs  zBaseBlockManager.take)T)N)NF)T)Fr)TTTT)FF)F)F)TN)T)NFTTF)N)rUT)7r __module__ __qualname____doc__ __slots____annotations__rK classmethodrPpropertyrSrTrcrf__bool__rjrmrqrQrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrr rrrrr8r8r8rJr6Ys 2    ?   Fc@seZdZdZdZdOdddddd Zd d d d ZeddddddZdddddZ dddddZ dddddZ e dd dd Z d!dd"d#d$Zdd%dd d&d'd(Zdd d)d*ZdPd,d-dd,d.d/d0ZdQd,d-dd1d.d2d3Zddd4d5d6Zdddd7d8d9Zd:d;d<d,d=dd,d>d?d@Zdd dAdBZdRddCdDdEZd+dFd+ejfddGddHdIdJdKZdFejfdGdHdLdMdNZdFS)S BlockManagerz0 BaseBlockManager that holds 2D blocks. rhTzSequence[Block]zSequence[Index]rB)r=r?rIc Cs|rtdd|Dst|D]t}|j|jkrHtd|jd|jdt|tr|jjdkrt|j|j|_z|jWqt k rYqXq| dS)Ncss|]}t|tVqdSrErXr%rxr8r8rJrusz(BlockManager.__init__..zNumber of Block dimensions (z) must equal number of axes ()rU) rrZrArXr,r[r._cacheclearAttributeError_verify_integrity)rHr=r?rIrr8r8rJrKms" zBlockManager.__init__rkrdcCs|j}tdd|jD}|jD]6}|jdd|ddkr t||jdd|jq t|j|krtdt|jd|dS)Ncss|]}t|jVqdSrE)rpr{r+r8r8rJrsz1BlockManager._verify_integrity..rUzINumber of manager items must equal union of block items # manager items: z, # tot_items: )rwsumr=construction_errorr?rprrZ)rHZ mgr_shape tot_itemsrr8r8rJr1s zBlockManager._verify_integrityrLr>r=r?rNcCs|||ddS)Z Constructor for BlockManager and SingleBlockManager with same signature. Frr8rOr8r8rJrPszBlockManager.from_blocksr@r)locrNcCst|jdkr&|jdtd|fStdd|jD}t|}t|tr\tj|t d}ntj||d}t |}|jD]*}t |j D]\}}|||f||<qqxt|tr| j||d}|S)z Return the array corresponding to `frame.iloc[loc]`. Parameters ---------- loc : int Returns ------- np.ndarray or ExtensionArray rUrNcSsg|] }|jqSr8rrrr8r8rJrsz(BlockManager.fast_xs..rr)rpr=igetr\r)rXrrurvobjectr!rzr{Zconstruct_array_typeZ_from_sequence)rHr7rsrrr_rrr8r8rJfast_xss    zBlockManager.fast_xsrY)rrNcCs\|j|j|}||j|}ttdt|}t|}t|||dd}t ||j dS)z: Return the data as a SingleBlockManager. rrUr) r=rSr8rTr r\rpr1r^rYr?)rHrrr[rarbr8r8rJr8s zBlockManager.igetcCs$|j|j|}||j|}|S)zY Return the data for column i as the values (ndarray or ExtensionArray). )r=rSr8rT)rHrrr[r8r8rJ iget_valuesszBlockManager.iget_valueszlist[np.ndarray]cCsbdd|jD}g}tt|jD]:}||j|}|jdkrN||j|}n|}||q"|S)z Used in the JSON C code to access column arrays. This optimizes compared to using `iget_values` by converting each block.values to a np.ndarray only once up front cSs*g|]"}t|tr|jjn t|jqSr8)rXr,r[Z_ndarrayruasarrayrr8r8rJrs z.BlockManager.column_arrays..rh)r=rrprrSrArTr)rHrrrr`r[r8r8rJ column_arrayss  zBlockManager.column_arrayszint | slice | np.ndarray)r7rcstddjdkr(jdkr(tdd}t|}|rNfdd}nNjdkr`jn tdd fd d}jddjddkrt d t |r|g}j |}j |}g}g} gtj|dd D]\} } j| } || j} | r| | || q|| jj| | | t| t| jkrT| q| | tt| j| jj<qtrtjjtjd }d|<tjjtj d }|!dtjt||<|j"_"t#fddt$jD_| rt%|}t|}g}|r^|&fdd|Dt|tjj"|<dj|<nP| d| dd}|t'||j|dtjj"|<t|j|<jt#|7_d_(dS)z Set new item in-place. Does not consolidate. Adds new Block if not contained in the current set of items TrNrUrscsSrEr8rVrr8rJ value_getitem sz(BlockManager.iset..value_getitemrhrics |jSrE)r|rVr>r8rJr?sz9Shape of new values must be compatible with manager shaper rrrtc3s"|]\}}|tkr|VqdSrE)r)rrr_)removed_blknosr8rJrDs z$BlockManager.iset..c3s(|] }tjt||ddVqdS)rUr[rArWN)r2rAr\)rr)rHrr8rJrQs  rrAF))r"r;rArQrrr5r.rwrZr is_integerrSrTrrrr=r|Z should_storeZ set_inplacerr{rrpdeleterur}zerosrbool_rvrxryr:rrzrextendr2rC)rHr7rZvdtypeZvalue_is_extension_typer?rSrTZunfit_mgr_locsZunfit_val_locsrZval_locsr_Zblk_locs is_deletedrZ unfit_countrZunfit_val_itemsr8)r@rHrrJisets                     zBlockManager.isetr)r7itemrrNc Cs\|j||}|jdkr |j}nt||jd}t||jt||dd}t|j|dD]Z\}}|j |}|t |j kr|j d|_ qZ|j j } | | |kd7<t| |_ qZ||jjdkrt|jd|_t|jt |j |_n*t|j|d|_t|j|t |j |_||jd<|j |f7_ d|_t |j dkrXtjd td d dS) z Insert item at selected position. Parameters ---------- loc : int item : hashable value : np.ndarray or ExtensionArray rhrirUrANrFdzDataFrame is highly fragmented. This is usually the result of calling `frame.insert` many times, which has poor performance. Consider using pd.concat instead. To get a de-fragmented frame, use `newframe = frame.copy()` stacklevel)rinsertrAr5r.r2r\_fast_count_smallintsrSr=rpr{addrrr rTrwrurr;r:r?rCwarningswarnr) rHr7rIrrrrcountr_rr8r8rJrNqs4      zBlockManager.insertcCsftj|jdtjd}d||<|d}|j|dd}|j|}||jdg}t|t ||S)zJ Delete selected locations, returning a new BlockManager. rrrT)rrU) rurDrwrEnonzerorrr?r^r)rHr|rGrr new_columnsr?r8r8rJideletes zBlockManager.ideleteFr5r)rHfuncrrNc Csg}|jD]}|jr`|D]@}z||}Wn"ttfk rP|sHYqYnXt||}qq z||}Wn"ttfk r|sYq YnXt||}q t|dkrtdg}ntt |dj j d}|r|j |d|dSt |||jd|gS)aP Apply grouped reduction function blockwise, returning a new BlockManager. Parameters ---------- func : grouped reduction function ignore_failures : bool, default False Whether to drop blocks where func raises TypeError. Returns ------- BlockManager rNrtFrr)r=rrrrrGr/rpr%rr[rwrr^rPr?)rHrWrrr_Zsbrrr8r8rJgrouped_reduces0       zBlockManager.grouped_reduceztuple[T, np.ndarray]c Cs|jdkstg}|jD]}|||}||qtdg}|r|rltdd|D}|j|d|d}qg}t | g|j dd|g}n&t |j d}t | ||j |g}||fS)a Apply reduction function blockwise, returning a single-row BlockManager. Parameters ---------- func : reduction function ignore_failures : bool, default False Whether to drop blocks where func raises TypeError. Returns ------- BlockManager np.ndarray Indexer of mgr_locs that are retained. rhNcSsg|] }|jjqSr8rrr8r8rJrsz'BlockManager.reduce..FrXr)rArZr=reducerFr%rurrr^rPrr}rw) rHrWrZ res_blocksr_rrr|Znew_mgrr8r8rJrZs     zBlockManager.reduce)rrNcCs t|||S)zO Apply array_op blockwise with another (aligned) BlockManager. )r4)rHrZarray_opr8r8rJr4 szBlockManager.operate_blockwiserHrrNcCs t||tS) Used in .equals defined in base class. Only check the column values assuming shape and indexes have already been checked. )r3r)rHrr8r8rJ _equal_valuesszBlockManager._equal_valuesrZlinear)rg interpolationr$)rHqsrgrNcsb|jdksttstdks&tt|j}t|d<fdd|jD}t|||S)a  Iterate over blocks applying quantile reduction. This routine is intended for reduction type operations and will do inference on the generated blocks. Parameters ---------- axis: reduction axis, default 0 consolidate: bool, default True. Join together blocks having same dtype interpolation : type of interpolation, default 'linear' qs : list of the quantiles to be computed Returns ------- BlockManager rhrUcsg|]}|jdqS))rgr_r^)quantilerrgr^r_r8rJr:sz)BlockManager.quantile..)rArZrrr?r$r=r^)rHr_rgr^rr=r8rarJr`s    zBlockManager.quantilecCs||j}|j}g}g}|jD]N}|j|jj}||} || } |j||| d\} } || || q ||}t |||g} | S)a( Return a BlockManager with all blocks unstacked.. Parameters ---------- unstacker : reshape._Unstacker fill_value : Any fill_value for newly introduced missing values. Returns ------- unstacked : BlockManager ) new_placement) Zget_new_columnsr new_indexr=r{r|Z get_indexerZ_unstackrFr))rHZ unstackerrrUrcrZ columns_maskr_Zblk_colsZ new_itemsrbr=rrr8r8rJunstackCs$       zBlockManager.unstackrcs@i}jD]}|t|jg|q fdd|DS)z Return a dict of str(dtype) -> BlockManager Parameters ---------- copy : bool, default True Returns ------- values : a dict of dtype -> BlockManager cs i|]\}}|j|dqS)re)r)rrsr=rrHr8rJrzsz(BlockManager.to_dict..)r= setdefaultrrsrr)rHrZbdrr8rfrJto_dicths  zBlockManager.to_dictNz Dtype | Noner9) transposersrrNcCst|jdkr.tj|jtd}|r*|S|S|p:|tjk }|j r|jd}|j rl|j j ||d |j}qt|}|r|j|dd}n|j||d}d}|r|}|tjk r||t|<|r|S|S)a| Convert the blockmanager data into an numpy array. Parameters ---------- transpose : bool, default False If True, transpose the return array. dtype : object, default None Data type of the return array. copy : bool, default False If True then guarantee that a copy is returned. A value of False does not guarantee that the underlying data is not copied. na_value : object, default lib.no_default Value to be used as the missing value sentinel. Returns ------- arr : ndarray rrrrsna_valueFre)rpr=rurvrwfloatrir rrqrr[to_numpyZreshaper< get_valuesr _interleaverr)rHrirsrrkr`r_r8r8rJr|s.   zBlockManager.as_arrayrsrNcCs|stdd|jD}t|tr*|j}n*t|tr@td}nt|t rTtd}tj |j |d}t |j d}|jD]@}|j }|jr|jj||d}n ||}|||j<d||j<qz|std|S) zt Return ndarray from blocks with specified item order Items must be contained in the blocks cSsg|] }|jqSr8rrrr8r8rJrsz,BlockManager._interleave..r9rrrrjrUz'Some items were not contained in blocks)r)r=rXr subtyperrursrrrvrwrDr{rr[rmrnr|rrZ)rHrsrkrZitemmaskr_rr`r8r8rJros.         zBlockManager._interleave)T)F)F)T)rr!r"r#rArKr1r&rPr:r8r;r'r=rHrNrVrYrZr4r]r`rdrhr rrror8r8r8rJr)csB    & z50' *%Dr)c@s~eZdZdZdZdZdZdZdZde j fdddd d d Z e d d ddddZ e ddddddZddZddZddZeddddZeddZed d!Zddd"d#ZdLd%d&dd'd(d)Zeddd*d+Zed,dd-d.Zd/dd0d1Zd2d3Zd4d5Zd6d7Zeddd8d9Zddd:d;Zdd?Z!ddd@dAZ"dBdCZ#ddDdEdFZ$dGdGddHdIdJZ%dKS)MrYzmanage a single block withrUTr8Fr*r%rB)rrgrIcCsZt|tstt|t|ts,tt||tjk rFtjdt dd|g|_ |f|_ dS)NzMThe `fastpath` keyword is deprecated and will be removed in a future version.rhrL) rXr*rZr^r%r rrQrR FutureWarningr?r=)rHrrgrIZfastpathr8r8rJrKs zSingleBlockManager.__init__rLr>r5cCs6t|dkstt|dks t||d|dddS)r6rUrFr)rprZrOr8r8rJrPszSingleBlockManager.from_blocksr)rrrNcCs"t|tdt|dd}|||S)zN Constructor for if we have an array that is not yet a Block. rrUr)r2r\rp)rMrrrr8r8rJ from_arrayszSingleBlockManager.from_arraycsTddjD}fddjD}tj}d|ddjDdi}||||fS)NcSsg|] }|jqSr8rrr8r8rJr#sz3SingleBlockManager.__getstate__..csg|]}j|jjqSr8)rr{r|rrRr8rJr$s0.14.1cSsg|]}|j|jjdqS))r[r{)r[r{r|rr8r8rJr*s)r?r=)r=rr?)rHrZ block_itemsZ axes_arrayZ extra_stater8rRrJ __getstate__"s  zSingleBlockManager.__getstate__csdddddt|tr|t|dkr|d|dkr||dd}d d |d D|_t|jtfd d |dD|_ntd|dS)Nr@r*)rArNcSst|dd}t|||dS)NTrr)r"r2)r[r{rAr8r8rJunpickle_block6s z7SingleBlockManager.__setstate__..unpickle_blockrtcSsg|] }t|qSr8)r&rr8r8rJr>sz3SingleBlockManager.__setstate__..r?c3s$|]}|d|ddVqdS)r[r{riNr8rrArvr8rJr@sz2SingleBlockManager.__setstate__..r=z*pre-0.14.1 pickles are no longer supported)rXrrpr?r=rG_post_setstate)rHstater8ryrJ __setstate__5s"   zSingleBlockManager.__setstate__cCsdSrEr8rRr8r8rJrzIsz!SingleBlockManager._post_setstaterdcCs |jdSrrrRr8r8rJ_blockLszSingleBlockManager._blockcCsdSzcompat with BlockManagerNr8rRr8r8rJr:PszSingleBlockManager._blknoscCsdSr~r8rRr8r8rJr;UszSingleBlockManager._blklocscCsZ|j}||}|jdkr"tdttdt|}|j||d}|j|}t |||S)NrUz(dimension-expanding indexing not allowedrrV) r}_slicerA ValueErrorr r\rpr]rr^)rHr|r_rrarZnew_idxr8r8rJ getitem_mgrZs   zSingleBlockManager.getitem_mgrrr\r@)rrgrNcCsrt|tstt|||jkr(td|j}||}ttdt |}|j ||d}|j |}t|||S)NrrrV) rXr\rZr^rArr}rr rpr]rZ_getitem_slice)rHrrgr_rrarrcr8r8rJ get_slicehs   zSingleBlockManager.get_slicecCs |jdSrrrRr8r8rJrtszSingleBlockManager.indexrcCs|jjSrE)r}rsrRr8r8rJrsxszSingleBlockManager.dtyper9cCst|jjgSrE)rurr}rsrRr8r8rJr|szSingleBlockManager.get_dtypescCs |jS)z$The array that Series.values returns)r}external_valuesrRr8r8rJrsz"SingleBlockManager.external_valuescCs|jjS)z%The array that Series._values returns)r}r[rRr8r8rJinternal_valuessz"SingleBlockManager.internal_valuescCs|jjS)z#The array that Series.array returns)r} array_valuesrRr8r8rJrszSingleBlockManager.array_valuescCs|jjSrE)r} _can_hold_narRr8r8rJrszSingleBlockManager._can_hold_nacCsdSrer8rRr8r8rJrsz"SingleBlockManager.is_consolidatedcCsdSrEr8rRr8r8rJrsz%SingleBlockManager._consolidate_checkcCsdSrEr8rRr8r8rJrsz'SingleBlockManager._consolidate_inplacecCs&|j||jd||jd<|S)zy Delete single location from SingleBlockManager. Ensures that self.blocks doesn't become empty. r)r}rCr?)rHr|r8r8rJrVs zSingleBlockManager.ideletecCs tddS)zY fast path for getting a cross-section return a view of the data zUse series._values[loc] insteadNrF)rHr7r8r8rJr:szSingleBlockManager.fast_xsrcCs(||jd_ttt||jd_dS)z Set the values of the single block in place. Use at your own risk! This does not check if the passed values are valid for the current Block/SingleBlockManager (length, dtype, etc). rN)r=r[r r\rpZ _mgr_locs)rHr[r8r8rJ set_valuess zSingleBlockManager.set_valuesr5r[cCs0|jdkrdS|jdj}|jdj}t||S)r\rUFr)rAr=r[r)rHrleftrightr8r8rJr]s    z SingleBlockManager._equal_valuesN)r)&rr!r"r#rArDrCr$rqr rrKr&rPrsrur|rzr'r}r:r;rrrrsrrrrrrrrrVr:rr]r8r8r8rJrYsR     rYTrLr>rB)r=r?rrNc Cs~zt||}Wn^tk rl}z@dd|D}tdd|D}t||djdd||W5d}~XYnX|rz||S)NcSsg|] }|jqSr8rrr8r8rJrsz4create_block_manager_from_blocks..css|]}|jdVqdS)rN)rw)rr`r8r8rJrsz3create_block_manager_from_blocks..rrU)r)rr2r3rwr)r=r?rmgrerrrr4r8r8rJ create_block_manager_from_blockss.rcCs t|ddS)NTr)r")rr8r8rJ_extract_arraysrr%)namesr?rrNc Cst|tstt|tsttdd|Ds2tdd|D}zt||||}t||}Wn:tk r}ztt ||dj ||W5d}~XYnX|r| |S)Ncss|]}t|tVqdSrEr*r+r8r8rJrsz3create_block_manager_from_arrays..cSsg|] }t|qSr8)rr+r8r8rJrsz4create_block_manager_from_arrays..r) rXr%rZrr _form_blocksr)rr3rprwr)rrr?rr=rer8r8rJ create_block_manager_from_arrayss*rr@rzValueError | None)r4rr?rcCsttt|gt|}t|dkr2|ddd}tdd|D}t|dkr^|ddd}||krr|dk rr|S|ddkrtdStd|d |S) z.raise a helpful message about our constructionrhNrtcss|]}t|VqdSrErprr8r8rJrsz%construction_error..rz)Empty data passed with indices specified.zShape of passed values is z, indices imply )rmapr@rrpr)r4rr?rZpassedZimpliedr8r8rJr3s   r3r)rrr?rrNcCstt}g}|}||dr.tt|}n"||djsBt| |d}t |D]@\}} | dkrt| |qX|| } t | } || j  || fqXg} t|drt|d|d} | | t|drt|d|d}| |t|drdd|dD}| |t|d dkrDt|d tj|d}| |t|d dkrrd d|d D}| |t|d rd d|d D}| |t|rt|ftdd|ddD}tj|td}|tjt||dd}| || S)NrrtZ NumericBlock)rZDatetimeLikeBlockr,cSs*g|]"\}}ttt|dt|ddqSrh)klassrWrA)r2r.r"r,rrrr8r8rJr3s z _form_blocks..Z ObjectBlockr+cSs g|]\}}t|t|ddqSr)r2r+rr8r8rJrEsr-cSs g|]\}}t|t|ddqSr)r2r-rr8r8rJrLscss|]}t|VqdSrErr+r8r8rJrTsz_form_blocks..rUrrrhr)rrequalsrur}rp intersectionZ is_uniquerZZget_indexer_forrzrr0r_multi_blockifyrF_simple_blockifyZobject_rrvr9ryrr2)rrr?rZ items_dictZ extra_locsZ names_idxZ names_indexerrZname_idxvZ block_typer=Znumeric_blocksZ dtlike_blocksZ dttz_blocksZ object_blocksZ cat_blocksZexternal_blocksrwrZna_blockr8r8rJr sp          $  r)rrNcCsN|st||dSt||\}}|dk r:|j|kr:||}t||dd}|gS)zr return a single array of a block that has a single dtype; if dtype is not None, coerce to this dtype rrNrhr) _tuples_to_blocks_no_consolidate _stack_arraysrsrr2)tuplesrsrr[rWrr8r8rJr`s  rzDtypeObj | None)rsrc Cs`|st||dSt|dd}g}|D]2\}}tt||\}}t||dd}||q(|S)z@return an array of blocks that potentially have different dtypesrrcSs |djSrnrrr,r8r8rJyz!_multi_blockify..rhr)r itertoolsgroupbyrrr2r) rrsrgrouperrZ tup_blockr[rWrr8r8rJrrs   rrpcs(dk rfdd|DSdd|DS)Nc s2g|]*}tt|djdd|dddqS)rUFrerrhr)r2ru atleast_2drr+rrr8rJrs z4_tuples_to_blocks_no_consolidate..cSs(g|] }tt|d|dddqS)rUrrhr)r2rurr+r8r8rJrsr8)rrsr8rrrJrs  rznp.dtyperrc CsTt|\}}|d}t|f|j}tj||d}t|D]\}}|||<q:||fS)Nrrr)rrprwrurvrz) rrsrWrfirstrwZstackedrr`r8r8rJrs  rr<)r=rNcCsRdd}tt||d|}g}|D](\\}}}tt|||d}t||}q$|S)zJ Merge blocks having same dtype, exclude non-consolidating blocks cSs|jSrE)Z_consolidate_keyrr8r8rJrrz_consolidate..)key)rscan_consolidate)rrsorted _merge_blocksrr/)r=ZgkeyrrrrsZ group_blocksZ merged_blocksr8r8rJrs rr)r=rsrrNc Cst|dkr|S|rtdd|D}t|djtjrPtdd|D}n.dd|D}ttt|}|dj |dd}t |}||}||}t |}t ||dd gS|S) NrUcSsg|] }|jjqSr8rrr8r8rJrsz!_merge_blocks..rcSsg|] }|jqSr8rrr8r8rJrscSsg|] }|jqSr8rrr8r8rJrs)rgrhr) rprurrXrsZvstackr rrZ_concat_same_typeargsortr r2) r=rsrrZ new_valuesZbvalsZbvals2rrar8r8rJrs  rr9)r`rNcCs0t|tj}|d}tj|||fS)z:Faster version of set(arr) for sequences of small numbers.r)ruZbincountrint_rTZc_)r`countsZnzr8r8rJrOs rOr)rlengthrcCsvt|trd|t||fSt|tjr4|jjdkrNt|dd}t t ||t |}|sdt ||}d|t |fSdS)Nr\rrsZfancy)rXr\rZ slice_lenruZndarrayrskindrrr^rr#rp)rrrrsr|r8r8rJrs      r)T)T)N)NT)b __future__r collectionsrrtypingrrrrrr r rQZnumpyruZ pandas._libsr rr Zpandas._libs.internalsr Zpandas._typingrrrrrZ pandas.errorsrZpandas.util._validatorsrZpandas.core.dtypes.castrZpandas.core.dtypes.commonrrrrZpandas.core.dtypes.dtypesrZpandas.core.dtypes.genericrrZpandas.core.dtypes.missingrrZpandas.core.algorithmscoreZ algorithmsrZpandas.core.arrays._mixinsrZpandas.core.arrays.sparser Zpandas.core.constructionr!r"Zpandas.core.indexersr#Zpandas.core.indexes.apir$r%r&Zpandas.core.internals.baser'r(r)Zpandas.core.internals.blocksr*r+r,r-r.r/r0r1r2Zpandas.core.internals.opsr3r4r5r6r)rYrrrr3rrrrrrrrOrr8r8r8rJsp  $         ,  ST %