U
    f/ej[                     @  s  d dl mZ d dlZd dlZd dlmZmZmZ d dlZ	d dl
mZ d dlmZmZmZmZ d dlmZ d dlmZmZ d dlmZmZmZmZmZmZ d d	lmZm Z  d d
l!m"Z" d dl#m$Z$m%Z% d dl&m'  m(Z) d dl*m+Z+m,Z, d dl-m.Z. d dl/m0Z0m1Z1 d dl2m3Z3m4Z4 d dl5m6Z6 er0d dl7m8Z8 dddddddZ9dddddZ:dddddddZ;d d!d"d#d$Z<G d%d& d&Z=d'dddd(d)d*Z>d+dd,d-d.Z?d/d+d0d1d2Z@d'dd0d3d4ZAdd5d6d7ZBd&dd&d8d9d:ZCdd;d<d=ZDdS )>    )annotationsN)TYPE_CHECKINGSequencecast)	internals)	ArrayLikeDtypeObjManagerShape)cache_readonly)ensure_dtype_can_hold_nafind_common_type)is_1d_only_ea_dtypeis_1d_only_ea_objis_datetime64tz_dtypeis_dtype_equalis_extension_array_dtype	is_sparse)cast_to_common_typeconcat_compat)ExtensionDtype)is_valid_na_for_dtypeisna_all)DatetimeArrayExtensionArray)ensure_wrapped_if_datetimelike)ArrayManagerNullArrayProxy)ensure_block_shape	new_block)BlockManager)Indexzlist[Index]intboolr	   )axesconcat_axiscopyreturnc           	   	     s   g  | D ]<\}}|  D ] \}}|j|| ||ddd}q | q|dkr fddtt d jD }t||d |d gddS |dksttt	j
d	d  D }t||d |d gddS d
S )z
    Concatenate array managers into one.

    Parameters
    ----------
    mgrs_indexers : list of (ArrayManager, {axis: indexer,...}) tuples
    axes : list of Index
    concat_axis : int
    copy : bool

    Returns
    -------
    ArrayManager
    T)axisZ
allow_dupsZuse_na_proxy   c                   s,   g | ]$ t  fd dttD qS )c                   s   g | ]}| j   qS  arrays).0i)jmgrsr*   @/tmp/pip-unpacked-wheel-tiezk1ph/pandas/core/internals/concat.py
<listcomp>^   s     z:_concatenate_array_managers.<locals>.<listcomp>.<listcomp>)concat_arraysrangelen)r-   r0   )r/   r1   r2   ]   s   z/_concatenate_array_managers.<locals>.<listcomp>r   F)Zverify_integrityc                 S  s   g | ]
}|j qS r*   r+   )r-   mgrr*   r*   r1   r2   e   s     N)itemsZreindex_indexerappendr4   r5   r,   r   AssertionErrorlist	itertoolschainfrom_iterable)	mgrs_indexersr$   r%   r&   r7   indexersaxindexerr,   r*   r6   r1   _concatenate_array_managers@   s&        
rC   r;   r   )	to_concatr'   c                   s   dd | D }t dd |D dk}|s<tdd |D  n
|d j  jdkrx fd	d| D } t|d j| dd
S  fdd| D } t| d trt| d }|| S t	| }t |dkrdd |D }t |dkrd|kr|
t}|S )a  
    Alternative for concat_compat but specialized for use in the ArrayManager.

    Differences: only deals with 1D arrays (no axis keyword), assumes
    ensure_wrapped_if_datetimelike and does not skip empty arrays to determine
    the dtype.
    In addition ensures that all NullArrayProxies get replaced with actual
    arrays.

    Parameters
    ----------
    to_concat : list of arrays

    Returns
    -------
    np.ndarray or ExtensionArray
    c                 S  s   g | ]}t |ts|qS r*   )
isinstancer   r-   xr*   r*   r1   r2   |   s     
 z!concat_arrays.<locals>.<listcomp>c                 S  s   h | ]
}|j qS r*   dtyperF   r*   r*   r1   	<setcomp>~   s     z concat_arrays.<locals>.<setcomp>r)   c                 S  s   g | ]
}|j qS r*   rH   r-   Zarrr*   r*   r1   r2      s     r   mMc                   s$   g | ]}t |tr| n|qS r*   )rE   r   to_arrayrK   Ztarget_dtyper*   r1   r2      s   r(   c                   s*   g | ]"}t |tr| nt| qS r*   )rE   r   rO   r   rK   rP   r*   r1   r2      s   c                 S  s   h | ]}|j jqS r*   )rI   kind)r-   objr*   r*   r1   rJ      s     b)r5   r   rI   rR   typeZ_concat_same_typerE   r   npconcatenateastypeobject)rD   Zto_concat_no_proxyZsingle_dtypeclsresultkindsr*   rP   r1   r3   i   s.    






r3   c                 C  sD  t | d d tr t| |||S dd | D }t||}g }|D ]\}}|d }	|	j}
t|dkr|d js|
j}|r| }n|	 }d}ntt
|rdd |D }|
jstj||
jd d}nt|dd}t||
j}t|}|
jj|jk}nt|||d}d	}|r|
j||d
}nt||t|d}|| q@tt||S )z
    Concatenate block managers into one.

    Parameters
    ----------
    mgrs_indexers : list of (BlockManager, {axis: indexer,...}) tuples
    axes : list of Index
    concat_axis : int
    copy : bool

    Returns
    -------
    BlockManager
    r   c                 S  s   g | ]\}}t ||qS r*   )_get_mgr_concatenation_plan)r-   r7   r@   r*   r*   r1   r2      s    z(concatenate_managers.<locals>.<listcomp>r)   Tc                 S  s   g | ]}|j jqS r*   )blockvaluesr-   Zjur*   r*   r1   r2      s     rQ   )r&   F)	placement)ra   ndim)rE   r   rC   _combine_concat_plansr^   r5   r@   r_   r&   view_is_uniform_join_unitsis_extensionrV   rW   rb   r   r   r   rI   _concatenate_join_unitsZmake_block_same_classr   r9   r    tuple)r?   r$   r%   r&   Zconcat_plansZconcat_planblocksra   
join_unitsunitblkr_   ZfastpathvalsrT   r*   r*   r1   concatenate_managers   s>    

rn   r    zdict[int, np.ndarray])r7   r@   c                 C  s  t | j}| D ]\}}t|||< qt|}d|krh|d}tj| j|dd}tj| j	|dd}n4| j
r| jd }	|	jt|	||fgS d}| j}| j	}g }
tj|ddD ]\}}|jst| }t |}t||d< t|}|dkrtd|}n| j| }	||j }t|t|	jkoT|dkrD|	jjrD|	jjjdkpTt|dk }|rj|dd n||d< t|	||}|
||f q|
S )z
    Construct concatenation plan for given block manager and indexers.

    Parameters
    ----------
    mgr : BlockManager
    indexers : dict of {axis: indexer}

    Returns
    -------
    plan : list of (BlockPlacement, JoinUnit) tuples

    r   )
fill_valueNF)groupr)   )r;   shaper8   r5   rh   popalgostake_ndblknosblklocsZis_single_blockri   Zmgr_locsJoinUnitlibinternalsZget_blkno_placementsZis_slice_liker:   r&   rB   as_slicesteprV   Zdiffallr9   )r7   r@   Zmgr_shape_listrA   rB   Z	mgr_shapeZax0_indexerrv   rw   rl   planZblkno
placementsZjoin_unit_indexersZ
shape_listrr   rk   Zax0_blk_indexerZunit_no_ax0_reindexingr*   r*   r1   r]      sN    






	r]   c                   @  sz   e Zd ZdddddZdddd	Zed
dddZedd Zdd
dddZed
dddZ	dddddZ
dS )rx   Nr
   )rr   c                 C  s"   |d kri }|| _ || _|| _d S Nr^   r@   rr   )selfr^   rr   r@   r*   r*   r1   __init__D  s
    zJoinUnit.__init__strr'   c                 C  s$   t | j dt| j d| j dS )N(z, ))rU   __name__reprr^   r@   )r   r*   r*   r1   __repr__L  s    zJoinUnit.__repr__r#   c                 C  s&   | j  D ]}|dk r
 dS q
dS )Nro   TF)r@   r_   any)r   rB   r*   r*   r1   needs_fillingO  s    zJoinUnit.needs_fillingc                 C  s,   | j }|d krtd| js"|jS t|jS )NzBlock is None, no dtype)r^   r:   r   rI   r   )r   rl   r*   r*   r1   rI   X  s    zJoinUnit.dtyper   )rI   r'   c                   s   | j s
dS | jdkrdS | jtkrH| jj}t fdd|jddD S | jj j  krbdkrvn nt| j svdS | jj	}t
| S )	z
        Check that we are all-NA of a type/dtype that is compatible with this dtype.
        Augments `self.is_na` with an additional check of the type of NA values.
        FNTc                 3  s   | ]}t | V  qd S r   )r   rF   rH   r*   r1   	<genexpr>n  s     z+JoinUnit.is_valid_na_for.<locals>.<genexpr>KorderrN   )is_nar^   rI   rY   r_   r|   ravelrR   r   rp   r   )r   rI   r_   na_valuer*   rH   r1   is_valid_na_forb  s    

  zJoinUnit.is_valid_na_forc                 C  sV   | j d krdS | j jsdS | j j}t| j jjr4dS | j jrB|}n|jdd}t|S )NTFr   r   )r^   Z_can_hold_nar_   r   rI   rf   r   r   )r   r_   Zvalues_flatr*   r*   r1   r   y  s    
zJoinUnit.is_nar   )empty_dtyper'   c                 C  s  |d kr| j j}| j  }nT|}| |r4t| j dd }|tdkrr| j jjdd}t	|rr|d d krrd }t
|rt| j|j}t||dS t|rnt|r
tt|}| }|jg |d}| j\}	}
|	dkst|	dtj|
ftjd }|j|d	|d
S ttj|}tj| j|d}|| |S | jsN| j jsN| j jS | j jrj| j tjj}n| j j}| js|  }n&| j! D ]\}}t"j#|||d}q|S )NrI   rY   r   r   r   rH   r)   ro   T)Z
allow_fillrp   rQ   )$r^   rp   Z
get_valuesr   getattrrV   rI   r_   r   r5   r   fullrr   valuer   r   r   r   r   Zconstruct_array_typeZ_from_sequencer:   ZonesZintpZtakeemptyfillr@   Z_can_consolidateZis_boolrX   Zobject_rd   r8   rt   ru   )r   r   upcasted_narp   r_   Z	blk_dtypeZi8valuesrZ   Zmissing_arrZncolsZnrowsZ	empty_arrrA   rB   r*   r*   r1   get_reindexed_values  sR    


  


zJoinUnit.get_reindexed_values)N)r   
__module____qualname__r   r   r   r   rI   r   r   r   r*   r*   r*   r1   rx   C  s   
	rx   zlist[JoinUnit])rj   r%   r&   r'   c                   s   |dkrt | dkrtdt|  tdd | D }t | fdd| D }t |dkr|d }|rt|tjr|jdk	r|	 }q|	 }nFtd	d |D rd
d |D }t
|ddd}t|d}nt
||d}|S )zI
    Concatenate values from several join units along selected axis.
    r   r)   z$Concatenating join units along axis0c                 s  s   | ]}|j d kV  qd S r   r^   r-   rk   r*   r*   r1   r     s     z*_concatenate_join_units.<locals>.<genexpr>c                   s   g | ]}|j  d qS )r   r   )r   r`   r   r*   r1   r2     s   z+_concatenate_join_units.<locals>.<listcomp>Nc                 s  s   | ]}t |V  qd S r   r   r-   tr*   r*   r1   r     s     c                 S  s(   g | ] }t |r|n|d ddf qS r   Nr   r   r*   r*   r1   r2     s   T)r(   Zea_compat_axis   rQ   )r5   r:   _get_empty_dtyper   _dtype_to_na_valuerE   rV   Zndarraybaser&   r   r   )rj   r%   r&   has_none_blocksrD   Zconcat_valuesr*   r   r1   rg     s.    



rg   r   rI   r   c                 C  sv   t | tr| jS | jdkr$| dS | jdkr8| dS | jdkrFdS | jdkr^|sXdS tjS | jdkrntjS tdS )	z2
    Find the NA value to go with this dtype.
    rL   ZNaT)fcNaNrT   N)r.   uO)rE   r   r   rR   rU   rV   nanNotImplementedErrorr   r*   r*   r1   r     s    







r   zSequence[JoinUnit])rj   r'   c                 C  s   t | dkr*| d j}|dkr*ttjS t| rB| d jj}|S tdd | D }dd | D }t |sxdd | D }t|}|rt|}|S )	z
    Return dtype and N/A values to use when concatenating specified units.

    Returned N/A value may be None which means there was no casting involved.

    Returns
    -------
    dtype
    r)   r   Nc                 s  s   | ]}|j d kV  qd S r   r   r   r*   r*   r1   r   7  s     z#_get_empty_dtype.<locals>.<genexpr>c                 S  s"   g | ]}|j d k	r|js|jqS r   )r^   r   rI   r   r*   r*   r1   r2   9  s    
  z$_get_empty_dtype.<locals>.<listcomp>c                 S  s   g | ]}|j d k	r|jqS r   )r^   rI   r   r*   r*   r1   r2   =  s     
 )	r5   r^   rV   rI   Zfloat64_is_uniform_reindexr   r   r   )rj   rl   r   r   ZdtypesrI   r*   r*   r1   r   #  s"    

r   c                   s\   t  fdd D oZt  fdd D oZt dd  D oZt dd  D oZt dkS )z
    Check if the join units consist of blocks of uniform type that can
    be concatenated using Block.concat_same_type instead of the generic
    _concatenate_join_units (which uses `concat_compat`).

    c                 3  s&   | ]}t |jt  d  jkV  qdS r   )rU   r^   r`   rj   r*   r1   r   N  s     z)_is_uniform_join_units.<locals>.<genexpr>c                 3  s$   | ]}t |jj d  jjV  qdS r   )r   r^   rI   r`   r   r*   r1   r   Q  s   c                 s  s   | ]}|j  p|jjV  qd S r   )r   r^   rf   r`   r*   r*   r1   r   X  s     c                 s  s   | ]}|j  V  qd S r   )r@   r`   r*   r*   r1   r   [  s     r)   r|   r5   r   r*   r   r1   re   E  s    	

re   r   c                 C  s(   t dd | D o&tdd | D dkS )Nc                 s  s   | ]}|j o|j jV  qd S r   )r^   rf   r`   r*   r*   r1   r   e  s     z&_is_uniform_reindex.<locals>.<genexpr>c                 S  s   h | ]}|j jjqS r*   )r^   rI   namer`   r*   r*   r1   rJ   f  s     z&_is_uniform_reindex.<locals>.<setcomp>r)   r   r   r*   r*   r1   r   b  s    r   )	join_unitlengthr'   c                 C  s   d| j krF| j }| jdkr d}q| jt|d}| jt|| _n>| j}t| j }|d |d |d< | j d d| | j d< | jd | f| jdd  }|f| jdd  | _t|||dS )z
    Reduce join_unit's shape along item axis to length.

    Extra items that didn't fit are returned as a separate block.
    r   Nr)   r   )r@   r^   Zgetitem_blockslicer&   rr   rx   )r   r   Zextra_indexersZextra_blockZextra_shaper*   r*   r1   _trim_join_unitj  s    

r   )r%   c                 #  s  t | dkr2| d D ]}|d |d gfV  qn~|dkrd}| D ]@}d}|D ]\}}|||gfV  |}qN|dk	rB||jj7 }qBn(dg  fdd}ttt| } tt|| }	 d t |	kr d dkrtdt|	 \}
}ttt |
}t	|t
| }}||kr4|
d |fV  t|| |	dd< qd}dgt |	 }t|	D ]T\}\}}|||< t ||kr||d t||f|	|< n|}|| | |	|< qN||fV  qdS )z`
    Combine multiple concatenation plans into one.

    existing_plan is updated in-place.
    r)   r   Nc                   s&   t | d }|d kr" d  d7  < |S )Nr   r)   )next)seqretvalZ	num_endedr*   r1   _next_or_none  s    
z,_combine_concat_plans.<locals>._next_or_nonezPlan shapes are not aligned)r5   addrz   stopr;   mapiter
ValueErrorzipminmax	enumerater   )Zplansr%   poffsetr}   Zlast_plcZplcrk   r   Z
next_itemsr~   ZunitslengthsZmin_lenmax_lenZyielded_placementZyielded_unitsr.   r*   r   r1   rc     sD    
rc   )E
__future__r   r&   r<   typingr   r   r   ZnumpyrV   Zpandas._libsr   ry   Zpandas._typingr   r   r	   r
   Zpandas.util._decoratorsr   Zpandas.core.dtypes.castr   r   Zpandas.core.dtypes.commonr   r   r   r   r   r   Zpandas.core.dtypes.concatr   r   Zpandas.core.dtypes.dtypesr   Zpandas.core.dtypes.missingr   r   Zpandas.core.algorithmscoreZ
algorithmsrt   Zpandas.core.arraysr   r   Zpandas.core.constructionr   Z#pandas.core.internals.array_managerr   r   Zpandas.core.internals.blocksr   r   Zpandas.core.internals.managersr    Zpandasr!   rC   r3   rn   r]   rx   rg   r   r   re   r   r   rc   r*   r*   r*   r1   <module>   sB    )>DX 5"