U
    /e                  	   @  s8  U d Z ddlmZ ddlZddlmZ ddlmZ ddlm	Z	 ddl
mZ ddlmZ dd	lmZ dd
lmZ ddlZddlmZmZmZ deediZded< ed ed< dZeeZee  ddlZej ej!ded< W 5 Q R X eeV ddl"Z"ze" e#d W n e$k
r*   edY nX e"j e"j!ded< dZW 5 Q R X eeT ddl%Z%ee%j&edk rzedddl'Z%e%j(j e	e%j(j!ddded< dZW 5 Q R X eeH ddl)Z)ee)j&edk reddd Z*dd Z+e*e+ded< W 5 Q R X d d! Z,e,  d"d# Z-d$d$d%efd&d'Z.d(d) Z!dS )*zY
Record known compressors

Includes utilities for determining whether or not to compress
    )annotationsN)Callable)suppress)partial)randint)Literal)parse)identity)ensure_memoryviewnbytes
no_default)compress
decompresszfdict[str | None | Literal[False], dict[Literal[('compress', 'decompress')], Callable[[bytes], bytes]]]compressionsFzlib    zNeed snappy >= 0.5.3snappyz0.23.1zNeed lz4 >= 0.23.1T)Zreturn_bytearraylz4z0.9.0zNeed zstandard >= 0.9.0c                 C  s(   t jtjdtjdd}|| S )Nzdistributed.comm.zstd.levelzdistributed.comm.zstd.threads)levelthreads)	zstandardZZstdCompressordaskconfiggetr   )dataZzstd_compressor r   D/tmp/pip-unpacked-wheel-g426oqom/distributed/protocol/compression.pyzstd_compressY   s
    

r   c                 C  s   t  }|| S )N)r   ZZstdDecompressorr   )r   Zzstd_decompressorr   r   r   zstd_decompress`   s    r   Zzstdc               	   C  sH   t jd} | dkrtS | tkr$| S td| dttt	tf d S )Ndistributed.comm.compressionautoz<Default compression '%s' not found.
Choices include auto, %sz, )
r   r   r   default_compressionr   
ValueErrorjoinsortedmapstr)defaultr   r   r   get_default_compressiong   s    r(   c           	      C  s   |dkr|dkst |dks$|dkr,tdS t| } |dg }| j| }td|}t|d D ]0}td|}t|| |}| || ||< |}q^| |||  |d< |dkr|d S td|S dS )a  Sample a bytestring from many locations

    Parameters
    ----------
    b : bytes or memoryview
    size : int
        target size of each sample to collect
        (may be smaller if samples collide)
    n : int
        number of samples to collect
    r   r   N   )AssertionError
memoryviewr
   r   r   rangeminr#   )	bsizenpartsZ	max_startstartiZ
next_startendr   r   r   byte_samplew   s     



r6   i'     c           	      C  s   |t krtjd}|s d| fS |t|   kr8dksBn d| fS |dkrNt}t| d }t| }t|||}t	||d|j kr||}t	|d|j kr||fS d| fS )aX  
    Maybe compress payload

    1.  We don't compress small messages
    2.  We sample the payload in a few spots, compress that, and if it doesn't
        do any good we return the original
    3.  We then compress the full original, it it doesn't compress well then we
        return the original
    4.  We return the compressed result
    r   Nl        r    r   g?)
r   r   r   r   r   r!   r   r
   r6   len)	payloadZmin_sizeZsample_sizeZnsamplescompressionr   mvsample
compressedr   r   r   maybe_compress   s     r>   c                 C  s   dd t | d |D S )z8Decompress frames according to information in the headerc                 S  s    g | ]\}}t | d  |qS )r   )r   ).0cframer   r   r   
<listcomp>   s   zdecompress.<locals>.<listcomp>r:   )zip)headerframesr   r   r   r      s    r   )/__doc__
__future__r   loggingcollections.abcr   
contextlibr   	functoolsr   randomr   typingr   Zpackaging.versionr   parse_versionZtlzr	   r   Zdistributed.utilsr
   r   r   r   __annotations__r!   	getLogger__name__loggerImportErrorr   r   r   r   r,   	TypeErrorr   __version__Z	lz4.blockblockr   r   r   r(   r6   r>   r   r   r   r   <module>   sl   






	
$
-