U
    d/epc                     @   s   d Z ddlmZmZmZ ddlZddlZddlZddlZddl	Z	ddl
Z
ddlZddlmZ ddlmZmZ ddlmZmZ ddlmZ d	Zd
ZdZdZdZe
dZe
de
jZe
dZ e
de
jZ!e
de
jZ"e
ed e
jZ#e
de
jZ$e
ee
jZ%e
de
jZ&dJddZ'dKddZ(dLddZ)dd  Z*d!d" Z+d#d$ Z,dMd&d'Z-d(d) Z.dNd*d+Z/d,d- Z0d.d/ Z1d0d1 Z2d2d3 Z3d4d5 Z4d6d7 Z5dOd8d9Z6d:d; Z7d<d= Z8d>d? Z9d@dA Z:G dBdC dCZ;G dDdE dEZ<G dFdG dGeZ=G dHdI dIZ>dS )PzHelper functions.    )absolute_importdivisionunicode_literalsN)Queue)BoundedSemaphoreThread   )SseSseCustomerKey)to_iso8601utci'  l        l        i  P z^(((bucket\.|accesspoint\.)vpce(-(?!_)[a-z_\d]+(?<!-)(?<!_))+\.s3\.)|((?!s3)(?!-)(?!_)[a-z_\d-]{1,63}(?<!-)(?<!_)\.)s3-control(-(?!_)[a-z_\d]+(?<!-)(?<!_))*\.|(s3(-(?!_)[a-z_\d]+(?<!-)(?<!_))*\.))z$^[a-z0-9][a-z0-9\.\-]{1,61}[a-z0-9]$z'^[a-z0-9][a-z0-9_\.\-\:]{1,61}[a-z0-9]$zm^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9][0-9]|[0-9])\.){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9][0-9]|[0-9])$zS^((?!-)(?!_)[a-z_\d-]{1,63}(?<!-)(?<!_)\.)*((?!_)(?!-)[a-z_\d-]{1,63}(?<!-)(?<!_))$z.*\.amazonaws\.com(|\.cn)$zF((?!s3)(?!-)(?!_)[a-z_\d-]{1,63}(?<!-)(?<!_)\.)*amazonaws\.com(|\.cn)$zc^(?!-)(?!_)[a-z_\d-]{1,63}(?<!-)(?<!_)\.(?!-)(?!_)[a-z_\d-]{1,63}(?<!-)(?<!_)\.elb\.amazonaws\.com$z)^((?!_)(?!-)[a-z_\d-]{1,63}(?<!-)(?<!_))$/c                 C   s   t jj| |||dddS )z^
    Wrapper to urllib.parse.quote() replacing back to '~' for older python
    versions.
    )safeencodingerrorsz%7E~)urllibparsequotereplace)resourcer   r   r    r   1/tmp/pip-unpacked-wheel-xery97c7/minio/helpers.pyr   H   s     r    c                 C   s   t | |||S )zEncode query parameter value.)r   )queryr   r   r   r   r   r   queryencodeU   s    r   Fc                    s6   fdd fddd  fdd|  D S )z*Convert HTTP headers to multi-line string.c                    s    r|   S | S N)titlekey
titled_keyr   r   _get_key\   s    z$headers_to_strings.<locals>._get_keyc                    s2    r.t ddt ddt| tr"| nt| S | S )NzCredential=([^/]+)zCredential=*REDACTED*zSignature=([0-9a-f]+)zSignature=*REDACTED*)resub
isinstancestr)valuer   r   r   
_get_value_   s    	z&headers_to_strings.<locals>._get_value
c                    s&   g | ]\}} | d | qS )z: r   .0r   r&   )r!   r'   r   r   
<listcomp>k   s   z&headers_to_strings.<locals>.<listcomp>)joinitems)headersr    r   )r!   r'   r    r   headers_to_stringsZ   s    r/   c                 C   sn   |dkr8|t k r td| d|tkr8td| d| dkrZ| tkrjtd|  dn|dkrjtddS )	zValidate object and part size.r   z
part size z' is not supported; minimum allowed 5MiBz' is not supported; maximum allowed 5GiBobject size z' is not supported; maximum allowed 5TiBz<valid part size must be provided when object size is unknownN)MIN_PART_SIZE
ValueErrorMAX_PART_SIZEMAX_MULTIPART_OBJECT_SIZEobject_size	part_sizer   r   r   _validate_sizesr   s$    


r8   c                 C   s|   t | | | dk r|dfS |dkrFt|| }||r@t| | ndfS tt| t t t }||rvt| | ndfS )2Compute part information for object and part size.r   r   )r8   minmathceilMAX_MULTIPART_COUNTr1   r5   r   r   r   _get_part_info   s    

r?   c                 C   s:   t | |\}}|tkr2td|  d| dt d||fS )r9   r0   z and part size z make more than z parts for upload)r?   r>   r2   )r6   r7   
part_countr   r   r   get_part_info   s    rA       c                 C   s^   |t |8 }|rZ| |}|s qZt|ts2td||7 }|t |8 }|r|t | q|S )z)Read part data of given size from stream.z!read() must return 'bytes' object)lenreadr$   bytesr2   update)streamsizeZ	part_dataprogressdatar   r   r   read_part_data   s    

rK   c              
   C   sf   z| rt |  W nN tk
r` } z0|jtjkr2 t j| sPtd|  d|W 5 d}~X Y nX dS )z.Wrapper of os.makedirs() ignores errno.EEXIST.zpath z is not a directoryN)osmakedirsOSErrorerrnoEEXISTpathisdirr2   )rQ   excr   r   r   rM      s    rM   c                    s   |rt  s6td  nt s6td  t rPtd  ddddg}t fdd|D rtd  d	|r d
s ds drtd  ddS )zECheck whether bucket name is valid optional with strict check or not.zinvalid bucket name zbucket name z' must not be formatted as an IP addressz..z.-z-.c                 3   s   | ]}| kV  qd S r   r   )r*   xbucket_namer   r   	<genexpr>   s     z$check_bucket_name.<locals>.<genexpr>z' contains invalid successive characterszxn--z-s3aliasz--ol-s3zJ must not start with 'xn--' and must not end with '--s3alias' or '--ol-s3'N)_BUCKET_NAME_REGEXmatchr2   _OLD_BUCKET_NAME_REGEX_IPV4_REGEXany
startswithendswith)rV   strictZs3_checkZunallowed_successive_charsr   rU   r   check_bucket_name   s&    



r`   c              
   C   sB   z|   st W n* tk
r< } zt |W 5 d}~X Y nX dS )z(Check whether given string is not empty.N)stripr2   AttributeError	TypeError)stringrS   r   r   r   check_non_empty_string   s
    
re   c                 C   s"   t | ttfstdt|  dS )z
    Validate if policy is type str

    :param policy: S3 style Bucket policy.
    :return: True if policy parameter is of a valid type, 'string'.
    Raise :exc:`TypeError` otherwise.
    z policy must be str or bytes typeT)r$   r%   rE   rc   re   )policyr   r   r   is_valid_policy_type   s    rg   c                 C   s   | rt | tstddS )z(Check sse is SseCustomerKey type or not.zSseCustomerKey type is requiredN)r$   r
   r2   sser   r   r   
check_ssec   s    rj   c                 C   s   | rt | tstddS )zCheck sse is Sse type or not.zSse type is requiredN)r$   r	   r2   rh   r   r   r   	check_sse   s    rk   c                 C   sZ   | dkrdS t jddd}|t| tr0|  n|  t| }t|t	rV|
 S |S )z<Compute MD5 of data and return hash as Base64 encoded value.Nmd5F)Zusedforsecurity)hashlibnewrF   r$   r%   encodebase64	b64encodedigestrE   decode)rJ   hasherZmd5sumr   r   r   md5sum_hash  s    ru   c                 C   sJ   | pd} t  }|t| tr&|  n|  | }t|trF| S |S )z=Compute SHA-256 of data and return hash as hex encoded value.rB   )	rm   sha256rF   r$   r%   ro   	hexdigestrE   rs   )rJ   rt   Z	sha256sumr   r   r   sha256_hash  s
    rx   c                 C   sZ   t j|dk	r|n| j|dk	r"|n| j|dk	r2|n| j|dk	rB|n| j|dk	rR|n| jS )z5Return new URL with replaced properties in given URL.N)r   r   SplitResultschemenetlocrQ   r   fragment)urlrz   r{   rQ   r   r|   r   r   r   url_replace  s    r~   c                    s8   dd  dd fdd fdd| p.i   D S )	z!Convert user metadata to headers.c                 S   s   |   dsd|  } | S )Nzx-amz-meta-zX-Amz-Meta-lowerr]   r   r   r   r   normalize_key+  s    z+_metadata_to_headers.<locals>.normalize_keyc              
   S   sP   t | } z| d W n4 tk
rJ } ztd|  d|W 5 d }~X Y nX | S )Nzus-asciizunsupported metadata value z0; only US-ASCII encoded characters are supported)r%   ro   UnicodeEncodeErrorr2   )r&   rS   r   r   r   	to_string0  s    
z'_metadata_to_headers.<locals>.to_stringc                    s&   t | ttfs| g}  fdd| D S )Nc                    s   g | ]} |qS r   r   r*   r&   r   r   r   r+   >  s     zA_metadata_to_headers.<locals>.normalize_value.<locals>.<listcomp>)r$   listtuple)valuesr   r   r   normalize_value;  s    z-_metadata_to_headers.<locals>.normalize_valuec                    s   i | ]\}} ||qS r   r   r)   )r   r   r   r   
<dictcomp>@  s    z(_metadata_to_headers.<locals>.<dictcomp>)r-   )metadatar   )r   r   r   r   _metadata_to_headers)  s    
r   c                    sX   dd pi   D dd   fdd  D }fdd|D }t| S )z?Normalize headers by prefixing 'X-Amz-Meta-' for user metadata.c                 S   s   i | ]\}}t ||qS r   )r%   r)   r   r   r   r   H  s      z%normalize_headers.<locals>.<dictcomp>c                 S   s   |   } | dp| dk S )Nzx-amz-)zcache-controlzcontent-encodingzcontent-typezcontent-dispositionzcontent-languager   r   r   r   r   guess_user_metadataJ  s    
z.normalize_headers.<locals>.guess_user_metadatac                    s   i | ]\}} |r||qS r   r   r)   )r   r   r   r   W  s    c                    s   g | ]}  |qS r   )pop)r*   r   )r.   r   r   r+   ]  s     z%normalize_headers.<locals>.<listcomp>)r-   rF   r   )r.   Zuser_metadata_r   )r   r.   r   normalize_headersF  s    
r   c                 C   sx   t | } | |r| ni  ddd |p.i  D }|rF|| d< |rh|jrh|j| d< t|j| d< |rtd| d< | S )	z&Generate headers for given parameters.&c                 S   s$   g | ]\}}t |d  t | qS =r   r)   r   r   r   r+   h  s   zgenheaders.<locals>.<listcomp>zx-amz-taggingzx-amz-object-lock-modez#x-amz-object-lock-retain-until-dateONzx-amz-object-lock-legal-hold)r   rF   r.   r,   r-   moder   Zretain_until_date)r.   ri   tagsZ	retentionZ
legal_holdZtaggingr   r   r   
genheadersc  s     


r   c           	      C   sZ  t | sdS t| r>| ddd dd }d|p:|fS t| sLdS t| sdtd|  t|  }| d| }d	|kr|std
|  | |d d}|d dk}|r|dd }d}|d dkr|d }|dd }d	|}| dkrd}| dkrd}|
drB|
dsB|sB|sBtd|  |||pN||ddfS )z Extract AWS domain information. )NNz.elb.amazonaws.comr   r   .r:   Nzinvalid Amazon AWS host zs3-accesspointzuse HTTPS scheme for host 	dualstack)ZvpceZ	amazonawss3-external-1.amazonaws.comz	us-east-1)s3-us-gov-west-1.amazonaws.com#s3-fips-us-gov-west-1.amazonaws.comzus-gov-west-1.cns3-accelerate.z+region missing in Amazon S3 China endpoint )	s3_prefixdomain_suffixregionr   )_HOSTNAME_REGEXrY   _AWS_ELB_ENDPOINT_REGEXsplit_AWS_ENDPOINT_REGEX_AWS_S3_ENDPOINT_REGEXr2   _AWS_S3_PREFIX_REGEXendr,   r^   )	hosthttpsr   region_in_hostr   aws_s3_prefixtokensr   Zaws_domain_suffixr   r   r   _get_aws_infoy  sV    






r   c                   @   s   e Zd ZdZdd Zedd Zedd Zedd	 Zed
d Z	edd Z
e
jdd Z
edd Zejdd Zedd Zejdd Zedd Zejdd Zdd Zdd ZdddZdS ) BaseURLzBase URL of S3 endpoint.c              
   C   s  t j|}|j}|j dkr(tdt||j d}|jrR|jdkrRtdt|dd}|j	rltd|j
rztd	z
|j W n, tk
r } ztd
|W 5 d }~X Y nX |jrtd|jrtd|jdkr|jdks|jdkr|jdkrt||d}|r&t|s&td| t||jdk|\| _}| jpN|d| _|| _|p`|| _d| _| jr| jd | _| jd d| _d S )N)httpr   z(scheme in endpoint must be http or https)rz   r   zpath in endpoint is not allowedr   rQ   z query in endpoint is not allowedz#fragment in endpoint is not allowedzinvalid portz#username in endpoint is not allowedz#password in endpoint is not allowedr   P   r   i  r{   zinvalid region zaliyuncs.comFr   r   r   )r   r   urlsplithostnamerz   r   r2   r~   rQ   r   r|   portusernamepassword_REGION_REGEXrY   r   	_aws_infor^   _virtual_style_flag_url_region_accelerate_host_flag)selfZendpointr   r}   r   rS   r   r   r   r   __init__  sZ    
  
zBaseURL.__init__c                 C   s   | j S )zGet region.)r   r   r   r   r   r     s    zBaseURL.regionc                 C   s   | j jdkS )zCheck if scheme is HTTPS.r   )r   rz   r   r   r   r   is_https  s    zBaseURL.is_httpsc                 C   s   | j jS )zGet hostname.)r   r{   r   r   r   r   r     s    zBaseURL.hostc                 C   s
   | j dk	S )z Check if URL points to AWS host.Nr   r   r   r   r   is_aws_host  s    zBaseURL.is_aws_hostc                 C   s   | j r| j d S dS )zGet AWS S3 domain prefix.r   Nr   r   r   r   r   r     s    zBaseURL.aws_s3_prefixc                 C   s,   t |std| | jr(|| jd< dS )zSet AWS s3 domain prefix.zinvalid AWS S3 domain prefix r   N)r   rY   r2   r   )r   r   r   r   r   r      s    
c                 C   s   | j S )zGet AWS accelerate host flag.r   r   r   r   r   accelerate_host_flag  s    zBaseURL.accelerate_host_flagc                 C   s
   || _ dS )zSet AWS accelerate host flag.Nr   r   flagr   r   r   r     s    c                 C   s   | j r| j d S dS )z*Check if URL points to AWS dualstack host.r   Fr   r   r   r   r   dualstack_host_flag  s    zBaseURL.dualstack_host_flagc                 C   s   | j r|| j d< dS )zSet AWS dualstack host.r   Nr   r   r   r   r   r     s    c                 C   s   | j S )"Check to use virtual style or not.r   r   r   r   r   virtual_style_flag  s    zBaseURL.virtual_style_flagc                 C   s
   || _ dS )r   Nr   r   r   r   r   r   "  s    c           	      C   s   | j d }| j d }| | }|dkr4t||dS |}d|krjd|krXtd| d|rj|d	d
d}| j d r||d7 }d|kr||d 7 }||7 }t||dS ) Build URL for given information.r   r   r   r   r   r   zs3-accelerater   zbucket name 'z1' with '.' is not allowed for accelerate endpointz-accelerater   r   r   z
dualstack.)r   r~   r2   r   )	r   r}   rV   enforce_path_styler   r   r   r   r{   r   r   r   _build_aws_url'  s&    



zBaseURL._build_aws_urlc                 C   s   | j s
|S | j d }| j d }| | }|dkr>t||dS |dsR|drrd}|drddnd}d	| }t|| | d
| dS )zBuild URL for ListBuckets API.r   r   r   r   zs3.zs3-r   r   zamazonaws.comr   )r   r~   r]   r^   )r   r}   r   r   r   r   Z	cn_suffixr   r   r   _build_list_buckets_urlD  s    


zBaseURL._build_list_buckets_urlNc                    sD  |s|rt d| t| jdd}g }t|p0i  D ]:\ }t|ttfrR|n|g}| fddt|D 7 }q8t|d|d}|s| 	||S |dkr| r| p|rd	|kpd
|ko| jj
dk}	| jr| |||	|}|j}
d}|	s| jsd| }n| d
|
 }
|r6||dr(dndt| 7 }t||
|dS )r   z"empty bucket name for object name r   r   c                    s"   g | ]}t   d t | qS r   r   r   r   r   r   r+   g  s   z!BaseURL.build.<locals>.<listcomp>r   )r   PUTlocationr   r   r   )r{   rQ   )r2   r~   r   sortedr-   r$   r   r   r,   r   rz   r   r   r{   r   r^   r   )r   methodr   rV   object_nameZquery_paramsr}   r   r   r   r{   rQ   r   r   r   buildX  sD    

    zBaseURL.build)NNN)__name__
__module____qualname____doc__r   propertyr   r   r   r   r   setterr   r   r   r   r   r   r   r   r   r   r     s@   6











     r   c                   @   sn   e Zd ZdZdddZedd Zedd Zed	d
 Zedd Z	edd Z
edd Zedd ZdS )ObjectWriteResultz/Result class of any APIs doing object creation.Nc                 C   s.   || _ || _|| _|| _|| _|| _|| _d S r   )_bucket_name_object_name_version_id_etag_http_headers_last_modified	_location)r   rV   r   
version_idetaghttp_headerslast_modifiedr   r   r   r   r     s    zObjectWriteResult.__init__c                 C   s   | j S )zGet bucket name.)r   r   r   r   r   rV     s    zObjectWriteResult.bucket_namec                 C   s   | j S )zGet object name.)r   r   r   r   r   r     s    zObjectWriteResult.object_namec                 C   s   | j S )zGet version ID.)r   r   r   r   r   r     s    zObjectWriteResult.version_idc                 C   s   | j S )z	Get etag.)r   r   r   r   r   r     s    zObjectWriteResult.etagc                 C   s   | j S )zGet HTTP headers.)r   r   r   r   r   r     s    zObjectWriteResult.http_headersc                 C   s   | j S )zGet last-modified time.)r   r   r   r   r   r     s    zObjectWriteResult.last_modifiedc                 C   s   | j S )zGet location.)r   r   r   r   r   r     s    zObjectWriteResult.location)NN)r   r   r   r   r   r   rV   r   r   r   r   r   r   r   r   r   r   r     s$      






r   c                   @   s    e Zd ZdZdd Zdd ZdS )Workerz1 Thread executing tasks from a given tasks queue c                 C   s,   t j| dd || _|| _|| _|   d S )NT)daemon)r   r   _tasks_queue_results_queue_exceptions_queuestart)r   Ztasks_queueZresults_queueZexceptions_queuer   r   r   r     s
    zWorker.__init__c              
   C   s   | j  }|s| j   q| j r|\}}}}zNz|||}| j| W n. tk
rz } z| j| W 5 d}~X Y nX W 5 |  X | j   q dS )z- Continuously receive tasks and execute them N)r   get	task_doner   emptyr   put	Exception)r   Ztaskfuncargskargscleanup_funcresultexr   r   r   run  s    



"z
Worker.runN)r   r   r   r   r   r   r   r   r   r   r     s   r   c                   @   s0   e Zd ZdZdd Zdd Zdd Zdd	 Zd
S )
ThreadPoolz. Pool of threads consuming tasks from a queue c                 C   s,   t  | _t  | _t  | _t|| _|| _d S r   )r   r   r   r   r   _sem_num_threads)r   Znum_threadsr   r   r   r     s
    
zThreadPool.__init__c                 O   s*   | j   | j j}| j||||f dS )a  
        Add a task to the queue. Calling this function can block
        until workers have a room for processing new tasks. Blocking
        the caller also prevents the latter from allocating a lot of
        memory while workers are still busy running their assigned tasks.
        N)r   acquirereleaser   r   )r   r   r   r   r   r   r   r   add_task  s    
zThreadPool.add_taskc                 C   s&   t | jD ]}t| j| j| j q
dS )z Prepare threads to run tasksN)ranger   r   r   r   r   r   r   r   r   r   start_parallel  s      zThreadPool.start_parallelc                 C   s@   t | jD ]}| jd q
| j  | j s:| j | jS )z8 Stop threads and return the result of all called tasks N)	r  r   r   r   r,   r   r   r   r   r  r   r   r   r     s    


zThreadPool.resultN)r   r   r   r   r   r  r  r   r   r   r   r   r     s
   r   )r   NN)r   NN)F)rB   N)FF)NNNNN)?r   
__future__r   r   r   rp   rO   rm   r<   rL   r"   urllib.parser   queuer   	threadingr   r   ri   r	   r
   timer   r>   r4   r3   r1   Z_AWS_S3_PREFIXcompilerX   
IGNORECASErZ   r[   r   r   r   r   r   r   r   r   r/   r8   r?   rA   rK   rM   r`   re   rg   rj   rk   ru   rx   r~   r   r   r   r   r   r   r   r   r   r   r   r   <module>   s   





	
         
5 `3!