U
    /eY                     @  s   d dl mZ d dlZd dlZd dlZd dlZd dlZd dlZd dlm	Z	 d dl
mZ d dlmZ d dlmZmZ d dlZd dlmZ d dlmZ d d	lmZ erd d
lmZ eeZdd ZdddddZd$ddddddZdddddZ G dd dZ!ddddddd d!d"d#Z"dS )%    )annotationsN)Iterable)import_module)
ModuleType)TYPE_CHECKINGcast)tmpfile)Server)import_file)Clientc                 C  s   |s| j dds|S |rr| j ddsrdd |D }|D ]}t|q:tdt|dkr`dndd	|f d
d | j dD }dd | D }t|dkrtdt|	  |r|std|s|S |d }tj
|dd} || t| |S )zKClick option callback providing validation of preload subcommand arguments.preloadNc                 S  s   g | ]}| d r|qS )-
startswith).0v r   :/tmp/pip-unpacked-wheel-g426oqom/distributed/preloading.py
<listcomp>$   s     
 z)validate_preload_argv.<locals>.<listcomp>z%Got unexpected extra argument%s: (%s)   s  c                 S  s   i | ]}t |s|t|qS r   )is_webaddress_import_module)r   namer   r   r   
<dictcomp>,   s    z)validate_preload_argv.<locals>.<dictcomp>c                 S  s,   g | ]$}t t|d dtjrt|d dqS )
dask_setupN)
isinstancegetattrclickCommand)r   mr   r   r   r   2   s   z<Multiple --preload modules with click-configurable setup: %szPUnknown argument specified: %r Was click-configurable --preload target provided?r   FZallow_extra_args)paramsgetr    ZNoSuchOptionZ
UsageErrorlenjoinvalueslistkeysContext
parse_args)ctxparamvalueunexpected_argsaZpreload_modulesZpreload_commandsZpreload_commandr   r   r   validate_preload_argv   sB    

r2   strbool)r   returnc                 C  s
   |  dS )N)zhttp://zhttps://r   )r   r   r   r   r   M   s    r   
str | Noner   )r   file_dirr5   c              
   C  s   |  drx|dk	rjtj| }tj||}tj|rPt| |sPt	d| t
| | t|d }qt| d }nrd| kr| tjkrt|  tj|  }nLtdd<}t|dd}||  W 5 Q R X t||d	W  5 Q R  S Q R X td
|  |S )a  Imports module and extract preload interface functions.

    Import modules specified by name and extract 'dask_setup'
    and 'dask_teardown' if present.

    Parameters
    ----------
    name : str
        Module name, file path, or text of module or script
    file_dir : string
        Path of a directory where files should be copied

    Returns
    -------
    Nest dict of names to extracted module interface components if present
    in imported module.
    z.pyNzFile name collision: %sr   r   )	extensionw)moder7   zImport preload module: %s)endswithospathbasenamer'   existsfilecmpcmploggererrorshutilcopyr
   sysmodulesr   r   openwriter   info)r   r7   r?   Zcopy_dstmodulefnfr   r   r   r   Q   s(    

r   )urlr5   c              	   C  s   t d|  t| stdd l}| 0}|jd| |jjddddgdd	d
}|j	}W 5 Q R X t
|| d}t| }t||j |S )NzDownloading preload at %sr   GETi  i  i  i  g?)status_forcelistbackoff_factor)methodrO   retriesexec)rC   rK   r   AssertionErrorurllib3PoolManagerrequestutilRetrydatacompiler   rU   __dict__)rO   rW   httpresponsesourceZcompiledrL   r   r   r   _download_module   s"    

	rb   c                   @  s^   e Zd ZU dZded< ded< ded< ded	< d
ed< dddddddZdd Zdd ZdS )Preloada  
    Manage state for setup/teardown of a preload module

    Parameters
    ----------
    dask_server: dask.distributed.Server
        The Worker or Scheduler
    name: str
        module name, file name, or web address to load
    argv: [str]
        List of string arguments passed to click-configurable `dask_setup`.
    file_dir: str
        Path of a directory where files should be copied
    Server | Clientdask_objectr3   r   	list[str]argvr6   r7   r   rL   zIterable[str])re   r   rg   r7   c                 C  sN   || _ || _t|| _|| _td| j t|r>t|| _	nt
||| _	d S )NzCreating preload: %s)re   r   r)   rg   r7   rC   rK   r   rb   rL   r   )selfre   r   rg   r7   r   r   r   __init__   s    
zPreload.__init__c                   s   t | jdd}|rtd| j t|tjrl|jd| j	dd}|j
| jf|j|j}t|r|I dH  n|| j}t|r|I dH  dS )z-Run when the server finishes its start methodr   NzRun preload setup: %sFr#   )r   rL   rC   rK   r   r   r    r!   Zmake_contextrg   callbackre   argsr$   inspectisawaitable)rh   r   contextresultfuturer   r   r   start   s*      


zPreload.startc                   sB   t | jdd}|r>td| j || j}t|r>|I dH  dS )z+Run when the server starts its close methoddask_teardownNzRun preload teardown: %s)r   rL   rC   rK   r   re   rl   rm   )rh   rr   rp   r   r   r   teardown   s    

zPreload.teardownN)__name__
__module____qualname____doc____annotations__ri   rq   rs   r   r   r   r   rc      s   
rc   r;   rd   zstr | list[str]zlist[str] | list[list[str]]zlist[Preload])dask_serverr   preload_argvr7   r5   c                  s   t |tr|g}|r8t |d tr8td|gt| }n|sPtdg gt| }t|t|kr|tdt| dt|  fddt||D S )Nr   rf   z1preload and preload_argv have mismatched lengths z != c                   s   g | ]\}}t  ||qS r   )rc   )r   prg   ry   r7   r   r   r      s   z$process_preloads.<locals>.<listcomp>)r   r3   r   r&   
ValueErrorzip)ry   r   rz   r7   r   r|   r   process_preloads   s    
r   )N)#
__future__r   rA   rl   loggingr=   rE   rG   collections.abcr   	importlibr   typesr   typingr   r   r    Z
dask.utilsr   Zdistributed.corer	   Zdistributed.utilsr
   Zdistributed.clientr   	getLoggerrt   rC   r2   r   r   rb   rc   r   r   r   r   r   <module>   s0   
10L