U n/e!@sXddlZddlZddlmZddlmZddlmZddlm Z m Z GdddeZ dS) N)tqdm)CallbackIOWrapper)Base)checkdatasetc@s<eZdZd ddZdddZdddZdd d Zd d ZdS)DatasetManagerNc Cst|tkr"tdt|dn$tj|sFtdtj|d|dkrVtdtj|j d||j d}|j d krtd |d |r|}n tj |}|dkr| ||n|}|d krd|}tj|j d |j ||t|tj||dd}|j dkr6d } |j dkr"|} t|j d| |} |jrX|| d| d<t|j} t|dD} t| dddd(} t| j| d}tj| d|dd}W5QRXW5QRX|j dkrd}n|jrt|jtd| d|idd | D} |dkrt j!|d!d"tj|j d | d#d$|j d!|id%}zt "|Wn8tk r}z| t|dd&W5d}~XYnX| d'dd(| S))an_summary_ Args: directory_id (_type_, optional): _description_. Defaults to None. file_path (_type_, optional): _description_. Defaults to None. description (str, optional): _description_. Defaults to "". lineage (_type_, optional): _description_. Defaults to None. Raises: Exception: _description_ Exception: _description_ Exception: _description_ Exception: _description_ Exception: _description_ Exception: _description_ Returns: _type_: _description_ zfile_path expect `str` but got z&, or please input `file_path=''`zfile z not foundNz-please input directory, like `directory=`z /directory/headersiz directory z> not found or your dose not has permission to access directoryr /file/)name description directorysizereplace)r data)iz, urlrbBT)totalunit unit_scaleZ unit_divisorreadF)rverifyrzSome thing wrong with datastoreuploadedcSsi|]\}}|dkr||qS))ridmsgpath).0keyvaluer"r"6/tmp/pip-unpacked-wheel-t2z2wkof/dsmlibrary/dataset.py Psz.DatasetManager.upload_file..lineage)rZdataNamerz /setLineage/r json)Z lineage_msgr(created)r(r()#typestr Exceptionosr!existsabspathrequestsget_discovery_api _jwt_header status_codebasenameZ_check_fileExistspostintgetsizer* _internal_replace_minio_apistatst_sizeopenrrupdateput_verboseprintcontentdecodeitemsrZcheck_lists_intcheck_http_status_code)self directory_id file_path file_namerr(r_resf_nametxtr file_sizeftZ wrapped_fileZ _uploadedrer"r"r& upload_file s          (  zDatasetManager.upload_filec Cs|dkrtdtj|jd|d|jd}|jdkrFtd||}tj|jd|d|jd}|jdkr~td |d }|jr||}t|}t j |j |d }|dkrt j |rt j |std nt j |j |d }t|jd d}|jr td|dd}t|ddd} t|d0} ||D]} | t| | | qHW5QRX| |t j ||d|S)a_summary_ Args: file_id (_type_, optional): _description_. Defaults to None. download_path (_type_, optional): _description_. Defaults to None. Raises: Exception: _description_ Exception: _description_ Exception: _description_ Exception: _description_ Returns: _type_: _description_ Nzplease input file_idr /r rzFile response code z /download/zsome thing wrong in datastorerr z1`download_path` is not existx or is not directoryzcontent-lengthrz total size : z BrZiBT)rrrwb)Zdownload_sucessf_path)r.r2r3r4r5r6r*r;r<r/r!joinZ _tmp_pathr0isdirr9r rBrCrr? iter_contentr@lenwriteclose) rHfile_id download_pathrLmeta_urlrWZtotal_size_in_bytes block_size progress_barrPrr"r"r& download_filejsB        zDatasetManager.download_filecCsltj|dtdtj|dttdtj|j||jd}tj |jd|j|||dd}tj |d | S) z_summary_ Args: name (_type_, optional): _description_. Defaults to None. tags (_type_, optional): _description_. Defaults to None. Returns: _type_: _description_ r variable variableNamedtypetags)rfrgrhchild)Zbase_uririheaderz/api/v2/dataset/)r titletagr)response) r check_typer-listrZ get_tag_ids_base_discovery_apir5r2r8rGr*)rHr riZtags_idsrRr"r"r& createDatasets  zDatasetManager.createDatasetcCsftj|dtdtj|dtdtj|dtdtj|jd|j|||dd}tj|d| S) a6_summary_ Args: directory_id (_type_, optional): _description_. Defaults to None. name (_type_, optional): _description_. Defaults to None. description (_type_, optional): _description_. Defaults to None. Returns: _type_: _description_ rIrer rz/api/v2/directory/)r r parent_dirr)rn) rrpr9r-r2r8rrr5rGr*)rHrIr rrRr"r"r&createDirectorys  zDatasetManager.createDirectorycCs>tj|jd|d|jd}|jdkr6td||S)Nr rUr rz!Cannot read meta data of file_id )r2r3r4r5r6r.r*)rHr^rLr"r"r& get_meta_files zDatasetManager.get_meta_file)NNNr NN)NN)NN)NNN)__name__ __module__ __qualname__rTrdrsrurvr"r"r"r&rs  ` 8  r) r/r2Z tqdm.autorZ tqdm.utilsrbaserutilsrrrr"r"r"r&s