U
    d/e5j                     @   s   d dl Z d dlZd dlmZ d dlmZ d dlZd dlmZmZm	Z	 ddl
mZmZ ddlmZmZ ddlmZ dd	lmZ dd
lmZ ddlmZmZmZmZ ddlmZ ddlmZmZm Z  G dd de!Z"dS )    N)contextmanager)time)urlparseparse_qsunquote   )errorsdefines)ColumnOrientedBlockRowOrientedBlock)
Connection)	log_block)ServerPacketTypes)IterQueryResultProgressQueryResultQueryResult	QueryInfo)escape_params)column_chunkschunksasboolc                       s  e Zd ZdZdZ fddZdd Zdd Zd	d
 Zdd Z	d<ddZ
d=ddZdd Zdd Zdd Zdd Zedd Zd>ddZd?dd Zd@d"d#ZdAd$d%ZdBd&d'ZdCd(d)ZdDd*d+ZdEd,d-ZdFd.d/Zd0d1 ZdGd2d3Zd4d5 ZdHd6d7Zd8d9 Ze d:d; Z!  Z"S )IClienta  
    Client for communication with the ClickHouse server.
    Single connection is established per each connected instance of the client.

    :param settings: Dictionary of settings that passed to every query (except
                     for the client settings, see below). Defaults to ``None``
                     (no additional settings). See all available settings in
                     `ClickHouse docs
                     <https://clickhouse.tech/docs/en/operations/settings/settings/>`_.
    :param \**kwargs: All other args are passed to the
                       :py:class:`~clickhouse_driver.connection.Connection`
                       constructor.

    The following keys when passed in ``settings`` are used for configuring the
    client itself:

        * ``insert_block_size`` -- chunk size to split rows for ``INSERT``.
          Defaults to ``1048576``.
        * ``strings_as_bytes`` -- turns off string column encoding/decoding.
        * ``strings_encoding`` -- specifies string encoding. UTF-8 by default.
        * ``use_numpy`` -- Use NumPy for columns reading. New in version
                           *0.2.0*.
        * ``opentelemetry_traceparent`` -- OpenTelemetry traceparent header as
                           described by W3C Trace Context recommendation.
                           New in version *0.2.2*.
        * ``opentelemetry_tracestate`` -- OpenTelemetry tracestate header as
                           described by W3C Trace Context recommendation.
                           New in version *0.2.2*.
        * ``quota_key`` -- A string to differentiate quotas when the user have
                           keyed quotas configured on server.
                           New in version *0.2.3*.
        * ``input_format_null_as_default`` -- Initialize null fields with
                           default values if data type of this field is not
                           nullable. Does not work for NumPy. Default: False.
                           New in version *0.2.4*.
    insert_block_sizestrings_as_bytesstrings_encoding	use_numpyopentelemetry_traceparentopentelemetry_tracestate	quota_keyinput_format_null_as_defaultc                    s*  | dd pi  | _t| j dtj| j dd| j dtj| j dd| j dd | j dd	| j d
d	| j ddd| _| jd rz*ddlm	}m
}m} || _|| _|| _W q tk
r   tdY qX nt| _t| _t| _t||| _| j| jj_| j| jj_|   tt|   d S )Nsettingsr   r   Fr   r   r   r    r   r    r   r   )NumpyIterQueryResultNumpyProgressQueryResultNumpyQueryResult"Extras for NumPy must be installed)popcopyr!   intr	   ZDEFAULT_INSERT_BLOCK_SIZEZSTRINGS_ENCODINGclient_settingsZnumpy.resultr#   r$   r%   query_result_clsiter_query_result_clsprogress_query_result_clsImportErrorRuntimeErrorr   r   r   r   
connectioncontextreset_last_querysuperr   __init__)selfargskwargsr#   r$   r%   	__class__ </tmp/pip-unpacked-wheel-mds_o03w/clickhouse_driver/client.pyr4   E   sd            

zClient.__init__c                 C   s   | S Nr:   r5   r:   r:   r;   	__enter__y   s    zClient.__enter__c                 C   s   |    d S r<   )
disconnect)r5   exc_typeexc_valexc_tbr:   r:   r;   __exit__|   s    zClient.__exit__c                 C   s   | j   |   dS )z.
        Disconnects from the server.
        N)r0   r?   r2   r=   r:   r:   r;   r?      s    
zClient.disconnectc                 C   s
   d | _ d S r<   )
last_queryr=   r:   r:   r;   r2      s    zClient.reset_last_queryFc                 C   s8   |   }|r| j|||dS | j|||d}| S d S )Nwith_column_typescolumnar)packet_generatorr-   r+   Z
get_result)r5   rF   progressrG   genresultr:   r:   r;   receive_result   s        zClient.receive_resultc                 c   s4   |   }| j||d}|D ]}|D ]
}|V  q"qd S NrF   )rH   r,   )r5   rF   rJ   rK   Zrowsrowr:   r:   r;   iter_receive_result   s     zClient.iter_receive_resultc              	   c   sP   z&|   }|sW qL|dkrW q |V  W q  ttfk
rH   |    Y q X q d S )NT)receive_packet	ExceptionKeyboardInterruptr?   r5   packetr:   r:   r;   rH      s    
zClient.packet_generatorc                 C   s   | j  }|jtjkr|jn|jtjkr<| j|j	 |S |jtj
krLdS |jtjkr\|S |jtjkrl|S |jtjkr||S |jtjkr| j|j dS dS d S )NFT)r0   rQ   typer   	EXCEPTION	exceptionZPROGRESSrD   Zstore_progressrI   END_OF_STREAMDATAZTOTALSZEXTREMESZPROFILE_INFOZstore_profileZprofile_inforT   r:   r:   r;   rQ      s$    
zClient.receive_packetc                 C   sd   t |pi }| j }| jD ]}||kr||||< q|| jj_| j }|| || jj_d S r<   )	dictr*   r(   available_client_settingsr'   r0   r1   r!   update)r5   r!   r*   keyZquery_settingsr:   r:   r;   make_query_settings   s    




zClient.make_query_settingsc                 C   s0   | d}| dr,|dd    | j_d S )Nz; zuse    )striplower
startswithr0   database)r5   queryr:   r:   r;   track_current_database   s    
zClient.track_current_databasec              	   c   sX   |  | z&| j  t | _d V  | | W n" ttfk
rR   |    Y nX d S r<   )	r_   r0   Zforce_connectr   rD   rf   rR   rS   r?   )r5   re   r!   r:   r:   r;   disconnect_on_error   s    

zClient.disconnect_on_errorNc	              
   C   s   t  }	| ||j t|tttjf}
|
rB| j||||||d}n| j|||||||d}| j	
t  |	  |W  5 Q R  S Q R X dS )a  
        Executes query.

        Establishes new connection if it wasn't established yet.
        After query execution connection remains intact for next queries.
        If connection can't be reused it will be closed and new connection will
        be created.

        :param query: query that will be send to server.
        :param params: substitution parameters for SELECT queries and data for
                       INSERT queries. Data for INSERT can be `list`, `tuple`
                       or :data:`~types.GeneratorType`.
                       Defaults to ``None`` (no parameters  or data).
        :param with_column_types: if specified column names and types will be
                                  returned alongside with result.
                                  Defaults to ``False``.
        :param external_tables: external tables to send.
                                Defaults to ``None`` (no external tables).
        :param query_id: the query identifier. If no query id specified
                         ClickHouse server will generate it.
        :param settings: dictionary of query settings.
                         Defaults to ``None`` (no additional settings).
        :param types_check: enables type checking of data for INSERT queries.
                            Causes additional overhead. Defaults to ``False``.
        :param columnar: if specified the result of the SELECT query will be
                         returned in column-oriented form.
                         It also allows to INSERT data in columnar form.
                         Defaults to ``False`` (row-like form).

        :return: * number of inserted rows for INSERT queries with data.
                   Returning rows count from INSERT FROM SELECT is not
                   supported.
                 * if `with_column_types=False`: `list` of `tuples` with
                   rows/columns.
                 * if `with_column_types=True`: `tuple` of 2 elements:
                    * The first element is `list` of `tuples` with
                      rows/columns.
                    * The second element information is about columns: names
                      and types.
        )external_tablesquery_idtypes_checkrG   paramsrF   rh   ri   rj   rG   N)r   rg   
isinstancelisttupletypesGeneratorTypeprocess_insert_queryprocess_ordinary_queryrD   store_elapsed)r5   re   rl   rF   rh   ri   r!   rj   rG   
start_timeZ	is_insertrvr:   r:   r;   execute   s.    ,      zClient.executec	           	   
   C   s<   |  ||& | j|||||||dW  5 Q R  S Q R X dS )a  
        Executes SELECT query with progress information.
        See, :ref:`execute-with-progress`.

        :param query: query that will be send to server.
        :param params: substitution parameters for SELECT queries and data for
                       INSERT queries. Data for INSERT can be `list`, `tuple`
                       or :data:`~types.GeneratorType`.
                       Defaults to ``None`` (no parameters  or data).
        :param with_column_types: if specified column names and types will be
                                  returned alongside with result.
                                  Defaults to ``False``.
        :param external_tables: external tables to send.
                                Defaults to ``None`` (no external tables).
        :param query_id: the query identifier. If no query id specified
                         ClickHouse server will generate it.
        :param settings: dictionary of query settings.
                         Defaults to ``None`` (no additional settings).
        :param types_check: enables type checking of data for INSERT queries.
                            Causes additional overhead. Defaults to ``False``.
        :param columnar: if specified the result will be returned in
                         column-oriented form.
                         Defaults to ``False`` (row-like form).
        :return: :ref:`progress-query-result` proxy.
        rk   N)rg   $process_ordinary_query_with_progress)	r5   re   rl   rF   rh   ri   r!   rj   rG   r:   r:   r;   execute_with_progress9  s        zClient.execute_with_progressr   c	           
   
   C   sP   |  ||: | j||||||d}	|dkr6t|	|n|	W  5 Q R  S Q R X dS )a  
        *New in version 0.0.14.*

        Executes SELECT query with results streaming. See, :ref:`execute-iter`.

        :param query: query that will be send to server.
        :param params: substitution parameters for SELECT queries and data for
                       INSERT queries. Data for INSERT can be `list`, `tuple`
                       or :data:`~types.GeneratorType`.
                       Defaults to ``None`` (no parameters  or data).
        :param with_column_types: if specified column names and types will be
                                  returned alongside with result.
                                  Defaults to ``False``.
        :param external_tables: external tables to send.
                                Defaults to ``None`` (no external tables).
        :param query_id: the query identifier. If no query id specified
                         ClickHouse server will generate it.
        :param settings: dictionary of query settings.
                         Defaults to ``None`` (no additional settings).
        :param types_check: enables type checking of data for INSERT queries.
                            Causes additional overhead. Defaults to ``False``.
        :param chunk_size: chunk query results.
        :return: :ref:`iter-query-result` proxy.
        )rl   rF   rh   ri   rj   r   N)rg   iter_process_ordinary_queryr   )
r5   re   rl   rF   rh   ri   r!   rj   
chunk_sizerv   r:   r:   r;   execute_iter^  s       zClient.execute_iterc           	   	   C   s`   zddl }W n tk
r(   tdY nX | j|dd||||d\}}|dd t||D S )a  
        *New in version 0.2.0.*

        Queries DataFrame with specified SELECT query.

        :param query: query that will be send to server.
        :param params: substitution parameters.
                       Defaults to ``None`` (no parameters  or data).
        :param external_tables: external tables to send.
                                Defaults to ``None`` (no external tables).
        :param query_id: the query identifier. If no query id specified
                         ClickHouse server will generate it.
        :param settings: dictionary of query settings.
                         Defaults to ``None`` (no additional settings).
        :return: pandas DataFrame.
        r   Nr&   T)rG   rF   rl   rh   ri   r!   c                 S   s$   i | ]\}}t d d|d |qS )z\W_r   )resub).0dcolr:   r:   r;   
<dictcomp>  s      z*Client.query_dataframe.<locals>.<dictcomp>)pandasr.   r/   rw   Z	DataFramezip)	r5   re   rl   rh   ri   r!   pddatacolumnsr:   r:   r;   query_dataframe  s         
zClient.query_dataframec              
      s   zddl }W n tk
r(   tdY nX t }| || | jj||d | j| |  }d}	|rdd |j	D }
 fdd|
D }| j
||dd	}	|   | jt |  |	W  5 Q R  S Q R X dS )
ay  
        *New in version 0.2.0.*

        Inserts pandas DataFrame with specified query.

        :param query: query that will be send to server.
        :param dataframe: pandas DataFrame.
        :param external_tables: external tables to send.
                                Defaults to ``None`` (no external tables).
        :param query_id: the query identifier. If no query id specified
                         ClickHouse server will generate it.
        :param settings: dictionary of query settings.
                         Defaults to ``None`` (no additional settings).
        :return: number of inserted rows.
        r   Nr&   ri   c                 S   s   g | ]}|d  qS )r   r:   )r   xr:   r:   r;   
<listcomp>  s     z+Client.insert_dataframe.<locals>.<listcomp>c                    s   g | ]} | j qS r:   )values)r   column	dataframer:   r;   r     s     T)rG   )r   r.   r/   r   rg   r0   
send_querysend_external_tablesreceive_sample_blockcolumns_with_types	send_datareceive_end_of_queryrD   rt   )r5   re   r   rh   ri   r!   r   ru   sample_blockrv   r   r   r:   r   r;   insert_dataframe  s"    zClient.insert_dataframec                 C   sJ   |d k	r|  ||| jj}| jj||d | jj||d | j|d|dS )Nr   rj   T)rF   rI   rG   substitute_paramsr0   r1   r   r   rL   r5   re   rl   rF   rh   ri   rj   rG   r:   r:   r;   rx     s       z+Client.process_ordinary_query_with_progressc                 C   sH   |d k	r|  ||| jj}| jj||d | jj||d | j||dS )Nr   r   rE   r   r   r:   r:   r;   rs     s      zClient.process_ordinary_queryc                 C   sF   |d k	r|  ||| jj}| jj||d | jj||d | j|dS )Nr   r   rN   )r   r0   r1   r   r   rP   )r5   re   rl   rF   rh   ri   rj   r:   r:   r;   rz     s      z"Client.iter_process_ordinary_queryc           	      C   sN   | j j||d | j j||d |  }|rJ| j||||d}|   |S d S )Nr   r   )rj   rG   )r0   r   r   r   r   r   )	r5   Zquery_without_datar   rh   ri   rj   rG   r   rv   r:   r:   r;   rr     s     zClient.process_insert_queryc                 C   sv   | j  }|jtjkr|jS |jtjkr0|jq |jtjkrHt	|j q |jtj
krVq | j d|j}t|q d S )Nz$Data, Exception, Log or TableColumns)r0   rQ   rV   r   rZ   blockrW   rX   LOGr   TABLE_COLUMNSunexpected_packet_messager   UnexpectedPacketFromServerErrorr5   rU   messager:   r:   r;   r     s    
 zClient.receive_sample_blockc                 C   s   d}| j jj}|rtnt}|d rdz"ddlm} |r:|}	ntdW qp tk
r`   t	dY qpX n|rltnt
}	|	||d D ]*}
||j|
|d}| j | ||j7 }q~| j |  |S )	Nr   r   r   )r   z0NumPy inserts is only allowed with columnar=Truer&   r   r   )r0   r1   r*   r
   r   Znumpy.helpersr   
ValueErrorr.   r/   r   r   r   Znum_rows)r5   r   r   rj   rG   Zinserted_rowsr*   Z	block_clsZnumpy_column_chunksZslicerchunkr   r:   r:   r;   r   !  s,    
zClient.send_datac                 C   st   | j  }|jtjkrqpq |jtjkr.|jq |jtjkrFt|j	 q |jtj
krTq | j d|j}t|q d S )NzException, EndOfStream or Log)r0   rQ   rV   r   rY   rW   rX   r   r   r   r   r   r   r   r   r:   r:   r;   r   B  s    
 zClient.receive_end_of_queryc                 C   s   | j   | j|dS rM   )r0   Zsend_cancelrL   )r5   rF   r:   r:   r;   cancelX  s    
zClient.cancelc                 C   s$   t |tstdt||}|| S )Nz$Parameters are expected in dict form)rm   r[   r   r   )r5   re   rl   r1   escapedr:   r:   r;   r   ^  s    

zClient.substitute_paramsc           
      C   s  t |}i }i }|j}|jdk	r*|j|d< |jddd}|rF||d< |jdk	r^t|j|d< |jdk	rvt|j|d< |jd	krd
|d< dddh}dddh}t	|j
 D ]<\}}	|	rt|	sq|	d }	|dkr|	 }	|	|kr|	||< nt|	||< q|dkrt|	||< q|dkr*t|	||< q|dkr>|	||< q||krVt|	||< q|dkrnt|	||< q|dkrt|	||< q|dkrt|	||< q|dkrtt|	||< q|dkr|	||< q|dkr|	|d< q|	||< q|r||d< | |f|S )a  
        Return a client configured from the given URL.

        For example::

            clickhouse://[user:password]@localhost:9000/default
            clickhouses://[user:password]@localhost:9440/default

        Three URL schemes are supported:
            clickhouse:// creates a normal TCP socket connection
            clickhouses:// creates a SSL wrapped TCP socket connection

        Any additional querystring arguments will be passed along to
        the Connection class's initializer.
        Nport/r"   r   rd   userpasswordZclickhousesTsecureZlz4Zlz4hcZzstdconnect_timeoutZsend_receive_timeoutZsync_request_timeoutr   compressionr   Zclient_nameZcompress_block_sizeZsettings_is_importantverifyssl_version)ca_certscipherskeyfilecertfileZ	alt_hostsr!   )r   hostnamer   pathreplaceusernamer   r   schemer   re   itemslenrb   r   floatr)   getattrssl)
clsurlr!   r7   hostr   Zcompression_algsZtimeoutsnamevaluer:   r:   r;   from_urle  sj    




















zClient.from_url)FFF)F)NFNNNFF)NFNNNFF)NFNNNFr   )NNNN)NNN)NFNNFF)NFNNFF)NFNNF)NNFF)FF)F)#__name__
__module____qualname____doc__r\   r4   r>   rC   r?   r2   rL   rP   rH   rQ   r_   rf   r   rg   rw   ry   r|   r   r   rx   rs   rz   rr   r   r   r   r   r   classmethodr   __classcell__r:   r:   r8   r;   r      s   %4  


         
D           
&           
%      
$    
*         
         
       
      

!
r   )#r~   r   
contextlibr   r   rp   urllib.parser   r   r   r"   r   r	   r   r
   r   r0   r   logr   protocolr   rK   r   r   r   r   Zutil.escaper   Zutil.helpersr   r   r   objectr   r:   r:   r:   r;   <module>   s   