U /ed@s>ddlZddlmZddlmZmZGdddZddZdS)N)_get_pyarrow_dtypes_meta_from_dtypesc@s<eZdZeddZeddZeddZeddZd S) ArrowORCEnginec Kst|dkr(||ds(||d}d}g} dd} |rd} |D]} || d} t| }|dkrr|j}n||jkrtdtt |j }| r| | |d| fg| |j kr| | || | t |fg| t |7} q|rt |dkr| |j 8} nd} W5QRXqFnR|D]L} |dkrV||dd} t| }|j}W5QRX| | dfgqt |dd}|dk rt|t|}|rtd|d t|d |||| } |dkrt|n|}t|tr|gn|}t|||g}| ||fS) NrcSs&|dkr|j}n||jkr"td|S)N,Incompatible schemas while parsing ORC files)schema ValueError)Z_orr ?/tmp/pip-unpacked-wheel-dbjnr7gq/dask/dataframe/io/orc/arrow.py _get_schemas  z1ArrowORCEngine.read_metadata.._get_schemarbr) categorieszRequested columns (z) not in schema ())lenisfilefindopenorcORCFilerrlistrangenstripesappendintrset_aggregate_files isinstancestrr)clsfspathscolumnsindex split_stripesaggregate_fileskwargsrpartsr offsetpathfo_stripesexmetar r r read_metadatasV        zArrowORCEngine.read_metadatac Cs|dkrt|dkrt|dkrg}|d}t|dd}|ddD]J}t|dd}|||kr||d||7}qH|||}|}qH|||S|SdS)NTrr)rrr) rr$r#r&Z new_partsZnew_partrpartZ next_nstripesr r r rPs     zArrowORCEngine._aggregate_filesc Ks:g}|D]\}}|t|||||7}qtj|jddS)NF)Zdate_as_object)_read_orc_stripespaTableZ from_batchesZ to_pandas) rrr&rr!r%batchesr(stripesr r r read_partitionds zArrowORCEngine.read_partitionc Ks@tj|}||j||gd}t||W5QRXdS)Nwb)r1r2Z from_pandasrsepjoinrZ write_table)rZdfr(rfilenamer%tabler)r r r write_partitionks zArrowORCEngine.write_partitionN)__name__ __module__ __qualname__ classmethodr.rr5r;r r r r rs G  rc Csl|dkrt|}g}||dB}t|}|dkr>t|jn|}|D]} ||| |qFW5QRX|S)Nr )rrrrrrrZ read_stripe) rr(r4rr!r3r)r*r+Zstriper r r r0rs r0) Zpyarrowr1Z pyarrow.orcrZdask.dataframe.io.utilsrrrr0r r r r s k