U /e!n@sddlZddlZddlmZddlmZmZmZddlm Z m Z ej ddgdddZ dQd d Z d d ZddZddZddZejjddddZddZddZddZddZd d!Zd"d#Zd$d%Zd&d'Zd(d)Zejjd*dd+d,Zd-d.Zd/d0Z d1d2Z!d3d4Z"d5d6Z#d7d8Z$d9d:Z%d;d<Z&d=d>Z'd?d@Z(dAdBZ)dCdDZ*dEdFZ+dGdHZ,dIdJZ-dKdLZ.dMdNZ/dOdPZ0dS)RN)get_deps) diagnostics ndependenciesorder)addincabcdeZedcba)paramscCs|jSN)param)requestr 9/tmp/pip-unpacked-wheel-dbjnr7gq/dask/tests/test_order.pyr sFcCst||d|kS)N)reverse)sorted)Lrr r rissortedsrcGsdSr r argsr r rfsrc sN|\}}}}fddtdD}||dftdfdff|dftdfdffit|}t|df|dfdkstt|df|dfdkstfddtdD}||dftdfdff|dftdfdffit|}t|df|dfdks$tt|df|dfdksJtdS) Ncsi|]}|ftfqSr r.0iar r sz7test_ordering_keeps_groups_together..rcsi|]}|ftfqSr rrrr rrs)rangeupdaterrabsAssertionError)rbcdeor rr#test_ordering_keeps_groups_togethers6$$6&r*c CsX|\}}}}}|dftf|dftf|dft|dff|dft|dff|dft|dffi}t|}||df||dfkszt|dftf|dftf|dft|dff|dft|dff|dft|dffi}t|}||df||dfkst|dftf|dftf|dft|dff|dft|dff|dft|dffi}t|}||df||dfksTtdS)zV b0 b1 b2 | \ / a0 a1 a0 should be run before a1 rrrNrrr$rrr%r&r'r(dskr)r r rtest_avoid_broker_nodes'sP          r.cs|\}}fdddD}tdff|df<|fdddDd|<t|}|dfdkstt|dfdkstd S) z a3 /| a2 | /| | a1 | | /| | | a0 | | | | | | | b0 b1 b2 b3 \ \ / / c We really want to run b0 quickly cs(i|] }|ft|df|ffqSrrrrr%r rrcsz1test_base_of_reduce_preferred..)rrr rcsi|]}|ftdfqSr/rr)r%r&r rres)rrrr rr N)rr"rr$)rr'r(r-r)r rr%r&rtest_base_of_reduce_preferredRsr2zCan't please 'em all)reasoncCs|\}}}}}|dft|dff|dft|dff|dft|df|dff|dft|dff|dft|dff|dft|dff|dft|dffi}t|}||df||dfkstdS)z a1 | a2 | a3 d1 / \ / b1 c1 | | b2 c2 | c3 Prefer b1 over c1 because it won't stick around waiting for d1 to complete rrr Nr+r,r r rtest_avoid_upwards_branchingns$       r4cCs,|\}}}}}|dft|dff|dft|dff|dft|df|dff|dft|dff|dftf|dft|dff|dft|dff|dftf|dft|dff|dft|dff|dft|dff|dft|dff|dft|dffi }t|}||df||dfkstt||df||dfdks(tdS)a a1 | e2 a2 d2 d3 | | \ / e1 a3 d1 \ / \ / b1 c1 | | b2 c2 | c3 Prefer c1 over b1 because c1 will stay in memory less long while b1 computes rrr N)rrr$r#r,r r r$test_avoid_upwards_branching_complexs>          r5c Csl|\}}}}}|t|||f|t||f|t|f|d|di}t|}||||ksTt||||kshtdS)a{ It's not clear who should run first, e or d 1. d is nicer because it exposes parallelism 2. e is nicer (hypothetically) because it will be sooner released (though in this case we need d to run first regardless) Regardless of e or d first, we should run b before c. a / | \ . b c | / \ | / e d rrNr+r,r r r#test_deep_bases_win_over_dependentss *r6c Csf|\}}}}}|d|t|f|t|f|d|t|fi}t|}||||ksNt||||ksbtdS)zq c | e b | | d a Prefer longer chains first so we should start with c rNr+r,r r rtest_prefer_deeps $r7cCs,ddtdD}t|\}}t||dS)NcSs"i|]}d|dtd|fqS)zx%srrrr r rrs z#test_stacklimit..i')r!rr)rr- dependencies dependentsr r rtest_stacklimits r;c Csl|\}}}}}ddtdD}t|}t||d<t|}ddi} | ddt|D|| kshtdS)NcSsi|]}d|ft|fqSxr8rr r rrsz*test_break_ties_by_str.. ycSsi|]\}}||qSr r )rrkr r rrs)r!rlistrr" enumerater$) rrr%r&r'r(r-Zx_keysr)expectedr r rtest_break_ties_by_strs rDcCs"ttdftdftddfddS)Nrrr=r?r)r=rEz)rrr)rr r r)test_order_doesnt_fail_on_mixed_type_keyssrGc s8td}d\}}|jj||fdd}}tdD]D}|dddddf|dddddfjdd}|jdd}q2||dddfjdd}t|}t |fd d | D} tfd d | Dt | d kst fdd | D} tfdd | Dt | d ks"t t | | ks4t dS)N dask.array)c)rN)sizechunksrrZaxis)rrcsg|] }|qSr r rr@r)r r sz test_gh_3055..c3s|]}|tdkVqdSrNlenrr=rOr r sztest_gh_3055..r csg|]}|D] }|q qSr r )rkkr@rOr rrPsc3s|]}|tdkVqdSrQrRrTrOr rrUs)pytest importorskiprandomnormalr!ZcumsumsumdictZ__dask_graph__rZ __dask_keys__rSr$r) daABorigr=_r?wr-rr rOr test_gh_3055s  4 &(rccCs4|\}}}}}|d|dfd||dfdi}t|dS)Nrrr r)rrr%r&r'r(r-r r rtest_type_comparisions_ok srec Csl|\}}}}}|tf|t|f|t|f|t|f|t|fi}t|}||||ksTt||||kshtdS)z a | d b e \ | / c Prefer to finish d and e before starting b. That way c can be released during the long computations. Nr+r,r r rtest_prefer_short_dependentss *rfz#This is challenging to do preciselyc s|\}}}}}dd||||fD\}}}} ||||||||| g } gfdd} || |f|| |f|| |f|| |f|| |||f|| |||f|| ||f|| |||f| | | |fi } t| ||| g| kstdS)z aa / | b d bb dd / \ /| | / a c e cc Prefer to run acb first because then we can get that out of the way css|]}|dVqdSrQr rTr r rrU1sz,test_run_smaller_sections..csfdd}|S)NcsdSr appendrlogr=r rra8sz/test_run_smaller_sections..f.._r r=rarjr<rr7sz$test_run_smaller_sections..fN)daskgetr$) rrr%r&r'r(ZaaZbbccddrCrr-r rlrtest_run_smaller_sections%s4        rqcs|\}}fdddD\}}}fdddD\}}}fdddD\} } } ||||||| | | g } gfdd} || |f|| ||f|| ||f|| |f|| |||f|| ||f| | | f| | | | |f| | | | fi }t|t|||| g| kstdS) z c1 | b1 c2 | /| a1 b2 c3 | /| a2 b3 | a3 Prefer to finish a1 stack before proceeding to b2 c3s|]}|VqdSr r rrr rrU^sz2test_local_parents_of_reduction..Z123c3s|]}|VqdSr r rr%r rrU_sc3s|]}|VqdSr r r)r&r rrU`scsfdd}|S)NcsdSr rgrrir rragsz5test_local_parents_of_reduction..f.._r rkrlr<rrfsz*test_local_parents_of_reduction..fN)rrmrnr$)rr'r(a1a2a3b1b2b3Zc1c2c3rCrr-r )rr%r&rjrtest_local_parents_of_reductionNs:        r{cs@|\}}}fdddD\ }}}}}}} } } fdddD\} } }}| tf| tf|tf|tf|t| f|t| f|t| | f|t| f|t| |f|t|f| t||f| t|f| t|fi }t|dtfdddDkrdksntd tfd ddDkrd ksntt| | ||gd ks.Z 123456789c3s|]}|VqdSr r rrrr rrUsZ1234r c3s&|]}|tdkVqdSrQrRr)rr)r rrUsrc3s&|]}|tdkVqdSrQrRr)r%r)r rrUsrrN)rrr[r$min)rr&rarsrtruZa4Za5Za6Za7Za8Za9rvrwrxZb4r-r )rr%r)rtest_nearest_neighbor~sD $.2r~cCs2tftftfd}t|}|ddddks.tdS)z#Prefer ordering tasks by name first)rrrrrr rrrNr+r-r)r r rtest_string_orderingsrcCs>tdftdftdftfd}t|}|dddddks:tdS) z=Prefer ordering tasks by name first even when in dependenciesr%)rrrr%rrrr )r%rrrNr+rr r rtest_string_ordering_dependentssrcCs|\}}}}}|dfd|dfd|dfd|dft|df|df|dff|dfd|dfd|dft|df|df|dffi}t|}||df||dfkst||df||dfkst||df||dfkstdS)Nrrrr+)rrr%r&rar-r)r r rtest_prefer_short_narrows( rcCs:|\}}}}}||}|d|dft|ddf|dft|ddf|dfd|dft|df|df|dff|dft|ddf|dft|ddf|dft|df|df|dffi}t|}||df||dfkst||df||dfkst||df||dfkst||df||dfkst||df||dfks6tdS)a From https://github.com/dask/dask-ml/issues/206#issuecomment-395869929 Two cases, one where chunks of an array are independent, and one where the chunks of an array have a shared source. We handled the independent one "well" earlier. Good: c2 / \ \ / \ \ c1 \ \ / | \ \ \ c0 a0 b0 a1 b1 Bad: c2 / \ \ / \ \ c1 \ \ / | \ \ \ c0 a0 b0 a1 b1 \ \ / / \ \ / / a-b The difference is that all the `a` and `b` tasks now have a common ancestor. We would like to choose c1 *before* a1, and b1 because * we can release a0 and b0 once c1 is done * we don't need a1 and b1 to compute c1. rrrNr+)rrr%r&raabr-r)r r rtest_prefer_short_ancestors2&     rc#Csh|\}}}}}|dftf|dft|dff|dft|dff|dft|df|dff|dftf|dft|df|df|dff|dftf|dft|dff|dft|dff|dft|df|df|dff|dftf|dft|df|df|dff|dftf|dft|dff|dft|dff|dft|df|dffi}t|}||df||dfksd||df||dfksdtdS)z b1 b3 b5 |\ / | \ / | c1 c2 c3 c4 c5 |/ | \ | / | \| d1 d2 d3 d4 d5 | | | e1 e2 e5 Want to finish b1 before we start on e5 rrr rNr+r,r r rtest_map_overlapsH       rc,sf|\}}}dftfdftfdftfdftfdftfdftfdftfdftfd ftfd ftfdftdffdftdfdffd ftdfdfdffdftdfdfdffdftd fdfdffdftdfd fdffdftdfdfd ffd ftdfdfdffdftd fdfdffdftdfd fdffi}t|}tfd d |D}tfd d |D}|ddkst|ddkr|dddddd ddddg kst|dddd ddddddg ksbtnD|ddddd dddddg ks@t|ddddd dddddg ksbtdS)zSee https://github.com/dask/dask/issues/5584#issuecomment-554963958 We were using key names to infer structure, which could result in funny behavior. rrrr rrr| c3s"|]\\}}}|kr|VqdSr r rletterravalrr rrUAs z.test_use_structure_not_keys..c3s"|]\\}}}|kr|VqdSr r rrrr rrUBs >rr r>  Nrrritemsr$)rrar-r)ZAsZBsr r0rtest_use_structure_not_keys$sf "$"rc s|\}}}}d}|dfd|dfd|dfddft|df|df|dffi}td|D]P}t|dff|||f<t|dff|||f<t|df||f||ff||f<qTt|}ddddd d d d d dg } tfdd|D} | | kstdS)zFFrom https://github.com/dask/dask-ml/issues/206#issuecomment-395873372r>rrrr rrrrrc3s"|]\\}}}|kr|VqdSr r )rrnumvr'r rrUWs z9test_dont_run_all_dependents_too_early..N)rr!rrrr$) rrr%r&r(depthr-rr)rCactualr rr&test_dont_run_all_dependents_too_earlyLs8&rc(s|\}}}|dfd|dft|dff|dft|dff|dft|dff|dft|dff|dft|dffdft|dffdftdffdft|dffdftdffdftdfdffdft|dffdftdffdft|dffdftdffdftdfdff|dft|df|df|dfdfdffi}t|}dddddddd d d g }tfd d |D}||kst||df||dfdkstdS)a=From https://github.com/dask/dask/pull/5646#issuecomment-562700533 Sometimes we need larger or wider DAGs to test behavior. This test ensures we choose the branch with more work twice in successtion. This is important, because ``order`` may search along dependencies and then along dependents. rrrr rrrr|rrr>c3s*|]"\\}}}|hkr|VqdSr r )rrrarr'rpr(eer rrU}s z7test_many_branches_use_ndependencies..Nr)rrr%r&r-r)rCrr rr$test_many_branches_use_ndependencies[sV              " rc Cs4tjtddtdtdfidW5QRXtjtddtdtdfiW5QRXtjtddtdtdfiW5QRXtjtdd"ttdftdftdfdW5QRXtjtdd&ttdftdftddfd d W5QRXtjtdd*ttdftdftddftdfd W5QRXdS) NzCycle detected)matchrrrr%r&r1r'r)rr%r&r')rWZraises RuntimeErrorrmrnrrr r r rtest_order_cycles&*rcCstiikstdSr )rr$r r r rtest_order_emptysrc$Cs|\}}}}}|dfd|dft|dff|dft|dff|dft|dff|dft|df|dff|dft|dff|dft|dff|dft|dff|d ft|dff|dfd|dft|dff|dft|dff|dft|dff|dft|dff|dft|dff|dft|dff|dft|dffi}t|}||dfdks6t||df||dfksTt||df||dfksrt||df||dfkstd S) ax a7 a8 <-- do these last | / a6 e6 | / a5 c5 d5 e5 | | / / a4 c4 d4 e4 | \ | / / a3 b3---/ | a2 | a1 | a0 <-- start here Test that we are able to switch to better dependents. In this graph, we expect to start at a0. To compute a4, we need to compute b3. After computing b3, three "better" paths become available. Confirm that we take the better paths before continuing down `a` path. This test is pretty specific to how `order` is implemented and is intended to increase code coverage. rrrr rrrr|rNr+r,r r rtest_switching_dependentssR              rcCsp|\}}}}}i}||||g}|D]}||dfd|dft|dff|ddft|dff|ddft|dffit|D]\} } ||d| ft|ddf| ddff|d| ft|d| ff|d| dft|d| ff|d| dft|d| ff|d| dft|d| dff|d| dft|d| dffiqtq"t|} d} |D]P}tt|D]<} | |d| df| |d| df} | dksht| | 7} q6q&| dkstt|}|D]6}tt|D]"} t|d| dff||d | df<qqt|} d} |D]P}tt|D]<} | |d | df| |d| df} | dkst| | 7} qq| d ks:tt|}|D](}tt|D]} ||d| df=qVqFt|} d} |D]P}tt|D]<} | |d| df| |d| df} | dkst| | 7} qq| d kstt|}|D](}tt|D]} ||d| df=qqt|} d} |D]D}tt|D]0} | |d| df| |d| dfdks6tq6q&d S) zFrom https://github.com/dask/dask/issues/5859#issuecomment-608422198 See the visualization of `(maxima, argmax)` example from the above comment. This DAG has enough structure to exercise more parts of `order` rrrr rrrnr|bN)r"rrBrr!rSr$r\)rrr%r&r'r(r-abcr=rr?r)totalrZdsk2Zdsk3Zdsk4r r r test_order_with_equal_dependentss           $  rcCsfdddtddftddfddd d d td d ftd dfddtddddfd}t|}|d|d ksbtdS)a https://github.com/dask/dask/issues/6745 We have 1. A terminal node that depends on the entire graph ('s') 2. Some shared dependencies near the roots ('a1', 'a4') 3. But the left and right halves are disconnected, other than the terminal node. s / / \ \ / / \ \ s00 s10 s01 s11 | | | | b00 b10 b01 b11 / \ / \ / \ / \ a0 a1 a2 a3 a4 a5 Previously we started at 'a', and worked up to 's00'. We'd like to finish 's00' completely, so we progress to 's' and work through its dependencies. Ideally, we would choose 's10', since we've already computed one of its (eventual) dependencies: 'a1'. However, all of 's00' through 's11' had equal metrics so we fell back to the name tie-breaker and started on 's01' (via 'a3', a4', 'b01', ...). )rr/)rrrrr%rr%r)r )r)rrrrrrr%rr%r storerrrrrrrrrrr)rrrrrrrrrrrrrrrNr+rr r rtest_terminal_node_backtracks.rc stdtd}ddlfddtdD}j|ddd}||}|j|d d }|jd ||j d d }|j |d d d}t |j ddD}t |fdddfdd|Dfdd|DtfddD} t fddD} | | kstdS)NrHzarrrcs g|]}jdddqS))rrdr)rL)Z from_arrayZonesr)r]npr rrP]sz0test_array_store_final_order..rrMrT) overwritedest)namedatarLrF)lockZcomputecSs(g|] }t|tr|ddr|qS)rz store-map-) isinstancetuple startswithrNr r rrPgs cs|Sr r )r@rOr rhz.test_array_store_final_order..)keycs g|]}|ddkr|qSr rN first_storer rrPiscs g|]}|ddkr|qSrr rNrr rrPjsc3s|]\}}|kr|VqdSr r rr@r)connected_storesr rrUlsz/test_array_store_final_order..c3s|]\}}|kr|VqdSr r r)disconnected_storesr rrUms)rWrXZnumpyr!Z concatenateZrechunkZDirectoryStoregroupZ empty_like chunksizerrrmr}maxrr$) ZtmpdirrZarraysr=rrootrr'ZstoresZ connected_maxZdisconnected_minr )rr]rrrr)rtest_array_store_final_orderUs"    rcIs^tddftddftddftddftd d ftd d ftd dftddftddftdftdftdftdftdftdftdftdftdftdftdftdftdftd ftd!ftd"ftd#ftd$ftd%ftd&ftd'ftd(ftd)ftd*d+ftd'ftd,d-ftd ftd%ftd$ftd.d/ftd0d1ftd2d3ftd4d5ftd!ftdftdftd6d7ftd8d9ftd"ftd&ftd#ftd:d;ftdftd?ftd@ftdAftdBftdCftdDftdEftdFftdGfttfttfttfttfttfttfttfttfttfttfdHH}t|\t|fdIdJD}fdKdLfdMdN|D}t|}t|dOdPdQksZt|dRksZtdOS)Sahttps://github.com/dask/dask/pull/7929 This graph begins with many motifs like the following: | | c1 c2 \ / b | a We want to compute c2 and c3 pretty close together, because if we choose to compute c1, then we should also compute c2 so we can release b. Being greedy here allows us to release memory sooner and be more globally optimal. a06a08a28a26a24a21a22a25a29a20a23a27a04a02a00a01a05a03a43a36a33a47a44a42a37a48a49a35a46a55a53a60a54a59a56a61a52a57a58a19a07a30a31a11a09a14a10a38a40a18a17a34a50a16a15a51a45a12a13a41a39a62a68a70a67a71a64a65a63a69a66)Hrrrrrrrrrrr rrrr rrrr rrrrrrrrrrrrrZa32rrrrrr rr rrrrrrrrrrrrrrrrrrrrrrr rrr"rrr!rrcs h|]\}}|s|qSr )pop)rrdeps)r9r r szAtest_eager_to_compute_dependent_to_free_parent..cs|\}}t||Sr )r#)r$rr%rOr rcostsz.costcsi|]}||qSr r )rr)r&r:r rrszBtest_eager_to_compute_dependent_to_free_parent..Nr)rrrrrvaluesr[r$)r-parentsZ cost_of_pairsZcostsr )r&r9r:r)r.test_eager_to_compute_dependent_to_free_parentqsJ   r+c Cs |\}}}}}|dftf|dftf|dftf|dftf|dftf|dft|df|df|dff|dft|df|df|dff|dft|df|df|dff|dft|df|dff|dft|dffi }t|\}}|ddddddddddg kstdd|D|dfd|dfd|dfd|dfd|dfd|dfd|dfd |dfd |dfd |dfd i ksVtd d|D|dfd|dfd|dfd |dfd|dfd|dfd|dfd|dfd|dfd|dfdi kstdd|D|dfd|dfd|dfd|dfd|dfd|dfd|dfd|dfd|dfd|dfdi ks.tdd|D|dfd|dfd|dfd|dfd|dfd|dfd|dfd|dfd|dfd|dfdi kstdd|D|dfd|dfd|dfd|dfd|dfd|dfd|dfd|dfd|dfd|dfdi kstdS)z a1 b1 c2 d1 e1 /|\ /|\ /|\ /| / / | X | X | X | / / |/ \|/ \|/ \|/ a0 b0 c0 d0 e0 rrrr cSsi|]\}}||jqSr rdrrrr r rrsz$test_diagnostics..rrrr|rrcSsi|]\}}||jqSr )Zager,r r rrscSsi|]\}}||jqSr )Znum_dependencies_freedr,r r rrscSsi|]\}}||jqSr )Znum_data_when_runr,r r rrscSsi|]\}}||jqSr )Znum_data_when_releasedr,r r rr sN)rrr$r) rrr%r&r'r(r-infoZmemory_over_timer r rtest_diagnosticss        r.)F)1rWrmZ dask.corerZ dask.orderrrrZdask.utils_testrrZfixturerrrr*r.r2markZxfailr4r5r6r7r;rDrGrcrerfrqr{r~rrrrrrrrrrrrrrr+r.r r r rsT   +  '   (0%<%('7P9h