U /e?@sHUdZddlmZddlZddlZddlZddlZddlZddlm Z ddl m Z ddl m Z mZddlZddlZddlm Z dd Zd d d d dZiZded<GdddZGdddeddZGdddeddZGdddeddZGdddeddZGd d!d!ZGd"d#d#ed$dZGd%d&d&ed'dZGd(d)d)ZdS)*aRA pytest plugin to trace resource leaks. Usage ----- This plugin is enabled from the command line with -L / --leaks. See `pytest --help` for further configuration settings. You may mark individual tests as known to be leaking with the fixture @pytest.mark.leaking(check1, check2, ...) Where the valid checks are 'fds', 'memory', 'threads', 'processes', 'tracemalloc'. e.g. @pytest.mark.leaking("threads") If you do, the specified checks won't report errors. Known issues ------------ - Tests that contain imports will be flagged as leaking RAM (memory and tracemallock checks) if it's the first time in the test suite that the import happens; e.g. def test1(): pytest.importorskip("numpy") Same issue happens with tests that populate global caches (e.g. linecache, re). A previous version of this plugin had an option to silently retry a test once after a failure; that version is no longer working as of the latest pytest. Reinstating the flag would solve this issue. See pytest_rerunfailures code for inspiration. - The @gen_cluster fixture leaks 2 fds on the first test decorated with it within a test suite; This issue would also be fixed by rerunning failing tests. - The @pytest.mark.flaky decorator (pytest_rerunfailures) completely disables this plugin for the decorated tests. - You cannot expect the process memory to go down immediately and deterministically as soon as you garbage collect Python objects. This makes the 'memory' check very unreliable. On Linux, this can be improved by reducing the MALLOC_TRIM glibc setting (see distributed.yaml). ) annotationsN) defaultdict)sleep)AnyClassVar)timecCs`|d}dtddtD}|jddd|dd |jd td d d |jdddddS)Nzresource leaksz, css|]}d|VqdS)z'%s'N).0srrD/tmp/pip-unpacked-wheel-g426oqom/distributed/pytest_resourceleaks.py ?sz#pytest_addoption..z-Lz--leakszList of resources to monitor for leaks before and after each test. Can be 'all' or a comma-separated list of resource names (possible values: z).)helpz--leaks-timeoutg?zWWait at most these many seconds before marking a test as leaking (default: %(default)s))typedefaultr z --leaks-fail store_truezMark leaked tests as failed)actionr )Zgetgroupjoinsorted all_checkersZ addoptionfloat)parsergroupZknown_checkersrrr pytest_addoption=s$  rz pytest.ConfigNone)configreturncCs|d}|sdS|dkr.tddtD}n |d}tt|tt}|r^td|dd|D}t||d |d d }|j|d dS) Nleaksallcss|]}|dkr|VqdS)demoNrr crrr r Zsz#pytest_configure..,zunknown resources: cSsg|]}t|qSrr)r Zleakrrr asz$pytest_configure..Z leaks_timeoutZ leaks_failcheckers grace_delay mark_failedZ leaks_checker) getvaluerrsplitset ValueError LeakCheckerZ pluginmanagerregister)rrunknownr%checkerrrr pytest_configureUs   r0z dict[str, type[ResourceChecker]]rc@seZdZUded<ddddZddd d Zddd d Zddd dZddddZddddddZ ddddddZ dS)ResourceCheckerz ClassVar[str]namestrr2cCs|tks t||_|t|<dSN)rAssertionErrorr2)clsr2rrr __init_subclass__ps z!ResourceChecker.__init_subclass__rrcCsdSr5rselfrrr on_start_testuszResourceChecker.on_start_testcCsdSr5rr:rrr on_stop_testxszResourceChecker.on_stop_testcCsdSr5rr:rrr on_retry{szResourceChecker.on_retryrcCstdSr5NotImplementedErrorr:rrr measure~szResourceChecker.measureboolbeforeafterrcCstdSr5r?r;rDrErrr has_leakszResourceChecker.has_leakcCstdSr5r?rFrrr formatszResourceChecker.formatN) __name__ __module__ __qualname____annotations__r8r<r=r>rArGrHrrrr r1ms r1c@sTeZdZUdZded<ddZddddZddd d d d Zddd d ddZdS) DemoCheckerzKChecker that always leaks. Used to test the core LeakChecker functionality.inticCs d|_dSNrrOr:rrr __init__szDemoChecker.__init__r9cCs|jd7_|jSNrQr:rrr rAszDemoChecker.measurerBrCcCs||kSr5rrFrrr rGszDemoChecker.has_leakr3cCsd|d|S)Nzcounter increased from z to rrFrrr rHszDemoChecker.formatN) rIrJrK__doc__rLrRrArGrHrrrr rMs rMrr4c@s>eZdZddddZddddddZddd dd d Zd S) FDCheckerrNr9cCstjdkrdStSdS)Nwin32r)sysplatformpsutilProcessZnum_fdsr:rrr rAs zFDChecker.measurerBrCcCs||kSr5rrFrrr rGszFDChecker.has_leakr3cCsd||d|d|dS)Nleaked z file descriptor(s) (z->)rrFrrr rHszFDChecker.formatNrIrJrKrArGrHrrrr rVsrVZfdsc@sBeZdZdZddddZdddddd Zddd dd d Zd S)RSSMemoryCheckerirNr9cCstjSr5)rZr[Z memory_infoZrssr:rrr rAszRSSMemoryChecker.measurerBrCcCs|||jkSr5LEAK_THRESHOLDrFrrr rGszRSSMemoryChecker.has_leakr3cCsd||dddS)Nr\.1fz MiB of RSS memoryrrFrrr rHszRSSMemoryChecker.formatN)rIrJrKrarArGrHrrrr r_sr_Zmemoryc@s>eZdZddddZddddddZddd dd d Zd S) ActiveThreadsCheckerzset[threading.Thread]r9cCs ttSr5)r* threading enumerater:rrr rAszActiveThreadsChecker.measurerBrCcCs ||k Sr5rrFrrr rGszActiveThreadsChecker.has_leakr3cCs,||}|stdt|dt|tdS)Nr\z Python thread(s): )key)r6lenrr3r;rDrEleakedrrr rHszActiveThreadsChecker.formatNr^rrrr rdsrdthreadsc@sneZdZUdZded<ded<ded<ded <d d d d ZddddZdddddZdddddZdS) ChildProcesszChild process info We use pid and creation time as keys to disambiguate between processes (and protect against pid reuse); other properties such as cmdline may change for a given process rNpidr3r2z list[str]cmdliner create_timezpsutil.Process)pcCs*|j|_||_||_||_dSr5)rmr2rnro)r;rprrr rRs  zChildProcess.__init__r9cCs|jSr5)rmr:rrr __hash__szChildProcess.__hash__objectrB)otherrcCs"t|to |j|jko |j|jkSr5) isinstancerlrmror;rsrrr __eq__s    zChildProcess.__eq__cCst|tst||j|jkSr5)rtrl TypeErrorrmrurrr __lt__s zChildProcess.__lt__N) rIrJrKrUrLrRrqrvrxrrrr rls rlc@s>eZdZddddZddddddZddd dd d Zd S) ChildProcessesCheckerzset[ChildProcess]r9c Cst}t}|jddD]}zl|Z||jkrptj | t j rpt dd|DrpW5QRWq|t|W5QRXWqtjk rYqXq|S)NT) recursivecss"|]}|dp|dVqdS)z1from multiprocessing.resource_tracker import mainz+from multiprocessing.forkserver import mainN) startswithr arrr r s z0ChildProcessesChecker.measure..)r*rZr[childrenZoneshotZppidrmospathsamefileZexerX executableanyrnaddrlZ NoSuchProcess)r;r~rpr rrr rAs$   zChildProcessesChecker.measurerBrCcCs ||k Sr5rrFrrr rG szChildProcessesChecker.has_leakr3cCs8t||}|stdt|dddd|DS)Nr\z processes:  css*|]"}d|jd|jd|jVqdS)z - pid=z, name=z , cmdline=N)rmr2rn)r rprrr r sz/ChildProcessesChecker.format..)rr6rhrrirrr rH s  zChildProcessesChecker.formatNr^rrrr rysryZ processesc@sfeZdZdZdZdZddddZdddd Zd dd d Zd d d dddZ d d ddddZ dS)TracemallocMemoryCheckerrbi rr9cCstddSrS) tracemallocstartr:rrr r<sz&TracemallocMemoryChecker.on_start_testcCs tdSr5)rstopr:rrr r=sz%TracemallocMemoryChecker.on_stop_testz tuple[int, tracemalloc.Snapshot]cCst\}}t}||fSr5)rZget_traced_memoryZ take_snapshot)r;current_Zsnaprrr rA"s z TracemallocMemoryChecker.measurerBrCcCs|d|d|jkSrPr`rFrrr rG'sz!TracemallocMemoryChecker.has_leakr3c Cs|\}}|\}}||d}d||dddg}|d|jD]`} | jpP| j} | |jkr`q| jpj| j} |d| ddd| dg7}|d d | jD7}qBd |S) N tracebackr\rbrcz MiB of traced Python memoryz - leaked z MiB in z calls at:cSsg|] }d|qS)z r)r linerrr r#Asz3TracemallocMemoryChecker.format..r) Z compare_toNDIFF size_diffsize MIN_SIZE_DIFFZ count_diffcountrrHr) r;rDrEZ bytes_beforeZ snap_beforeZ bytes_afterZ snap_afterZdifflinesstatrrrrr rH.s    zTracemallocMemoryChecker.formatN) rIrJrKrarrr<r=rArGrHrrrr rsrrc@seZdZUded<ded<ded<ded<d ed <d ed <d ed<ddddddZddddZdddddZdddddZdddddZddddd Z e j d!d"d#d$Z e j d!d"d%d&Z e j d!d!d'd(d)Ze j d!d!d'd*d+Ze j d,d-Zd.S)/r,zlist[ResourceChecker]r%rr&rBr'zdict[str, set[ResourceChecker]] skip_checkersz7dict[str, dict[ResourceChecker, list[tuple[Any, Any]]]]countersz1dict[str, list[tuple[ResourceChecker, Any, Any]]]rzdefaultdict[str, set[str]]outcomesr$cCs2||_||_||_i|_i|_i|_tt|_dSr5) r%r&r'rrrrr*r)r;r%r&r'rrr rRTszLeakChecker.__init__rr9cCs tdSr5)gcZcollectr:rrr cleanupcszLeakChecker.cleanupr3)nodeidrcsfddjDS)Ncs"g|]}|jdkr|qS)r)rgetrrr;rr r#gsz/LeakChecker.checks_for_item..)r%r;rrrr checks_for_itemfszLeakChecker.checks_for_itemz!list[tuple[ResourceChecker, Any]]cCsdd||DS)NcSsg|]}||fqSr)rArrrr r#ksz'LeakChecker.measure..)rrrrr rAiszLeakChecker.measurecCsT||D] }|q ||D],\}}|dk s6t|j|||dfq"dSr5)rr<rAr6rappend)r;rr/rDrrr measure_before_testms   zLeakChecker.measure_before_testc sj}d|krdSddfdd }t}|j}|}|rj|D]\}}}|qP|}|rt|krtd|D]\}}}|q|}qj|r|j<njdD] } | qdS)NZpassedz&list[tuple[ResourceChecker, Any, Any]]r9cs^g}D]J\}}j|}|d\}}||f|d<|||r||||fq|S)N)rArrGr)rr/rEr rDrrrr run_measurements|s   z8LeakChecker.measure_after_test..run_measurementsg?) rrr&rr>rrpoprr=) r;rrrt1deadlinerr rr/rrr measure_after_testts.      zLeakChecker.measure_after_testT) hookwrapperc#s|js dS|j}||jkstdd|jD|j|<|d}|rtt|jtt}|rtt d|dt tt dd|jDfdd |jD|j |<dVdS) NcSsi|] }|gqSrrrrrr sz7LeakChecker.pytest_runtest_protocol..Zleakingz'pytest.mark.leaking: unknown resources z; must be one of css|]}t|VqdSr5r"r|rrr r sz6LeakChecker.pytest_runtest_protocol..csh|]}t|r|qSr)rtrclassesrr s z6LeakChecker.pytest_runtest_protocol..) r%rrr6Zget_closest_markerrr*argsrr+listtupler)r;itemnextitemrZ leaking_markr.rrr pytest_runtest_protocols"   z#LeakChecker.pytest_runtest_protocolccs||jdVdSr5)rr)r;rrrr pytest_runtest_setups z LeakChecker.pytest_runtest_setup)rZtrylastccs6dV||j|j|j}|r2|jr2tdSr5)rrrrr'pytestZfail)r;rrrrr pytest_runtest_teardowns   z#LeakChecker.pytest_runtest_teardownc#sz|j|j|jdV}|jdkrv|j|j}|rv|jrl|dd|_d fdd|D|_ n |ddS)NZteardown)failedLLEAKEDrrc s0g|](\}}}d|jd|||qS) leaking : )r2rH)r r/rDrErrr r#sz8LeakChecker.pytest_report_teststatus..)rjrr) rrroutcomewhenrrr'Z force_resultrZlongrepr)r;reportrrrrr pytest_report_teststatuss   z$LeakChecker.pytest_report_teststatusc Csn|}|d}|rj|dd|D]F}|j}|j|D]0\}}} ||jd|jd||| q6q"dS)Nrj=zRESOURCE LEAKSrr)Z getreportsZ write_seprrrr2rH) r;ZterminalreporterZ exitstatustrrjreprr/rDrErrr pytest_terminal_summarys  z#LeakChecker.pytest_terminal_summaryN)rIrJrKrLrRrrrArrrZhookimplrrrrrrrrr r,Fs. 5        r,)rU __future__rrrrXrer collectionsrrrtypingrrrZrZdistributed.metricsrr0rrLr1rMrVr_rdrlryrr,rrrr s0+      ")2