EVOLUTION-MANAGER
Edit File: parallel.html
<!DOCTYPE html> <html> <head> <meta charset="utf-8" /> <meta name="generator" content="pandoc" /> <meta http-equiv="X-UA-Compatible" content="IE=EDGE" /> <meta name="viewport" content="width=device-width, initial-scale=1" /> <title>Running tests in parallel</title> <script>// Pandoc 2.9 adds attributes on both header and div. We remove the former (to // be compatible with the behavior of Pandoc < 2.8). document.addEventListener('DOMContentLoaded', function(e) { var hs = document.querySelectorAll("div.section[class*='level'] > :first-child"); var i, h, a; for (i = 0; i < hs.length; i++) { h = hs[i]; if (!/^h[1-6]$/i.test(h.tagName)) continue; // it should be a header h1-h6 a = h.attributes; while (a.length > 0) h.removeAttribute(a[0].name); } }); </script> <style type="text/css"> code{white-space: pre-wrap;} span.smallcaps{font-variant: small-caps;} span.underline{text-decoration: underline;} div.column{display: inline-block; vertical-align: top; width: 50%;} div.hanging-indent{margin-left: 1.5em; text-indent: -1.5em;} ul.task-list{list-style: none;} </style> <style type="text/css"> code { white-space: pre; } .sourceCode { overflow: visible; } </style> <style type="text/css" data-origin="pandoc"> pre > code.sourceCode { white-space: pre; position: relative; } pre > code.sourceCode > span { display: inline-block; line-height: 1.25; } pre > code.sourceCode > span:empty { height: 1.2em; } .sourceCode { overflow: visible; } code.sourceCode > span { color: inherit; text-decoration: inherit; } div.sourceCode { margin: 1em 0; } pre.sourceCode { margin: 0; } @media screen { div.sourceCode { overflow: auto; } } @media print { pre > code.sourceCode { white-space: pre-wrap; } pre > code.sourceCode > span { text-indent: -5em; padding-left: 5em; } } pre.numberSource code { counter-reset: source-line 0; } pre.numberSource code > span { position: relative; left: -4em; counter-increment: source-line; } pre.numberSource code > span > a:first-child::before { content: counter(source-line); position: relative; left: -1em; text-align: right; vertical-align: baseline; border: none; display: inline-block; -webkit-touch-callout: none; -webkit-user-select: none; -khtml-user-select: none; -moz-user-select: none; -ms-user-select: none; user-select: none; padding: 0 4px; width: 4em; color: #aaaaaa; } pre.numberSource { margin-left: 3em; border-left: 1px solid #aaaaaa; padding-left: 4px; } div.sourceCode { } @media screen { pre > code.sourceCode > span > a:first-child::before { text-decoration: underline; } } code span.al { color: #ff0000; font-weight: bold; } code span.an { color: #60a0b0; font-weight: bold; font-style: italic; } code span.at { color: #7d9029; } code span.bn { color: #40a070; } code span.bu { color: #008000; } code span.cf { color: #007020; font-weight: bold; } code span.ch { color: #4070a0; } code span.cn { color: #880000; } code span.co { color: #60a0b0; font-style: italic; } code span.cv { color: #60a0b0; font-weight: bold; font-style: italic; } code span.do { color: #ba2121; font-style: italic; } code span.dt { color: #902000; } code span.dv { color: #40a070; } code span.er { color: #ff0000; font-weight: bold; } code span.ex { } code span.fl { color: #40a070; } code span.fu { color: #06287e; } code span.im { color: #008000; font-weight: bold; } code span.in { color: #60a0b0; font-weight: bold; font-style: italic; } code span.kw { color: #007020; font-weight: bold; } code span.op { color: #666666; } code span.ot { color: #007020; } code span.pp { color: #bc7a00; } code span.sc { color: #4070a0; } code span.ss { color: #bb6688; } code span.st { color: #4070a0; } code span.va { color: #19177c; } code span.vs { color: #4070a0; } code span.wa { color: #60a0b0; font-weight: bold; font-style: italic; } </style> <script> // apply pandoc div.sourceCode style to pre.sourceCode instead (function() { var sheets = document.styleSheets; for (var i = 0; i < sheets.length; i++) { if (sheets[i].ownerNode.dataset["origin"] !== "pandoc") continue; try { var rules = sheets[i].cssRules; } catch (e) { continue; } var j = 0; while (j < rules.length) { var rule = rules[j]; // check if there is a div.sourceCode rule if (rule.type !== rule.STYLE_RULE || rule.selectorText !== "div.sourceCode") { j++; continue; } var style = rule.style.cssText; // check if color or background-color is set if (rule.style.color === '' && rule.style.backgroundColor === '') { j++; continue; } // replace div.sourceCode by a pre.sourceCode rule sheets[i].deleteRule(j); sheets[i].insertRule('pre.sourceCode{' + style + '}', j); } } })(); </script> <style type="text/css">body { background-color: #fff; margin: 1em auto; max-width: 700px; overflow: visible; padding-left: 2em; padding-right: 2em; font-family: "Open Sans", "Helvetica Neue", Helvetica, Arial, sans-serif; font-size: 14px; line-height: 1.35; } #TOC { clear: both; margin: 0 0 10px 10px; padding: 4px; width: 400px; border: 1px solid #CCCCCC; border-radius: 5px; background-color: #f6f6f6; font-size: 13px; line-height: 1.3; } #TOC .toctitle { font-weight: bold; font-size: 15px; margin-left: 5px; } #TOC ul { padding-left: 40px; margin-left: -1.5em; margin-top: 5px; margin-bottom: 5px; } #TOC ul ul { margin-left: -2em; } #TOC li { line-height: 16px; } table { margin: 1em auto; border-width: 1px; border-color: #DDDDDD; border-style: outset; border-collapse: collapse; } table th { border-width: 2px; padding: 5px; border-style: inset; } table td { border-width: 1px; border-style: inset; line-height: 18px; padding: 5px 5px; } table, table th, table td { border-left-style: none; border-right-style: none; } table thead, table tr.even { background-color: #f7f7f7; } p { margin: 0.5em 0; } blockquote { background-color: #f6f6f6; padding: 0.25em 0.75em; } hr { border-style: solid; border: none; border-top: 1px solid #777; margin: 28px 0; } dl { margin-left: 0; } dl dd { margin-bottom: 13px; margin-left: 13px; } dl dt { font-weight: bold; } ul { margin-top: 0; } ul li { list-style: circle outside; } ul ul { margin-bottom: 0; } pre, code { background-color: #f7f7f7; border-radius: 3px; color: #333; white-space: pre-wrap; } pre { border-radius: 3px; margin: 5px 0px 10px 0px; padding: 10px; } pre:not([class]) { background-color: #f7f7f7; } code { font-family: Consolas, Monaco, 'Courier New', monospace; font-size: 85%; } p > code, li > code { padding: 2px 0px; } div.figure { text-align: center; } img { background-color: #FFFFFF; padding: 2px; border: 1px solid #DDDDDD; border-radius: 3px; border: 1px solid #CCCCCC; margin: 0 5px; } h1 { margin-top: 0; font-size: 35px; line-height: 40px; } h2 { border-bottom: 4px solid #f7f7f7; padding-top: 10px; padding-bottom: 2px; font-size: 145%; } h3 { border-bottom: 2px solid #f7f7f7; padding-top: 10px; font-size: 120%; } h4 { border-bottom: 1px solid #f7f7f7; margin-left: 8px; font-size: 105%; } h5, h6 { border-bottom: 1px solid #ccc; font-size: 105%; } a { color: #0033dd; text-decoration: none; } a:hover { color: #6666ff; } a:visited { color: #800080; } a:visited:hover { color: #BB00BB; } a[href^="http:"] { text-decoration: underline; } a[href^="https:"] { text-decoration: underline; } code > span.kw { color: #555; font-weight: bold; } code > span.dt { color: #902000; } code > span.dv { color: #40a070; } code > span.bn { color: #d14; } code > span.fl { color: #d14; } code > span.ch { color: #d14; } code > span.st { color: #d14; } code > span.co { color: #888888; font-style: italic; } code > span.ot { color: #007020; } code > span.al { color: #ff0000; font-weight: bold; } code > span.fu { color: #900; font-weight: bold; } code > span.er { color: #a61717; background-color: #e3d2d2; } </style> </head> <body> <h1 class="title toc-ignore">Running tests in parallel</h1> <p>To take advantage of parallel tests, add the following line to the <code>DESCRIPTION</code>:</p> <pre><code>Config/testthat/parallel: true</code></pre> <p>You’ll also need to be using the 3rd edition:</p> <pre><code>Config/testthat/edition: 3</code></pre> <div id="basic-operation" class="section level2"> <h2>Basic operation</h2> <p>Starting a new R process is relatively expensive, so testthat begins by creating a pool of workers. The size of the pool will be determined by <code>getOption("Ncpus")</code>, then the <code>TESTTHAT_CPUS</code> envvar. If neither are set, then two processes are started. In any case, testthat will never start more subprocesses than test files.</p> <p>Each worker begins by loading testthat and the package being tested. It then runs any setup files (so if you have existing setup files you’ll need to make sure they work when executed in parallel).</p> <p>testthat runs test <em>files</em> in parallel. Once the worker pool is initialized, testthat then starts sending test files to workers, by default in alphabetical order: as soon as a subprocess has finished, it receives another file, until all files are done. This means that state is persisted across test files: options are <em>not</em> reset, loaded packages are <em>not</em> unloaded, the global environment is <em>not</em> cleared, etc. You are responsible for making sure each file leaves the world as it finds it.</p> <p>Because files are run in alphabetical order, you may want to rename your slowest test files so that they start first, e.g. <code>test-1-slowest.R</code>, <code>test-2-next-slowest.R</code>, etc.</p> </div> <div id="common-problems" class="section level2"> <h2>Common problems</h2> <ul> <li><p>If tests fail stochastically (i.e. they sometimes work and sometimes fail) you may have accidentally introduced a dependency between your test files. This sort of dependency is hard to track down due to the random nature, and you’ll need to check all tests to make sure that they’re not accidentally changing global state.</p></li> <li><p>If you use <a href="https://testthat.r-lib.org/articles/test-fixtures.html#package">packaged scope test fixtures</a>, you’ll need to review them to make sure that they work in parallel. For example, if you were previously creating a temporary database in the test directory, you’d need to instead create it in the session temporary directory so that each process gets its own independent version.</p></li> </ul> </div> <div id="performance" class="section level2"> <h2>Performance</h2> <p>There is some overhead associated with running tests in parallel:</p> <ul> <li><p>Startup cost is linear in the number of subprocesses, because we need to create them in a loop. This is about 50ms on my laptop. Each subprocess needs to load testthat and the tested package, this happens in parallel, and we cannot do too much about it.</p></li> <li><p>Clean up time is again linear in the number of subprocesses, and it about 80ms per subprocess on my laptop.</p></li> <li><p>It seems that sending a message (i.e. a passing or failing expectation) is about 2ms currently. This is the total cost that includes sending the message, receiving it, and replying it to a non-parallel reporter.</p></li> </ul> <p>This overhead generally means that if you have many test files that take a short amount of time, you’re unlikely to see a huge benefit by using parallel tests. For example, testthat itself takes about 10s to run tests in serial, and 8s to run the tests in parallel.</p> <div id="changing-the-order-of-the-test-files" class="section level3"> <h3>Changing the order of the test files</h3> <p>By default testthat starts the test files in alphabetical order. If you have a few number of test files that take longer than the rest, then this might not be the best order. Ideally the slow files would start first, as the whole test suite will take at least as much time as its slowest test file. You can change the order with the <code>Config/testthat/start-first</code> option in <code>DESCRIPTION</code>. For example testthat currently has:</p> <pre><code>Config/testthat/start-first: watcher, parallel*</code></pre> <p>The format is a comma separated list of glob patterns, see <code>?utils::glob2rx</code>. The matching test files will start first. (The <code>test-</code> prefix is ignored.)</p> </div> </div> <div id="reporters" class="section level2"> <h2>Reporters</h2> <div id="default-reporters" class="section level3"> <h3>Default reporters</h3> <p>See <code>default_reporter()</code> for how testthat selects the default reporter for <code>devtools::test()</code> and <code>testthat::test_local()</code>. In short, testthat selects <code>ProgressReporter</code> for non-parallel and <code>ParallelProgressReporter</code> for parallel tests by default. (Other testthat test functions, like <code>test_check()</code>, <code>test_file()</code> , etc. select different reporters by default.)</p> </div> <div id="parallel-support" class="section level3"> <h3>Parallel support</h3> <p>Most reporters support parallel tests. If a reporter is passed to <code>devtools::test()</code>, <code>testthat::test_dir()</code>, etc. directly, and it does not support parallel tests, then testthat runs the test files sequentially.</p> <p>Currently the following reporters <em>don’t</em> support parallel tests:</p> <ul> <li><p><code>DebugReporter</code>, because it is not currently possible to debug subprocesses.</p></li> <li><p><code>JunitReporter</code>, because this reporter records timing information for each test block, and this is currently only available for reporters that support multiple active test files. (See “Writing parallel reporters” below.)</p></li> <li><p><code>LocationReporter</code> because testthat currently does not include location information for successful tests when running in parallel, to minimize messaging between the processes.</p></li> <li><p><code>StopReporter</code>, as this is a reporter that testthat uses for interactive <code>expect_that()</code> calls.</p></li> </ul> <p>The other built-in reporters all support parallel tests, with some subtle differences:</p> <ul> <li><p>Reporters that stop after a certain number of failures can only stop at the end of a test file.</p></li> <li><p>Reporters report all information about a file at once, unless they support <em>parallel updates</em>. E.g. <code>ProgressReporter</code> does not update its display until a test file is complete.</p></li> <li><p>The standard output and standard error, i.e. <code>print()</code>, <code>cat()</code>, <code>message()</code>, etc. output from the test files are lost currently. If you want to use <code>cat()</code> or <code>message()</code> for print-debugging test cases, then the best is to temporarily run tests sequentially, by changing the <code>Config</code> entry in <code>DESCRIPTION</code> or selecting a non-parallel reporter, e.g. the <code>CheckReporter</code>:</p> <div class="sourceCode" id="cb4"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb4-1"><a href="#cb4-1" aria-hidden="true" tabindex="-1"></a>devtools<span class="sc">::</span><span class="fu">test</span>(<span class="at">filter =</span> <span class="st">"badtest"</span>, <span class="at">reporter =</span> <span class="st">"check"</span>)</span></code></pre></div></li> </ul> </div> <div id="writing-parallel-reporters" class="section level3"> <h3>Writing parallel reporters</h3> <p>To support parallel tests, a reporter must be able to function when the test files run in a subprocess. For example <code>DebugReporter</code> does not support parallel tests, because it requires direct interaction with the frames in the subprocess. When running in parallel, testthat does not provide location information (source references) for test successes.</p> <p>To support parallel tests, a reporter must set <code>self$capabilities$parallel_support</code> to <code>TRUE</code> in its <code>initialize()</code> method:</p> <div class="sourceCode" id="cb5"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb5-1"><a href="#cb5-1" aria-hidden="true" tabindex="-1"></a>...</span> <span id="cb5-2"><a href="#cb5-2" aria-hidden="true" tabindex="-1"></a>initialize <span class="ot">=</span> <span class="cf">function</span>(...) {</span> <span id="cb5-3"><a href="#cb5-3" aria-hidden="true" tabindex="-1"></a> super<span class="sc">$</span><span class="fu">initialize</span>(...)</span> <span id="cb5-4"><a href="#cb5-4" aria-hidden="true" tabindex="-1"></a> self<span class="sc">$</span>capabilities<span class="sc">$</span>parallel_support <span class="ot"><-</span> <span class="cn">TRUE</span></span> <span id="cb5-5"><a href="#cb5-5" aria-hidden="true" tabindex="-1"></a> ...</span> <span id="cb5-6"><a href="#cb5-6" aria-hidden="true" tabindex="-1"></a>}</span> <span id="cb5-7"><a href="#cb5-7" aria-hidden="true" tabindex="-1"></a>...</span></code></pre></div> <p>When running in parallel, testthat runs the reporter in the main process, and relays information between the reporter and the test code transparently. (Currently the reporter does not even know that the tests are running in parallel.)</p> <p>If a reporter does not support parallel updates (see below), then testthat internally caches all calls to the reporter methods from subprocesses, until a test file is complete. This is because these reporters are not prepared for running multiple test files concurrently. Once a test file is complete, testthat calls the reporter’s <code>$start_file()</code> method, relays all <code>$start_test()</code> , <code>$end_test()</code>, <code>$add_result()</code>, etc. calls in the order they came in from the subprocess, and calls <code>$end_file()</code> .</p> </div> <div id="parallel-updates" class="section level3"> <h3>Parallel updates</h3> <p>The <code>ParallelProgressReporter</code> supports parallel updates. This means that once a message from a subprocess comes in, the reporter is updated immediately. For this to work, a reporter must be able to handle multiple test files concurrently.</p> <p>A reporter declares parallel update support by setting <code>self$capabilities$parallel_updates</code> to <code>TRUE</code>:</p> <div class="sourceCode" id="cb6"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb6-1"><a href="#cb6-1" aria-hidden="true" tabindex="-1"></a>...</span> <span id="cb6-2"><a href="#cb6-2" aria-hidden="true" tabindex="-1"></a>initialize <span class="ot">=</span> <span class="cf">function</span>(...) {</span> <span id="cb6-3"><a href="#cb6-3" aria-hidden="true" tabindex="-1"></a> super<span class="sc">$</span><span class="fu">initialize</span>(...)</span> <span id="cb6-4"><a href="#cb6-4" aria-hidden="true" tabindex="-1"></a> self<span class="sc">$</span>capabilities<span class="sc">$</span>parallel_support <span class="ot"><-</span> <span class="cn">TRUE</span></span> <span id="cb6-5"><a href="#cb6-5" aria-hidden="true" tabindex="-1"></a> self<span class="sc">$</span>capabilities<span class="sc">$</span>parallel_updates <span class="ot"><-</span> <span class="cn">TRUE</span></span> <span id="cb6-6"><a href="#cb6-6" aria-hidden="true" tabindex="-1"></a> ...</span> <span id="cb6-7"><a href="#cb6-7" aria-hidden="true" tabindex="-1"></a>}</span> <span id="cb6-8"><a href="#cb6-8" aria-hidden="true" tabindex="-1"></a>...</span></code></pre></div> <p>For these reporters, testthat does not cache the messages from the subprocesses. Instead, when a message comes in:</p> <ul> <li><p>It calls the <code>$start_file()</code> method, letting the reporter know which file the following calls apply to. This means that the reporter can receive multiple <code>$start_file()</code> calls for the same file.</p></li> <li><p>Then relays the message from the subprocess, calling the appropriate <code>$start_test()</code> , <code>$add_result()</code>, etc. method.</p></li> </ul> <p>testthat also calls the new <code>$update()</code> method of the reporter regularly, even if it does not receive any messages from the subprocess. (Currently aims to do this every 100ms, but there are no guarantees.) The <code>$update()</code> method may implement a spinner to let the user know that the tests are running.</p> </div> </div> <!-- code folding --> <!-- dynamically load mathjax for compatibility with self-contained --> <script> (function () { var script = document.createElement("script"); script.type = "text/javascript"; script.src = "https://mathjax.rstudio.com/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML"; document.getElementsByTagName("head")[0].appendChild(script); })(); </script> </body> </html>