easylink 0.1.20__tar.gz → 0.1.21__tar.gz

{easylink-0.1.20 → easylink-0.1.21}/.gitignore

@@ -147,5 +147,5 @@ src/easylink/_version.py
 
 # Local specification files
 .specifications/
-
 .snakemake/
+*/copilot-instructions.md

{easylink-0.1.20 → easylink-0.1.21}/CHANGELOG.rst

@@ -1,3 +1,8 @@
+**0.1.21 - 6/16/25**
+
+ - Update the "parallel" pipeline configuration key to "clones"
+ - Update "ParallelStep" and "EmbarrassinglyParallelStep" class names to "CloneableStep" and "AutoParallelStep", respectively
+
 **0.1.20 - 6/11/25**
 
  - Update implementation creator tool for new implementation_metadata.yaml format

{easylink-0.1.20 → easylink-0.1.21}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: easylink
-Version: 0.1.20
+Version: 0.1.21
 Summary: Research repository for the EasyLink ER ecosystem project.
 Home-page: https://github.com/ihmeuw/easylink
 Author: The EasyLink developers

{easylink-0.1.20 → easylink-0.1.21}/docs/source/concepts/workarounds.rst

@@ -74,7 +74,7 @@ The ``pipeline.yaml`` section for the cloneable section diagrammed above might l
    :emphasize-lines: 7,12,17,22
 
    determining_exclusions_and_removing_records:
-     parallel:
+     clones:
        - determining_exclusions:
            implementation:
              name: default_determining_exclusions
@@ -103,7 +103,7 @@ both steps in the above diagram received the output of the splitter.
 
 It is very important that the user:
 
-* Create a number of clones (items in the list under ``parallel``) that matches the number of files
+* Create a number of clones (items in the list under ``clones``) that matches the number of files
 * Pass each clone the name of one of the files
 * Pass each sub-step *within* each clone the same file name
 

easylink-0.1.21/docs/source/user_guide/tutorials/DAG-e2e-pipeline-expanded.svg

@@ -0,0 +1,283 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
+ "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+<!-- Generated by graphviz version 12.2.1 (0)
+ -->
+<!-- Title: snakemake_dag Pages: 1 -->
+<svg width="1106pt" height="980pt"
+ viewBox="0.00 0.00 1106.25 980.00" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 976)">
+<title>snakemake_dag</title>
+<polygon fill="white" stroke="none" points="-4,4 -4,-976 1102.25,-976 1102.25,4 -4,4"/>
+<!-- 0 -->
+<g id="node1" class="node">
+<title>0</title>
+<path fill="none" stroke="#d6d856" stroke-width="2" d="M630.12,-36C630.12,-36 600.12,-36 600.12,-36 594.12,-36 588.12,-30 588.12,-24 588.12,-24 588.12,-12 588.12,-12 588.12,-6 594.12,0 600.12,0 600.12,0 630.12,0 630.12,0 636.12,0 642.12,-6 642.12,-12 642.12,-12 642.12,-24 642.12,-24 642.12,-30 636.12,-36 630.12,-36"/>
+<text text-anchor="middle" x="615.12" y="-14.88" font-family="sans" font-size="10.00">all</text>
+</g>
+<!-- 1 -->
+<g id="node2" class="node">
+<title>1</title>
+<path fill="none" stroke="#56d8d8" stroke-width="2" d="M670,-180C670,-180 560.25,-180 560.25,-180 554.25,-180 548.25,-174 548.25,-168 548.25,-168 548.25,-156 548.25,-156 548.25,-150 554.25,-144 560.25,-144 560.25,-144 670,-144 670,-144 676,-144 682,-150 682,-156 682,-156 682,-168 682,-168 682,-174 676,-180 670,-180"/>
+<text text-anchor="middle" x="615.12" y="-158.88" font-family="sans" font-size="10.00">step_4b_python_pandas</text>
+</g>
+<!-- 1&#45;&gt;0 -->
+<g id="edge1" class="edge">
+<title>1&#45;&gt;0</title>
+<path fill="none" stroke="grey" stroke-width="2" d="M603.38,-143.19C597.54,-133.28 591.11,-120.45 588.12,-108 584.39,-92.44 584.39,-87.56 588.12,-72 590.06,-63.93 593.45,-55.7 597.18,-48.2"/>
+<polygon fill="grey" stroke="grey" stroke-width="2" points="600.22,-49.93 601.93,-39.47 594.08,-46.58 600.22,-49.93"/>
+</g>
+<!-- 19 -->
+<g id="node20" class="node">
+<title>19</title>
+<path fill="none" stroke="#d8ac56" stroke-width="2" d="M715.5,-108C715.5,-108 608.75,-108 608.75,-108 602.75,-108 596.75,-102 596.75,-96 596.75,-96 596.75,-84 596.75,-84 596.75,-78 602.75,-72 608.75,-72 608.75,-72 715.5,-72 715.5,-72 721.5,-72 727.5,-78 727.5,-84 727.5,-84 727.5,-96 727.5,-96 727.5,-102 721.5,-108 715.5,-108"/>
+<text text-anchor="middle" x="662.12" y="-86.88" font-family="sans" font-size="10.00">results_result_validator</text>
+</g>
+<!-- 1&#45;&gt;19 -->
+<g id="edge25" class="edge">
+<title>1&#45;&gt;19</title>
+<path fill="none" stroke="grey" stroke-width="2" d="M626.99,-143.34C631.85,-136.1 637.6,-127.53 643.02,-119.45"/>
+<polygon fill="grey" stroke="grey" stroke-width="2" points="645.75,-121.67 648.42,-111.41 639.94,-117.77 645.75,-121.67"/>
+</g>
+<!-- 2 -->
+<g id="node3" class="node">
+<title>2</title>
+<path fill="none" stroke="#569ad8" stroke-width="2" d="M603,-324C603,-324 493.25,-324 493.25,-324 487.25,-324 481.25,-318 481.25,-312 481.25,-312 481.25,-300 481.25,-300 481.25,-294 487.25,-288 493.25,-288 493.25,-288 603,-288 603,-288 609,-288 615,-294 615,-300 615,-300 615,-312 615,-312 615,-318 609,-324 603,-324"/>
+<text text-anchor="middle" x="548.12" y="-302.88" font-family="sans" font-size="10.00">step_4a_python_pandas</text>
+</g>
+<!-- 2&#45;&gt;1 -->
+<g id="edge3" class="edge">
+<title>2&#45;&gt;1</title>
+<path fill="none" stroke="grey" stroke-width="2" d="M615.72,-298.8C676.7,-291.64 759.84,-277.44 780.12,-252 809.98,-214.56 751.02,-190.76 695.47,-177.28"/>
+<polygon fill="grey" stroke="grey" stroke-width="2" points="696.45,-173.92 685.92,-175.08 694.88,-180.74 696.45,-173.92"/>
+</g>
+<!-- 18 -->
+<g id="node19" class="node">
+<title>18</title>
+<path fill="none" stroke="#d88556" stroke-width="2" d="M429.12,-252C429.12,-252 167.12,-252 167.12,-252 161.12,-252 155.12,-246 155.12,-240 155.12,-240 155.12,-228 155.12,-228 155.12,-222 161.12,-216 167.12,-216 167.12,-216 429.12,-216 429.12,-216 435.12,-216 441.12,-222 441.12,-228 441.12,-228 441.12,-240 441.12,-240 441.12,-246 435.12,-252 429.12,-252"/>
+<text text-anchor="middle" x="298.12" y="-230.88" font-family="sans" font-size="10.00">step_4b_python_pandas_step_4b_main_input_validator</text>
+</g>
+<!-- 2&#45;&gt;18 -->
+<g id="edge24" class="edge">
+<title>2&#45;&gt;18</title>
+<path fill="none" stroke="grey" stroke-width="2" d="M484.07,-287.06C450.55,-277.68 409.29,-266.13 374.16,-256.29"/>
+<polygon fill="grey" stroke="grey" stroke-width="2" points="375.5,-253.03 364.93,-253.71 373.62,-259.77 375.5,-253.03"/>
+</g>
+<!-- 3 -->
+<g id="node4" class="node">
+<title>3</title>
+<path fill="none" stroke="#d86e56" stroke-width="2" d="M591.5,-468C591.5,-468 370.75,-468 370.75,-468 364.75,-468 358.75,-462 358.75,-456 358.75,-456 358.75,-444 358.75,-444 358.75,-438 364.75,-432 370.75,-432 370.75,-432 591.5,-432 591.5,-432 597.5,-432 603.5,-438 603.5,-444 603.5,-444 603.5,-456 603.5,-456 603.5,-462 597.5,-468 591.5,-468"/>
+<text text-anchor="middle" x="481.12" y="-446.88" font-family="sans" font-size="10.00">step_3_loop_2_aggregate_step_3_main_output</text>
+</g>
+<!-- 3&#45;&gt;2 -->
+<g id="edge6" class="edge">
+<title>3&#45;&gt;2</title>
+<path fill="none" stroke="grey" stroke-width="2" d="M604.44,-434.84C652.08,-426.61 698.64,-414.17 713.12,-396 742.98,-358.56 684.02,-334.76 628.47,-321.28"/>
+<polygon fill="grey" stroke="grey" stroke-width="2" points="629.45,-317.92 618.92,-319.08 627.88,-324.74 629.45,-317.92"/>
+</g>
+<!-- 16 -->
+<g id="node17" class="node">
+<title>16</title>
+<path fill="none" stroke="#56d882" stroke-width="2" d="M362.12,-396C362.12,-396 100.12,-396 100.12,-396 94.12,-396 88.12,-390 88.12,-384 88.12,-384 88.12,-372 88.12,-372 88.12,-366 94.12,-360 100.12,-360 100.12,-360 362.12,-360 362.12,-360 368.12,-360 374.12,-366 374.12,-372 374.12,-372 374.12,-384 374.12,-384 374.12,-390 368.12,-396 362.12,-396"/>
+<text text-anchor="middle" x="231.12" y="-374.88" font-family="sans" font-size="10.00">step_4a_python_pandas_step_4a_main_input_validator</text>
+</g>
+<!-- 3&#45;&gt;16 -->
+<g id="edge23" class="edge">
+<title>3&#45;&gt;16</title>
+<path fill="none" stroke="grey" stroke-width="2" d="M417.07,-431.06C383.55,-421.68 342.29,-410.13 307.16,-400.29"/>
+<polygon fill="grey" stroke="grey" stroke-width="2" points="308.5,-397.03 297.93,-397.71 306.62,-403.77 308.5,-397.03"/>
+</g>
+<!-- 4 -->
+<g id="node5" class="node">
+<title>4</title>
+<path fill="none" stroke="#97d856" stroke-width="2" d="M573.88,-540C573.88,-540 388.38,-540 388.38,-540 382.38,-540 376.38,-534 376.38,-528 376.38,-528 376.38,-516 376.38,-516 376.38,-510 382.38,-504 388.38,-504 388.38,-504 573.88,-504 573.88,-504 579.88,-504 585.88,-510 585.88,-516 585.88,-516 585.88,-528 585.88,-528 585.88,-534 579.88,-540 573.88,-540"/>
+<text text-anchor="middle" x="481.12" y="-518.88" font-family="sans" font-size="10.00">step_3_loop_2_step_3_main_input_split</text>
+</g>
+<!-- 4&#45;&gt;3 -->
+<g id="edge9" class="edge">
+<title>4&#45;&gt;3</title>
+<path fill="none" stroke="grey" stroke-width="2" d="M481.12,-503.34C481.12,-496.75 481.12,-489.08 481.12,-481.67"/>
+<polygon fill="grey" stroke="grey" stroke-width="2" points="484.63,-481.93 481.13,-471.93 477.63,-481.93 484.63,-481.93"/>
+</g>
+<!-- 5 -->
+<g id="node6" class="node">
+<title>5</title>
+<path fill="none" stroke="#5682d8" stroke-width="2" d="M591.5,-612C591.5,-612 370.75,-612 370.75,-612 364.75,-612 358.75,-606 358.75,-600 358.75,-600 358.75,-588 358.75,-588 358.75,-582 364.75,-576 370.75,-576 370.75,-576 591.5,-576 591.5,-576 597.5,-576 603.5,-582 603.5,-588 603.5,-588 603.5,-600 603.5,-600 603.5,-606 597.5,-612 591.5,-612"/>
+<text text-anchor="middle" x="481.12" y="-590.88" font-family="sans" font-size="10.00">step_3_loop_1_aggregate_step_3_main_output</text>
+</g>
+<!-- 5&#45;&gt;4 -->
+<g id="edge10" class="edge">
+<title>5&#45;&gt;4</title>
+<path fill="none" stroke="grey" stroke-width="2" d="M481.12,-575.34C481.12,-568.75 481.12,-561.08 481.12,-553.67"/>
+<polygon fill="grey" stroke="grey" stroke-width="2" points="484.63,-553.93 481.13,-543.93 477.63,-553.93 484.63,-553.93"/>
+</g>
+<!-- 6 -->
+<g id="node7" class="node">
+<title>6</title>
+<path fill="none" stroke="#56a9d8" stroke-width="2" d="M573.88,-684C573.88,-684 388.38,-684 388.38,-684 382.38,-684 376.38,-678 376.38,-672 376.38,-672 376.38,-660 376.38,-660 376.38,-654 382.38,-648 388.38,-648 388.38,-648 573.88,-648 573.88,-648 579.88,-648 585.88,-654 585.88,-660 585.88,-660 585.88,-672 585.88,-672 585.88,-678 579.88,-684 573.88,-684"/>
+<text text-anchor="middle" x="481.12" y="-662.88" font-family="sans" font-size="10.00">step_3_loop_1_step_3_main_input_split</text>
+</g>
+<!-- 6&#45;&gt;5 -->
+<g id="edge11" class="edge">
+<title>6&#45;&gt;5</title>
+<path fill="none" stroke="grey" stroke-width="2" d="M481.12,-647.34C481.12,-640.75 481.12,-633.08 481.12,-625.67"/>
+<polygon fill="grey" stroke="grey" stroke-width="2" points="484.63,-625.93 481.13,-615.93 477.63,-625.93 484.63,-625.93"/>
+</g>
+<!-- 7 -->
+<g id="node8" class="node">
+<title>7</title>
+<path fill="none" stroke="#d89556" stroke-width="2" d="M533,-756C533,-756 429.25,-756 429.25,-756 423.25,-756 417.25,-750 417.25,-744 417.25,-744 417.25,-732 417.25,-732 417.25,-726 423.25,-720 429.25,-720 429.25,-720 533,-720 533,-720 539,-720 545,-726 545,-732 545,-732 545,-744 545,-744 545,-750 539,-756 533,-756"/>
+<text text-anchor="middle" x="481.12" y="-734.88" font-family="sans" font-size="10.00">step_2_python_pandas</text>
+</g>
+<!-- 7&#45;&gt;6 -->
+<g id="edge12" class="edge">
+<title>7&#45;&gt;6</title>
+<path fill="none" stroke="grey" stroke-width="2" d="M481.12,-719.34C481.12,-712.75 481.12,-705.08 481.12,-697.67"/>
+<polygon fill="grey" stroke="grey" stroke-width="2" points="484.63,-697.93 481.13,-687.93 477.63,-697.93 484.63,-697.93"/>
+</g>
+<!-- 8 -->
+<g id="node9" class="node">
+<title>8</title>
+<path fill="none" stroke="#68d856" stroke-width="2" d="M415.12,-900C415.12,-900 231.12,-900 231.12,-900 225.12,-900 219.12,-894 219.12,-888 219.12,-888 219.12,-876 219.12,-876 219.12,-870 225.12,-864 231.12,-864 231.12,-864 415.12,-864 415.12,-864 421.12,-864 427.12,-870 427.12,-876 427.12,-876 427.12,-888 427.12,-888 427.12,-894 421.12,-900 415.12,-900"/>
+<text text-anchor="middle" x="323.12" y="-878.88" font-family="sans" font-size="10.00">step_1_clone_1_step_1_python_pandas</text>
+</g>
+<!-- 8&#45;&gt;7 -->
+<g id="edge13" class="edge">
+<title>8&#45;&gt;7</title>
+<path fill="none" stroke="grey" stroke-width="2" d="M338.55,-863.03C354.95,-844.37 382.06,-814.78 408.12,-792 419.23,-782.3 432.04,-772.56 443.81,-764.13"/>
+<polygon fill="grey" stroke="grey" stroke-width="2" points="445.59,-767.15 451.76,-758.53 441.57,-761.42 445.59,-767.15"/>
+</g>
+<!-- 14 -->
+<g id="node15" class="node">
+<title>14</title>
+<path fill="none" stroke="#c6d856" stroke-width="2" d="M689.12,-828C689.12,-828 439.12,-828 439.12,-828 433.12,-828 427.12,-822 427.12,-816 427.12,-816 427.12,-804 427.12,-804 427.12,-798 433.12,-792 439.12,-792 439.12,-792 689.12,-792 689.12,-792 695.12,-792 701.12,-798 701.12,-804 701.12,-804 701.12,-816 701.12,-816 701.12,-822 695.12,-828 689.12,-828"/>
+<text text-anchor="middle" x="564.12" y="-806.88" font-family="sans" font-size="10.00">step_2_python_pandas_step_2_main_input_validator</text>
+</g>
+<!-- 8&#45;&gt;14 -->
+<g id="edge20" class="edge">
+<title>8&#45;&gt;14</title>
+<path fill="none" stroke="grey" stroke-width="2" d="M384.88,-863.06C417.05,-853.72 456.61,-842.23 490.38,-832.42"/>
+<polygon fill="grey" stroke="grey" stroke-width="2" points="491,-835.88 499.63,-829.73 489.05,-829.16 491,-835.88"/>
+</g>
+<!-- 9 -->
+<g id="node10" class="node">
+<title>9</title>
+<path fill="none" stroke="#80d856" stroke-width="2" d="M342.25,-972C342.25,-972 12,-972 12,-972 6,-972 0,-966 0,-960 0,-960 0,-948 0,-948 0,-942 6,-936 12,-936 12,-936 342.25,-936 342.25,-936 348.25,-936 354.25,-942 354.25,-948 354.25,-948 354.25,-960 354.25,-960 354.25,-966 348.25,-972 342.25,-972"/>
+<text text-anchor="middle" x="177.12" y="-950.88" font-family="sans" font-size="10.00">step_1_clone_1_step_1_python_pandas_step_1_main_input_validator</text>
+</g>
+<!-- 9&#45;&gt;8 -->
+<g id="edge17" class="edge">
+<title>9&#45;&gt;8</title>
+<path fill="none" stroke="grey" stroke-width="2" d="M214.34,-935.15C232.47,-926.47 254.49,-915.91 273.98,-906.56"/>
+<polygon fill="grey" stroke="grey" stroke-width="2" points="275.49,-909.72 282.99,-902.24 272.46,-903.41 275.49,-909.72"/>
+</g>
+<!-- 10 -->
+<g id="node11" class="node">
+<title>10</title>
+<path fill="none" stroke="#afd856" stroke-width="2" d="M641.12,-900C641.12,-900 457.12,-900 457.12,-900 451.12,-900 445.12,-894 445.12,-888 445.12,-888 445.12,-876 445.12,-876 445.12,-870 451.12,-864 457.12,-864 457.12,-864 641.12,-864 641.12,-864 647.12,-864 653.12,-870 653.12,-876 653.12,-876 653.12,-888 653.12,-888 653.12,-894 647.12,-900 641.12,-900"/>
+<text text-anchor="middle" x="549.12" y="-878.88" font-family="sans" font-size="10.00">step_1_clone_2_step_1_python_pandas</text>
+</g>
+<!-- 10&#45;&gt;7 -->
+<g id="edge14" class="edge">
+<title>10&#45;&gt;7</title>
+<path fill="none" stroke="grey" stroke-width="2" d="M463.39,-863.07C445.58,-855.27 428.96,-844.07 418.12,-828 409.18,-814.73 411.37,-806.51 418.12,-792 422.99,-781.55 431.04,-772.42 439.79,-764.82"/>
+<polygon fill="grey" stroke="grey" stroke-width="2" points="441.77,-767.72 447.45,-758.78 437.44,-762.23 441.77,-767.72"/>
+</g>
+<!-- 10&#45;&gt;14 -->
+<g id="edge21" class="edge">
+<title>10&#45;&gt;14</title>
+<path fill="none" stroke="grey" stroke-width="2" d="M552.91,-863.34C554.34,-856.67 556.01,-848.89 557.61,-841.39"/>
+<polygon fill="grey" stroke="grey" stroke-width="2" points="560.98,-842.37 559.65,-831.86 554.14,-840.91 560.98,-842.37"/>
+</g>
+<!-- 11 -->
+<g id="node12" class="node">
+<title>11</title>
+<path fill="none" stroke="#56c1d8" stroke-width="2" d="M714.25,-972C714.25,-972 384,-972 384,-972 378,-972 372,-966 372,-960 372,-960 372,-948 372,-948 372,-942 378,-936 384,-936 384,-936 714.25,-936 714.25,-936 720.25,-936 726.25,-942 726.25,-948 726.25,-948 726.25,-960 726.25,-960 726.25,-966 720.25,-972 714.25,-972"/>
+<text text-anchor="middle" x="549.12" y="-950.88" font-family="sans" font-size="10.00">step_1_clone_2_step_1_python_pandas_step_1_main_input_validator</text>
+</g>
+<!-- 11&#45;&gt;10 -->
+<g id="edge18" class="edge">
+<title>11&#45;&gt;10</title>
+<path fill="none" stroke="grey" stroke-width="2" d="M549.12,-935.34C549.12,-928.75 549.12,-921.08 549.12,-913.67"/>
+<polygon fill="grey" stroke="grey" stroke-width="2" points="552.63,-913.93 549.13,-903.93 545.63,-913.93 552.63,-913.93"/>
+</g>
+<!-- 12 -->
+<g id="node13" class="node">
+<title>12</title>
+<path fill="none" stroke="#566bd8" stroke-width="2" d="M867.12,-900C867.12,-900 683.12,-900 683.12,-900 677.12,-900 671.12,-894 671.12,-888 671.12,-888 671.12,-876 671.12,-876 671.12,-870 677.12,-864 683.12,-864 683.12,-864 867.12,-864 867.12,-864 873.12,-864 879.12,-870 879.12,-876 879.12,-876 879.12,-888 879.12,-888 879.12,-894 873.12,-900 867.12,-900"/>
+<text text-anchor="middle" x="775.12" y="-878.88" font-family="sans" font-size="10.00">step_1_clone_3_step_1_python_pandas</text>
+</g>
+<!-- 12&#45;&gt;7 -->
+<g id="edge15" class="edge">
+<title>12&#45;&gt;7</title>
+<path fill="none" stroke="grey" stroke-width="2" d="M767.05,-863.32C756.79,-842.9 737.11,-809.82 710.12,-792 665.23,-762.35 605.83,-749.18 558.78,-743.38"/>
+<polygon fill="grey" stroke="grey" stroke-width="2" points="559.19,-739.91 548.87,-742.27 558.42,-746.86 559.19,-739.91"/>
+</g>
+<!-- 12&#45;&gt;14 -->
+<g id="edge22" class="edge">
+<title>12&#45;&gt;14</title>
+<path fill="none" stroke="grey" stroke-width="2" d="M721.06,-863.06C693.39,-853.88 659.47,-842.63 630.25,-832.94"/>
+<polygon fill="grey" stroke="grey" stroke-width="2" points="631.51,-829.67 620.92,-829.84 629.31,-836.31 631.51,-829.67"/>
+</g>
+<!-- 13 -->
+<g id="node14" class="node">
+<title>13</title>
+<path fill="none" stroke="#d85656" stroke-width="2" d="M1086.25,-972C1086.25,-972 756,-972 756,-972 750,-972 744,-966 744,-960 744,-960 744,-948 744,-948 744,-942 750,-936 756,-936 756,-936 1086.25,-936 1086.25,-936 1092.25,-936 1098.25,-942 1098.25,-948 1098.25,-948 1098.25,-960 1098.25,-960 1098.25,-966 1092.25,-972 1086.25,-972"/>
+<text text-anchor="middle" x="921.12" y="-950.88" font-family="sans" font-size="10.00">step_1_clone_3_step_1_python_pandas_step_1_main_input_validator</text>
+</g>
+<!-- 13&#45;&gt;12 -->
+<g id="edge19" class="edge">
+<title>13&#45;&gt;12</title>
+<path fill="none" stroke="grey" stroke-width="2" d="M883.91,-935.15C865.78,-926.47 843.76,-915.91 824.27,-906.56"/>
+<polygon fill="grey" stroke="grey" stroke-width="2" points="825.79,-903.41 815.26,-902.24 822.76,-909.72 825.79,-903.41"/>
+</g>
+<!-- 14&#45;&gt;7 -->
+<g id="edge16" class="edge">
+<title>14&#45;&gt;7</title>
+<path fill="none" stroke="grey" stroke-width="2" d="M543.18,-791.34C533.82,-783.44 522.58,-773.96 512.26,-765.26"/>
+<polygon fill="grey" stroke="grey" stroke-width="2" points="514.57,-762.62 504.67,-758.85 510.05,-767.98 514.57,-762.62"/>
+</g>
+<!-- 15 -->
+<g id="node16" class="node">
+<title>15</title>
+<path fill="none" stroke="#59d856" stroke-width="2" d="M691.88,-396C691.88,-396 404.38,-396 404.38,-396 398.38,-396 392.38,-390 392.38,-384 392.38,-384 392.38,-372 392.38,-372 392.38,-366 398.38,-360 404.38,-360 404.38,-360 691.88,-360 691.88,-360 697.88,-360 703.88,-366 703.88,-372 703.88,-372 703.88,-384 703.88,-384 703.88,-390 697.88,-396 691.88,-396"/>
+<text text-anchor="middle" x="548.12" y="-374.88" font-family="sans" font-size="10.00">step_4a_python_pandas_step_4a_secondary_input_validator</text>
+</g>
+<!-- 15&#45;&gt;2 -->
+<g id="edge7" class="edge">
+<title>15&#45;&gt;2</title>
+<path fill="none" stroke="grey" stroke-width="2" d="M548.12,-359.34C548.12,-352.75 548.12,-345.08 548.12,-337.67"/>
+<polygon fill="grey" stroke="grey" stroke-width="2" points="551.63,-337.93 548.13,-327.93 544.63,-337.93 551.63,-337.93"/>
+</g>
+<!-- 16&#45;&gt;2 -->
+<g id="edge8" class="edge">
+<title>16&#45;&gt;2</title>
+<path fill="none" stroke="grey" stroke-width="2" d="M312.35,-359.06C360.17,-348.5 420.42,-335.2 468.05,-324.68"/>
+<polygon fill="grey" stroke="grey" stroke-width="2" points="468.53,-328.16 477.54,-322.59 467.02,-321.33 468.53,-328.16"/>
+</g>
+<!-- 17 -->
+<g id="node18" class="node">
+<title>17</title>
+<path fill="none" stroke="#56d86b" stroke-width="2" d="M758.88,-252C758.88,-252 471.38,-252 471.38,-252 465.38,-252 459.38,-246 459.38,-240 459.38,-240 459.38,-228 459.38,-228 459.38,-222 465.38,-216 471.38,-216 471.38,-216 758.88,-216 758.88,-216 764.88,-216 770.88,-222 770.88,-228 770.88,-228 770.88,-240 770.88,-240 770.88,-246 764.88,-252 758.88,-252"/>
+<text text-anchor="middle" x="615.12" y="-230.88" font-family="sans" font-size="10.00">step_4b_python_pandas_step_4b_secondary_input_validator</text>
+</g>
+<!-- 17&#45;&gt;1 -->
+<g id="edge4" class="edge">
+<title>17&#45;&gt;1</title>
+<path fill="none" stroke="grey" stroke-width="2" d="M615.12,-215.34C615.12,-208.75 615.12,-201.08 615.12,-193.67"/>
+<polygon fill="grey" stroke="grey" stroke-width="2" points="618.63,-193.93 615.13,-183.93 611.63,-193.93 618.63,-193.93"/>
+</g>
+<!-- 18&#45;&gt;1 -->
+<g id="edge5" class="edge">
+<title>18&#45;&gt;1</title>
+<path fill="none" stroke="grey" stroke-width="2" d="M379.35,-215.06C427.17,-204.5 487.42,-191.2 535.05,-180.68"/>
+<polygon fill="grey" stroke="grey" stroke-width="2" points="535.53,-184.16 544.54,-178.59 534.02,-177.33 535.53,-184.16"/>
+</g>
+<!-- 19&#45;&gt;0 -->
+<g id="edge2" class="edge">
+<title>19&#45;&gt;0</title>
+<path fill="none" stroke="grey" stroke-width="2" d="M650.26,-71.34C645.4,-64.1 639.65,-55.53 634.23,-47.45"/>
+<polygon fill="grey" stroke="grey" stroke-width="2" points="637.31,-45.77 628.83,-39.41 631.5,-49.67 637.31,-45.77"/>
+</g>
+</g>
+</svg>

{easylink-0.1.20 → easylink-0.1.21}/docs/source/user_guide/tutorials/getting_started.rst

@@ -139,11 +139,11 @@ configures the pipeline for this run, by specifying configuration details for ea
 defined by the pipeline schema. The schema steps, and the edges between them, are defined in 
 ``pipeline_schema_constants/development.py``. The schema steps, or nodes, define input and output slots for 
 data used or produced by the schema steps, as well as any logical or behavioral structure of the step,
-such as defining a step as a ``LoopStep``, ``ParallelStep``, ``ChoiceStep``, or ``HierarchicalStep``. The edges 
+such as defining a step as a ``LoopStep``, ``CloneableStep``, ``ChoiceStep``, or ``HierarchicalStep``. The edges 
 define how data moves between steps' input and output slots.
 
 ``pipeline_schema_constants/development.py`` defines that the pipeline schema requires four steps, that the 
-third step is ``EmbarrassinglyParallel``, that the fourth step is a ``ChoiceStep``, and that all steps have 
+third step is ``AutoParallel``, that the fourth step is a ``ChoiceStep``, and that all steps have 
 one input except the fourth step, which has two.
 The edges in the ``EDGES`` variable in that file connect the steps, so an output from one becomes an input
 to another.
@@ -196,15 +196,14 @@ except that ``step_4`` has two inputs, as defined in
 
 .. warning::
    Note that this diagram doesn't show the dependencies on original (user-provided) input data, and 
-   displays validations, and doesn't show Step 3 because it is embarrassingly parallel. See 
+   displays validations, and doesn't show Step 3 because it is an auto-parallel step. See 
    `this ticket <https://jira.ihme.washington.edu/browse/MIC-5767>`_ where we plan to fix these issues.
 
 Now we can understand why the final output has 60k rows. For the current dummy implementation, when there are multiple input data files, the rows 
 in the files are concatenated. So ``step_1`` concatenates three 10k row datasets, and ``step_4`` concatenates these 
 30k rows with another 30k rows.
 
-``step_3`` is aggregated and split because it is defined as 
-``EmbarrassinglyParallel``.
+``step_3`` is aggregated and split because it is defined as ``AutoParallel``.
 
 We've already viewed the final output, but if we want to see how the data is transformed over the course 
 of the pipeline, we can view intermediary outputs as well::

easylink-0.1.21/src/easylink/_version.py

@@ -0,0 +1 @@
+__version__ = "0.1.21"

{easylink-0.1.20 → easylink-0.1.21}/src/easylink/devtools/implementation_creator.py

@@ -21,8 +21,8 @@ from loguru import logger
 
 from easylink.pipeline_schema_constants import SCHEMA_PARAMS
 from easylink.step import (
+    AutoParallelStep,
     ChoiceStep,
-    EmbarrassinglyParallelStep,
     HierarchicalStep,
     IOStep,
     Step,
@@ -319,7 +319,7 @@ class ImplementationCreator:
             elif isinstance(node, TemplatedStep):
                 _process_step(node.template_step)
                 return
-            elif isinstance(node, EmbarrassinglyParallelStep):
+            elif isinstance(node, AutoParallelStep):
                 _process_step(node.step)
                 return
             elif isinstance(node, ChoiceStep):

{easylink-0.1.20 → easylink-0.1.21}/src/easylink/implementation.py

@@ -55,7 +55,7 @@ class Implementation:
         implementation_config: LayeredConfigTree,
         input_slots: Iterable[InputSlot] = (),
         output_slots: Iterable[OutputSlot] = (),
-        is_embarrassingly_parallel: bool = False,
+        is_auto_parallel: bool = False,
     ):
         self.name = implementation_config.name
         """The name of this ``Implementation``."""
@@ -74,7 +74,7 @@ class Implementation:
         implemented by this particular ``Implementation``."""
         self.requires_spark = self._metadata.get("requires_spark", False)
         """Whether this ``Implementation`` requires a Spark environment."""
-        self.is_embarrassingly_parallel = is_embarrassingly_parallel
+        self.is_auto_parallel = is_auto_parallel
 
     def __repr__(self) -> str:
         return f"Implementation.{self.name}"

{easylink-0.1.20 → easylink-0.1.21}/src/easylink/pipeline.py

@@ -45,9 +45,9 @@ class Pipeline:
         The :class:`~easylink.pipeline_graph.PipelineGraph` object.
     spark_is_required
         A boolean indicating whether the pipeline requires Spark.
-    any_embarrassingly_parallel
+    any_auto_parallel
         A boolean indicating whether any implementation in the pipeline is to be
-        run in an embarrassingly parallel manner.
+        automatically run in parallel.
 
     """
 
@@ -55,7 +55,7 @@ class Pipeline:
         self.config = config
         self.pipeline_graph = PipelineGraph(config)
         self.spark_is_required = self.pipeline_graph.spark_is_required
-        self.any_embarrassingly_parallel = self.pipeline_graph.any_embarrassingly_parallel
+        self.any_auto_parallel = self.pipeline_graph.any_auto_parallel
 
         # TODO [MIC-4880]: refactor into validation object
         self._validate()
@@ -179,7 +179,7 @@ class Pipeline:
     #################################
 
     def _write_imports(self) -> None:
-        if not self.any_embarrassingly_parallel:
+        if not self.any_auto_parallel:
             imports = "from easylink.utilities import validation_utils\n"
         else:
             imports = """import glob
@@ -193,7 +193,7 @@ from easylink.utilities import aggregator_utils, splitter_utils, validation_util
             f.write(imports)
 
     def _write_wildcard_constraints(self) -> None:
-        if self.any_embarrassingly_parallel:
+        if self.any_auto_parallel:
             with open(self.snakefile_path, "a") as f:
                 f.write(
                     """
@@ -301,12 +301,10 @@ use rule start_spark_worker from spark_cluster with:
             The name of the ``Implementation`` to write the rule(s) for.
         """
 
-        is_embarrassingly_parallel = self.pipeline_graph.get_whether_embarrassingly_parallel(
-            node_name
-        )
+        is_auto_parallel = self.pipeline_graph.get_whether_auto_parallel(node_name)
         input_slots, _output_slots = self.pipeline_graph.get_io_slot_attributes(node_name)
         validation_files, validation_rules = self._get_validations(
-            node_name, input_slots, is_embarrassingly_parallel
+            node_name, input_slots, is_auto_parallel
         )
         for validation_rule in validation_rules:
             validation_rule.write_to_snakefile(self.snakefile_path)
@@ -334,7 +332,7 @@ use rule start_spark_worker from spark_cluster with:
             image_path=self.config.images_dir / implementation.singularity_image_name,
             script_cmd=implementation.script_cmd,
             requires_spark=implementation.requires_spark,
-            is_embarrassingly_parallel=is_embarrassingly_parallel,
+            is_auto_parallel=is_auto_parallel,
         ).write_to_snakefile(self.snakefile_path)
 
     def _write_checkpoint_rule(self, node_name: str, checkpoint_filepath: str) -> None:
@@ -377,7 +375,7 @@ use rule start_spark_worker from spark_cluster with:
         input_files, output_files = self.pipeline_graph.get_io_filepaths(node_name)
         if len(output_slots) > 1:
             raise NotImplementedError(
-                "FIXME [MIC-5883] Multiple output slots/files of EmbarrassinglyParallelSteps not yet supported"
+                "FIXME [MIC-5883] Multiple output slots/files of AutoParallelSteps not yet supported"
             )
         if len(output_files) > 1:
             raise ValueError(
@@ -388,7 +386,7 @@ use rule start_spark_worker from spark_cluster with:
         output_slot_attrs = list(output_slots.values())[0]
         if len(output_slot_attrs["filepaths"]) > 1:
             raise NotImplementedError(
-                "FIXME [MIC-5883] Multiple output slots/files of EmbarrassinglyParallelSteps not yet supported"
+                "FIXME [MIC-5883] Multiple output slots/files of AutoParallelSteps not yet supported"
             )
         checkpoint_rule_name = f"checkpoints.{implementation.splitter_node_name}"
         AggregationRule(
@@ -404,7 +402,7 @@ use rule start_spark_worker from spark_cluster with:
     def _get_validations(
         node_name: str,
         input_slots: dict[str, dict[str, str | list[str]]],
-        is_embarrassingly_parallel: bool,
+        is_auto_parallel: bool,
     ) -> tuple[list[str], list[InputValidationRule]]:
         """Gets the validation rule and its output filepath for each slot for a given node.
 
@@ -423,10 +421,10 @@ use rule start_spark_worker from spark_cluster with:
         validation_rules = []
 
         for input_slot_name, input_slot_attrs in input_slots.items():
-            # embarrassingly parallel implementations rely on snakemake wildcards
+            # auto-parallel implementations rely on snakemake wildcards
             # TODO: [MIC-5787] - need to support multiple wildcards at once
             validation_file = f"input_validations/{node_name}/{input_slot_name}_validator" + (
-                "-{chunk}" if is_embarrassingly_parallel else ""
+                "-{chunk}" if is_auto_parallel else ""
             )
             validation_files.append(validation_file)
             validation_rules.append(

{easylink-0.1.20 → easylink-0.1.21}/src/easylink/pipeline_graph.py

@@ -72,31 +72,26 @@ class PipelineGraph(ImplementationGraph):
         return any([implementation.requires_spark for implementation in self.implementations])
 
     @property
-    def any_embarrassingly_parallel(self) -> bool:
+    def any_auto_parallel(self) -> bool:
         """Whether or not any :class:`~easylink.implementation.Implementation` is
-        to be run in an embarrassingly parallel way."""
+        to be automatically run in parallel."""
         return any(
-            [
-                self.get_whether_embarrassingly_parallel(node)
-                for node in self.implementation_nodes
-            ]
+            [self.get_whether_auto_parallel(node) for node in self.implementation_nodes]
         )
 
-    def get_whether_embarrassingly_parallel(self, node: str) -> dict[str, bool]:
-        """Determines whether a node is to be run in an embarrassingly parallel way.
+    def get_whether_auto_parallel(self, node: str) -> dict[str, bool]:
+        """Determines whether a node is to be automatically run in parallel.
 
         Parameters
         ----------
         node
-            The node name to determine whether or not it is to be run in an
-            embarrassingly parallel way.
+            The node name to determine whether or not it is to be automatically run in parallel.
 
         Returns
         -------
-            A boolean indicating whether the node is to be run in an embarrassingly
-            parallel way.
+            A boolean indicating whether the node is to be automatically run in parallel.
         """
-        return self.nodes[node]["implementation"].is_embarrassingly_parallel
+        return self.nodes[node]["implementation"].is_auto_parallel
 
     def get_io_filepaths(self, node: str) -> tuple[list[str], list[str]]:
         """Gets all of a node's input and output filepaths from its edges.
@@ -482,9 +477,9 @@ class PipelineGraph(ImplementationGraph):
                         str(
                             Path("intermediate")
                             / node
-                            # embarrassingly parallel implementations rely on snakemake wildcards
+                            # auto-parallel implementations rely on snakemake wildcards
                             # TODO: [MIC-5787] - need to support multiple wildcards at once
-                            / ("{chunk}" if implementation.is_embarrassingly_parallel else "")
+                            / ("{chunk}" if implementation.is_auto_parallel else "")
                             / imp_outputs[edge_attrs["output_slot"].name]
                         ),
                     )

{easylink-0.1.20 → easylink-0.1.21}/src/easylink/pipeline_schema_constants/__init__.py

@@ -23,8 +23,8 @@ SCHEMA_PARAMS = {
     "combine_with_iteration": testing.SCHEMA_PARAMS_COMBINE_WITH_ITERATION,
     "combine_with_iteration_cycle": testing.SCHEMA_PARAMS_COMBINE_WITH_ITERATION,
     "combine_with_extra_node": testing.SCHEMA_PARAMS_THREE_STEPS,
-    "looping_ep_step": testing.SCHEMA_PARAMS_LOOPING_EP_STEP,
-    "ep_parallel_step": testing.SCHEMA_PARAMS_EP_PARALLEL_STEP,
-    "ep_loop_step": testing.SCHEMA_PARAMS_EP_LOOP_STEP,
-    "ep_hierarchical_step": testing.SCHEMA_PARAMS_EP_HIERARCHICAL_STEP,
+    "looping_auto_parallel_step": testing.SCHEMA_PARAMS_LOOPING_AUTO_PARALLEL_STEP,
+    "auto_parallel_cloneable_step": testing.SCHEMA_PARAMS_AUTO_PARALLEL_CLONEABLE_STEP,
+    "auto_parallel_loop_step": testing.SCHEMA_PARAMS_AUTO_PARALLEL_LOOP_STEP,
+    "auto_parallel_hierarchical_step": testing.SCHEMA_PARAMS_AUTO_PARALLEL_HIERARCHICAL_STEP,
 }

{easylink-0.1.20 → easylink-0.1.21}/src/easylink/pipeline_schema_constants/development.py

@@ -18,13 +18,13 @@ from easylink.graph_components import (
     OutputSlotMapping,
 )
 from easylink.step import (
+    AutoParallelStep,
     ChoiceStep,
-    EmbarrassinglyParallelStep,
+    CloneableStep,
     HierarchicalStep,
     InputStep,
     LoopStep,
     OutputStep,
-    ParallelStep,
     Step,
 )
 from easylink.utilities.aggregator_utils import concatenate_datasets
@@ -33,7 +33,7 @@ from easylink.utilities.validation_utils import validate_input_file_dummy
 
 NODES = [
     InputStep(),
-    ParallelStep(
+    CloneableStep(
         template_step=Step(
             step_name="step_1",
             input_slots=[
@@ -58,7 +58,7 @@ NODES = [
         output_slots=[OutputSlot("step_2_main_output")],
     ),
     LoopStep(
-        template_step=EmbarrassinglyParallelStep(
+        template_step=AutoParallelStep(
             step=Step(
                 step_name="step_3",
                 input_slots=[