easylink 0.1.23__tar.gz → 0.1.24__tar.gz

{easylink-0.1.23 → easylink-0.1.24}/CHANGELOG.rst

@@ -1,3 +1,7 @@
+**0.1.24 - 6/26/25**
+
+ - Properly escape special characters in envars specified via pipeline configuration
+
 **0.1.23 - 6/25/25**
 
  - Remove "dummy" pipeline references from codebase

{easylink-0.1.23 → easylink-0.1.24}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: easylink
-Version: 0.1.23
+Version: 0.1.24
 Summary: Research repository for the EasyLink ER ecosystem project.
 Home-page: https://github.com/ihmeuw/easylink
 Author: The EasyLink developers

{easylink-0.1.23 → easylink-0.1.24}/docs/source/user_guide/tutorials/pipeline_demo_improved.yaml

@@ -47,14 +47,14 @@ steps:
                 implementation:
                   name: splink_blocking_and_filtering
                   configuration:
-                    BLOCKING_RULES: "'l.first_name == r.first_name,l.last_name == r.last_name'"
+                    BLOCKING_RULES: "l.first_name == r.first_name,l.last_name == r.last_name"
                     LINK_ONLY: true
               evaluating_pairs:
                 implementation:
                   name: splink_evaluating_pairs
                   configuration:
-                    BLOCKING_RULES_FOR_TRAINING: "'l.first_name == r.first_name,l.last_name == r.last_name'"
-                    COMPARISONS: "'ssn:levenshtein,first_name:name,middle_initial:exact,last_name:name'"
+                    BLOCKING_RULES_FOR_TRAINING: "l.first_name == r.first_name,l.last_name == r.last_name"
+                    COMPARISONS: "ssn:levenshtein,first_name:name,middle_initial:exact,last_name:name"
                     PROBABILITY_TWO_RANDOM_RECORDS_MATCH: 0.0001  # == 1 / len(w2)
                     THRESHOLD_MATCH_PROBABILITY: 0
                     LINK_ONLY: true

{easylink-0.1.23 → easylink-0.1.24}/docs/source/user_guide/tutorials/pipeline_demo_naive.yaml

@@ -47,14 +47,14 @@ steps:
                 implementation:
                   name: splink_blocking_and_filtering
                   configuration:
-                    BLOCKING_RULES: "'l.first_name == r.first_name,l.last_name == r.last_name'"
+                    BLOCKING_RULES: "l.first_name == r.first_name,l.last_name == r.last_name"
                     LINK_ONLY: true
               evaluating_pairs:
                 implementation:
                   name: splink_evaluating_pairs
                   configuration:
-                    BLOCKING_RULES_FOR_TRAINING: "'l.first_name == r.first_name,l.last_name == r.last_name'"
-                    COMPARISONS: "'ssn:exact,first_name:exact,middle_initial:exact,last_name:exact'"
+                    BLOCKING_RULES_FOR_TRAINING: "l.first_name == r.first_name,l.last_name == r.last_name"
+                    COMPARISONS: "ssn:exact,first_name:exact,middle_initial:exact,last_name:exact"
                     PROBABILITY_TWO_RANDOM_RECORDS_MATCH: 0.0001  # == 1 / len(w2)
                     THRESHOLD_MATCH_PROBABILITY: 0
                     LINK_ONLY: true

easylink-0.1.24/src/easylink/_version.py

@@ -0,0 +1 @@
+__version__ = "0.1.24"

{easylink-0.1.23 → easylink-0.1.24}/src/easylink/cli.py

@@ -243,6 +243,7 @@ def generate_dag(
         input_data=input_data,
         computing_environment=None,
         results_dir=results_dir,
+        images_dir=None,
         schema_name=schema,
     )
     logger.info("*** DAG saved to result directory ***")

{easylink-0.1.23 → easylink-0.1.24}/src/easylink/rule.py

@@ -14,6 +14,7 @@ dynamically as strings and appended to the Snakefile.
 """
 
 import os
+import shlex
 from abc import ABC, abstractmethod
 from collections.abc import Callable
 from dataclasses import dataclass
@@ -204,7 +205,7 @@ rule:
         export  SPARK_MASTER_URL"""
         for var_name, var_value in self.envvars.items():
             shell_cmd += f"""
-        export {var_name}={var_value}"""
+        export {var_name}={shlex.quote(str(var_value))}"""
         # Log stdout/stderr to diagnostics directory
         shell_cmd += f"""
         {self.script_cmd} > {{log}} 2>&1

{easylink-0.1.23 → easylink-0.1.24}/src/easylink.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: easylink
-Version: 0.1.23
+Version: 0.1.24
 Summary: Research repository for the EasyLink ER ecosystem project.
 Home-page: https://github.com/ihmeuw/easylink
 Author: The EasyLink developers

{easylink-0.1.23 → easylink-0.1.24}/src/easylink.egg-info/SOURCES.txt

@@ -194,7 +194,7 @@ tests/e2e/pipeline_improved_results.csv
 tests/e2e/pipeline_improved_results_2030.csv
 tests/e2e/pipeline_naive_results.csv
 tests/e2e/pipeline_splink_dummy_results.csv
-tests/e2e/test_easylink_run.py
+tests/e2e/test_easylink_cli.py
 tests/e2e/test_pipelines_main_schema.py
 tests/e2e/test_step_types.py
 tests/integration/test_compositions.py

easylink-0.1.23/tests/e2e/test_easylink_run.py → easylink-0.1.24/tests/e2e/test_easylink_cli.py

@@ -121,3 +121,26 @@ def test_easylink_run(
             "\n\n*** END OF TEST ***\n"
             f"[{pipeline_specification}, {input_data}, {computing_environment}]\n"
         )
+
+
+@pytest.mark.skipif(
+    not is_on_slurm(),
+    reason="Must be on slurm to run this test.",
+)
+def test_easylink_generate_dag(test_specific_results_dir):
+    cmd = (
+        "easylink generate-dag "
+        f"-p {SPECIFICATIONS_DIR / 'e2e' / 'pipeline.yaml'} "
+        f"-i {SPECIFICATIONS_DIR / 'common' / 'input_data.yaml'} "
+        f"-o {str(test_specific_results_dir)} "
+        "--no-timestamp "
+        "--schema development "
+    )
+    subprocess.run(
+        cmd,
+        shell=True,
+        stdout=sys.stdout,
+        stderr=sys.stderr,
+        check=True,
+    )
+    assert (test_specific_results_dir / "DAG.svg").exists()

{easylink-0.1.23 → easylink-0.1.24}/tests/specifications/e2e/pipeline_cascade.yaml

@@ -48,13 +48,13 @@ steps:
                     implementation:
                       name: splink_blocking_and_filtering
                       configuration:
-                        BLOCKING_RULES: "'l.last_name == r.last_name'"
+                        BLOCKING_RULES: "l.last_name == r.last_name"
                   evaluating_pairs:
                     implementation:
                       name: splink_evaluating_pairs
                       configuration:
-                        BLOCKING_RULES_FOR_TRAINING: "'l.first_name == r.first_name,l.last_name == r.last_name'"
-                        COMPARISONS: "'first_name:exact,last_name:exact'"
+                        BLOCKING_RULES_FOR_TRAINING: "l.first_name == r.first_name,l.last_name == r.last_name"
+                        COMPARISONS: "first_name:exact,last_name:exact"
                         PROBABILITY_TWO_RANDOM_RECORDS_MATCH: 0.01 
                         THRESHOLD_MATCH_PROBABILITY: 0.1
               links_to_clusters:
@@ -112,13 +112,13 @@ steps:
                     implementation:
                       name: splink_blocking_and_filtering
                       configuration:
-                        BLOCKING_RULES: "'l.first_name == r.first_name'"
+                        BLOCKING_RULES: "l.first_name == r.first_name"
                   evaluating_pairs:
                     implementation:
                       name: splink_evaluating_pairs
                       configuration:
-                        BLOCKING_RULES_FOR_TRAINING: "'l.first_name == r.first_name,l.last_name == r.last_name'"
-                        COMPARISONS: "'first_name:exact,last_name:exact'"
+                        BLOCKING_RULES_FOR_TRAINING: "l.first_name == r.first_name,l.last_name == r.last_name"
+                        COMPARISONS: "first_name:exact,last_name:exact"
                         PROBABILITY_TWO_RANDOM_RECORDS_MATCH: 0.01 
                         THRESHOLD_MATCH_PROBABILITY: 0.1
               links_to_clusters:

{easylink-0.1.23 → easylink-0.1.24}/tests/specifications/e2e/pipeline_splink_dummy.yaml

@@ -47,13 +47,13 @@ steps:
                 implementation:
                   name: splink_blocking_and_filtering
                   configuration:
-                    BLOCKING_RULES: "'l.last_name == r.last_name'"
+                    BLOCKING_RULES: "l.last_name == r.last_name"
               evaluating_pairs:
                 implementation:
                   name: splink_evaluating_pairs
                   configuration:
-                    BLOCKING_RULES_FOR_TRAINING: "'l.first_name == r.first_name,l.last_name == r.last_name'"
-                    COMPARISONS: "'first_name:exact,last_name:exact'"
+                    BLOCKING_RULES_FOR_TRAINING: "l.first_name == r.first_name,l.last_name == r.last_name"
+                    COMPARISONS: "first_name:exact,last_name:exact"
                     PROBABILITY_TWO_RANDOM_RECORDS_MATCH: 0.01 
                     THRESHOLD_MATCH_PROBABILITY: 0.1
           links_to_clusters:

{easylink-0.1.23 → easylink-0.1.24}/tests/specifications/e2e/pipeline_with_fastLink.yaml

@@ -47,12 +47,12 @@ steps:
                 implementation:
                   name: splink_blocking_and_filtering
                   configuration:
-                    BLOCKING_RULES: "'l.last_name == r.last_name'"
+                    BLOCKING_RULES: "l.last_name == r.last_name"
               evaluating_pairs:
                 implementation:
                   name: fastLink_evaluating_pairs
                   configuration:
-                    COMPARISONS: "'first_name:exact,last_name:exact'"
+                    COMPARISONS: "first_name:exact,last_name:exact"
                     THRESHOLD_MATCH_PROBABILITY: 0.1
           links_to_clusters:
             implementation:

{easylink-0.1.23 → easylink-0.1.24}/tests/unit/test_rule.py

@@ -1,5 +1,6 @@
 # mypy: ignore-errors
 import os
+import shlex
 from pathlib import Path
 from tempfile import TemporaryDirectory
 
@@ -246,6 +247,103 @@ def test_validation_rule_build_rule():
     _check_rule(rule, file_path)
 
 
+@pytest.mark.parametrize(
+    "name, value, expected",
+    [
+        # Basic cases
+        ("SIMPLE_VAR", "simple_value", "export SIMPLE_VAR=simple_value"),
+        (
+            "VAR_WITH_SPACES",
+            "value with spaces",
+            "export VAR_WITH_SPACES='value with spaces'",
+        ),
+        # Quote cases
+        (
+            "VAR_WITH_SINGLE_QUOTES",
+            "value with 'single quotes'",
+            "export VAR_WITH_SINGLE_QUOTES='value with '\"'\"'single quotes'\"'\"''",
+        ),
+        (
+            "VAR_WITH_DOUBLE_QUOTES",
+            'with "double" quotes',
+            "export VAR_WITH_DOUBLE_QUOTES='with \"double\" quotes'",
+        ),
+        (
+            "VAR_WITH_MIXED_QUOTES",
+            "with both \"double\" and 'single' quotes",
+            "export VAR_WITH_MIXED_QUOTES='with both \"double\" and '\"'\"'single'\"'\"' quotes'",
+        ),
+        # Complex structured data
+        (
+            "VAR_WITH_CHARS",
+            "col1='value1',col2='value2'",
+            "export VAR_WITH_CHARS='col1='\"'\"'value1'\"'\"',col2='\"'\"'value2'\"'\"''",
+        ),
+        (
+            "COMPLEX_VAR",
+            "l.first_name == r.first_name,l.last_name == r.last_name,col1='value1',col2='value2'",
+            "export COMPLEX_VAR='l.first_name == r.first_name,l.last_name == r.last_name,col1='\"'\"'value1'\"'\"',col2='\"'\"'value2'\"'\"''",
+        ),
+        # Edge cases that could cause shell injection or parsing issues
+        ("EMPTY_VAR", "", "export EMPTY_VAR=''"),
+        (
+            "SHELL_METACHAR_VAR",
+            "with$dollar`backtick",
+            "export SHELL_METACHAR_VAR='with$dollar`backtick'",
+        ),
+        (
+            "COMMAND_SEP_VAR",
+            "with;semicolon&ampersand",
+            "export COMMAND_SEP_VAR='with;semicolon&ampersand'",
+        ),
+        (
+            "PIPE_REDIRECT_VAR",
+            "with|pipe>redirect",
+            "export PIPE_REDIRECT_VAR='with|pipe>redirect'",
+        ),
+        (
+            "WHITESPACE_VAR",
+            "with\nnewline\ttab",
+            "export WHITESPACE_VAR='with\nnewline\ttab'",
+        ),
+        # Original bug cases
+        (
+            "BLOCKING_RULES",
+            "l.last_name == r.last_name",
+            "export BLOCKING_RULES='l.last_name == r.last_name'",
+        ),
+        (
+            "BLOCKING_RULES_FOR_TRAINING",
+            "l.first_name == r.first_name,l.last_name == r.last_name",
+            "export BLOCKING_RULES_FOR_TRAINING='l.first_name == r.first_name,l.last_name == r.last_name'",
+        ),
+    ],
+)
+def test_implemented_rule_envvar_quoting(name, value, expected):
+    """Test that environment variables with special characters are properly quoted in shell commands."""
+
+    rule = ImplementedRule(
+        name="test_rule",
+        step_name="test_step",
+        implementation_name="test_impl",
+        input_slots={},
+        validations=[],
+        output=["test_output.txt"],
+        resources=None,
+        envvars={name: value},
+        diagnostics_dir="test_diagnostics",
+        image_path="test.sif",
+        script_cmd="echo test",
+        requires_spark=False,
+    )
+
+    shell_cmd = rule._build_shell_cmd()
+
+    # Verify that each environment variable is properly quoted using shlex.quote
+    assert expected == f"export {name}={shlex.quote(str(value))}"
+    assert expected in shell_cmd
+
+
 ####################
 # Helper functions #
 ####################

easylink-0.1.23/src/easylink/_version.py

@@ -1 +0,0 @@
-__version__ = "0.1.23"