causal-testing-framework 14.1.0__tar.gz → 14.3.0__tar.gz

{causal_testing_framework-14.1.0 → causal_testing_framework-14.3.0}/.github/workflows/ci-tests-drafts.yaml

@@ -13,7 +13,7 @@ jobs:
     strategy:
       matrix:
         os: ["ubuntu-latest", "windows-latest", "macos-latest"]
-        python-version: ["3.10", "3.11", "3.12"]
+        python-version: ["3.11", "3.12", "3.13", "3.14"]
     steps:
     - uses: actions/checkout@v4
     - name: Set up Python

{causal_testing_framework-14.1.0 → causal_testing_framework-14.3.0}/.github/workflows/ci-tests.yaml

@@ -18,7 +18,7 @@ jobs:
     strategy:
       matrix:
         os: ["ubuntu-latest", "windows-latest", "macos-latest"]
-        python-version: ["3.10", "3.11", "3.12", "3.13"]
+        python-version: ["3.11", "3.12", "3.13", "3.14"]
     steps:
     - uses: actions/checkout@v4
     - name: Set up Python

{causal_testing_framework-14.1.0 → causal_testing_framework-14.3.0}/.github/workflows/publish-to-dafni.yaml

@@ -40,7 +40,7 @@ jobs:
 
       - uses: actions/setup-python@v3
         with:
-          python-version: '3.10'
+          python-version: '3.11'
 
       - name: Install dependencies
         run: |

{causal_testing_framework-14.1.0 → causal_testing_framework-14.3.0}/.github/workflows/publish-to-pypi.yaml

@@ -16,7 +16,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v3
         with:
-          python-version: '3.10'
+          python-version: '3.11'
 
       - name: Install build tools
         run: |

{causal_testing_framework-14.1.0 → causal_testing_framework-14.3.0}/.gitignore

@@ -67,3 +67,5 @@ temp/
 
 # log files
 *.log
+
+causal_test_results.json

{causal_testing_framework-14.1.0 → causal_testing_framework-14.3.0}/.pre-commit-config.yaml

@@ -19,7 +19,7 @@ repos:
     rev: 25.9.0
     hooks:
       - id: black
-        args: ['--line-length=120', '--target-version=py310']
+        args: ['--line-length=120', '--target-version=py311']
         files: ^causal_testing/
 
   - repo: https://github.com/pycqa/isort

{causal_testing_framework-14.1.0 → causal_testing_framework-14.3.0}/.readthedocs.yaml

@@ -7,9 +7,9 @@ version: 2
 
 # Set the version of Python
 build:
-  os: ubuntu-20.04
+  os: ubuntu-24.04
   tools:
-    python: "3.10"
+    python: "3.11"
 
 # Build documentation in the docs/ directory with Sphinx
 sphinx:
@@ -24,4 +24,4 @@ python:
    install:
    - requirements: docs/source/requirements.txt
    - method: pip
-     path: . # This should tell RTD to install the CTF from root to render the Estimators page correctly.
+     path: . # This should tell RTD to install the CTF from root to render the Estimators page correctly.

{causal_testing_framework-14.1.0/causal_testing_framework.egg-info → causal_testing_framework-14.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: causal_testing_framework
-Version: 14.1.0
+Version: 14.3.0
 Summary: A framework for causal testing using causal directed acyclic graphs.
 Author: The CITCOM team
 License: MIT
@@ -9,26 +9,25 @@ Project-URL: Documentation, https://causal-testing-framework.readthedocs.io/
 Project-URL: Repository, https://github.com/CITCOM-project/CausalTestingFramework
 Project-URL: Issues, https://github.com/CITCOM-project/CausalTestingFramework/issues
 Keywords: causal inference,verification
-Requires-Python: <3.14,>=3.10
+Requires-Python: <3.15,>=3.11
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: lifelines<=0.28.0; python_version <= "3.10"
-Requires-Dist: lifelines; python_version > "3.10"
-Requires-Dist: networkx<3.5,>=3.4
-Requires-Dist: numpy<=2.2.0,>=1.26.0
-Requires-Dist: pandas<3,>=2.1
+Requires-Dist: lifelines
+Requires-Dist: networkx<3.7,>=3.4
+Requires-Dist: numpy<=2.4.4,>=1.26.0
+Requires-Dist: pandas>=2.1
 Requires-Dist: scikit_learn~=1.4
-Requires-Dist: scipy<1.14,>=1.12.0; python_version <= "3.10"
-Requires-Dist: scipy<=1.16.2,>=1.12.0; python_version > "3.10"
+Requires-Dist: scipy<=1.17.1,>=1.12.0
 Requires-Dist: statsmodels~=0.14
 Requires-Dist: tabulate~=0.9
 Requires-Dist: pydot~=2.0
 Requires-Dist: pygad~=3.3
 Requires-Dist: deap~=1.4.1
-Requires-Dist: sympy~=1.13.1
-Requires-Dist: pyarrow~=19.0.1
-Requires-Dist: fastparquet~=2024.11.0
+Requires-Dist: sympy~=1.14.0
+Requires-Dist: pyarrow<25,>=19.0.1
+Requires-Dist: fastparquet>=2024.11.0
 Requires-Dist: tqdm~=4.67.1
+Requires-Dist: rustworkx>=0.18.0
 Provides-Extra: dev
 Requires-Dist: astroid==3.3.8; extra == "dev"
 Requires-Dist: isort; extra == "dev"
@@ -80,7 +79,7 @@ a deliberate modification to the input configuration expected to produce a corre
 ## Installation
 
 ### Requirements
-- Python 3.10, 3.11, 3.12 and 3.13
+- Python 3.11, 3.12, 3.13 and 3.14
 
 ### Recommended: Install from conda-forge
 
@@ -142,12 +141,12 @@ For more information on how to use the Causal Testing Framework, please refer to
 
 2. If you do not already have causal test cases, you can convert your causal DAG to causal tests by running the following command.
 ```
-python -m causal_testing generate --dag-path $PATH_TO_DAG --output $PATH_TO_TESTS
+causal-testing generate --dag-path $PATH_TO_DAG --output $PATH_TO_TESTS
 ```
 
 3. You can now execute your tests by running the following command.
 ```
-python -m causal_testing test --dag-path $PATH_TO_DAG --data-paths $PATH_TO_DATA --test-config $PATH_TO_TESTS --output $OUTPUT
+causal-testing test --dag-path $PATH_TO_DAG --data-paths $PATH_TO_DATA --test-config $PATH_TO_TESTS --output $OUTPUT
 ```
 The results will be saved for inspection in a JSON file located at `$OUTPUT`.
 In the future, we hope to add a visualisation tool to assist with this.

{causal_testing_framework-14.1.0 → causal_testing_framework-14.3.0}/README.md

@@ -27,7 +27,7 @@ a deliberate modification to the input configuration expected to produce a corre
 ## Installation
 
 ### Requirements
-- Python 3.10, 3.11, 3.12 and 3.13
+- Python 3.11, 3.12, 3.13 and 3.14
 
 ### Recommended: Install from conda-forge
 
@@ -89,12 +89,12 @@ For more information on how to use the Causal Testing Framework, please refer to
 
 2. If you do not already have causal test cases, you can convert your causal DAG to causal tests by running the following command.
 ```
-python -m causal_testing generate --dag-path $PATH_TO_DAG --output $PATH_TO_TESTS
+causal-testing generate --dag-path $PATH_TO_DAG --output $PATH_TO_TESTS
 ```
 
 3. You can now execute your tests by running the following command.
 ```
-python -m causal_testing test --dag-path $PATH_TO_DAG --data-paths $PATH_TO_DATA --test-config $PATH_TO_TESTS --output $OUTPUT
+causal-testing test --dag-path $PATH_TO_DAG --data-paths $PATH_TO_DATA --test-config $PATH_TO_TESTS --output $OUTPUT
 ```
 The results will be saved for inspection in a JSON file located at `$OUTPUT`.
 In the future, we hope to add a visualisation tool to assist with this.

causal_testing_framework-14.3.0/causal_testing/__main__.py

@@ -0,0 +1,137 @@
+"""This module contains the main entrypoint functionality to the Causal Testing Framework."""
+
+import json
+import logging
+import os
+import tempfile
+from importlib.metadata import entry_points
+from pathlib import Path
+
+import networkx as nx
+import pandas as pd
+
+from causal_testing.testing.metamorphic_relation import generate_causal_tests
+
+from .main import CausalTestingFramework, CausalTestingPaths, Command, parse_args, setup_logging
+
+
+def main() -> None:
+    """
+
+    Main entry point for the Causal Testing Framework
+
+    """
+
+    # Parse arguments
+    args = parse_args()
+    # Setup logging
+    setup_logging(args.log_level)
+
+    match args.command:
+        case Command.GENERATE:
+            logging.info("Generating causal tests")
+            generate_causal_tests(
+                args.dag_path,
+                args.output,
+                args.ignore_cycles,
+                args.threads,
+                effect_type=args.effect_type,
+                estimate_type=args.estimate_type,
+                estimator=args.estimator,
+                skip=False,
+            )
+            logging.info("Causal test generation completed successfully.")
+
+        case Command.DISCOVER:
+            discover_map = {ff.name: ff for ff in entry_points(group="discovery")}
+            if args.technique not in discover_map:
+                raise ValueError(
+                    f"Unsupported technique {args.technique}. Supported: {sorted(discover_map)}. "
+                    "If you have implemented a custom technique, you will need to add this to your entrypoints via "
+                    "your pyproject.toml file."
+                )
+            kwargs = {}
+            for argument in args.technique_kwargs:
+                split = argument.split("=")
+                if len(split) != 2:
+                    raise ValueError(f"Malformed argument {argument}. Should be specified as `arg_name=arg_value`")
+                kwargs[split[0]] = split[1]
+
+            logging.info("Discovering causal structure")
+            # Need to reset index to allow for multiple files having the same index (i.e. starting at zero).
+            # Otherwise you end up with duplicate indices, which causes problems further down the line
+            df = pd.concat([pd.read_csv(path) for path in args.data_paths]).reset_index()
+            if args.variables:
+                df = df[args.variables]
+
+            discover_class = discover_map[args.technique].load()
+            discover = discover_class(
+                df=df,
+                exclude_edges=(
+                    list(nx.nx_pydot.read_dot(args.exclude_edges).edges()) if args.exclude_edges is not None else []
+                ),
+                include_edges=(
+                    list(nx.nx_pydot.read_dot(args.include_edges).edges()) if args.include_edges is not None else []
+                ),
+                alpha=args.alpha,
+                **kwargs,
+            )
+            evolved_dag = discover.discover()
+            discover.write_dot(evolved_dag, args.output)
+            logging.info("Causal structure discovery completed successfully.")
+        case Command.TEST:
+            # Create paths object
+            paths = CausalTestingPaths(
+                dag_path=args.dag_path,
+                data_paths=args.data_paths,
+                test_config_path=args.test_config,
+                output_path=args.output,
+            )
+
+            # Create and setup framework
+            framework = CausalTestingFramework(paths, ignore_cycles=args.ignore_cycles, query=args.query)
+            framework.setup()
+
+            # Load and run tests
+            framework.load_tests()
+
+            if args.batch_size > 0:
+                logging.info(f"Running tests in batches of size {args.batch_size}")
+                with tempfile.TemporaryDirectory() as tmpdir:
+                    output_files = []
+                    for i, results in enumerate(
+                        framework.run_tests_in_batches(
+                            batch_size=args.batch_size,
+                            silent=args.silent,
+                            adequacy=args.adequacy,
+                            bootstrap_size=args.bootstrap_size,
+                        )
+                    ):
+                        temp_file_path = os.path.join(tmpdir, f"output_{i}.json")
+                        framework.save_results(results, temp_file_path)
+                        output_files.append(temp_file_path)
+                        del results
+
+                    # Now stitch the results together from the temporary files
+                    all_results = []
+                    for file_path in output_files:
+                        with open(file_path, "r", encoding="utf-8") as f:
+                            all_results.extend(json.load(f))
+
+                    output_path = Path(args.output)
+                    output_path.parent.mkdir(parents=True, exist_ok=True)
+
+                    with open(args.output, "w", encoding="utf-8") as f:
+                        json.dump(all_results, f, indent=4)
+            else:
+                logging.info("Running tests in regular mode")
+                results = framework.run_tests(
+                    silent=args.silent, adequacy=args.adequacy, bootstrap_size=args.bootstrap_size
+                )
+                framework.save_results(results)
+
+            logging.info("Causal testing completed successfully.")
+
+
+if __name__ == "__main__":
+    main()

causal_testing_framework-14.3.0/causal_testing/_version.py

@@ -0,0 +1,24 @@
+# file generated by vcs-versioning
+# don't change, don't track in version control
+from __future__ import annotations
+
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
+
+version: str
+__version__: str
+__version_tuple__: tuple[int | str, ...]
+version_tuple: tuple[int | str, ...]
+commit_id: str | None
+__commit_id__: str | None
+
+__version__ = version = '14.3.0'
+__version_tuple__ = version_tuple = (14, 3, 0)
+
+__commit_id__ = commit_id = 'ge6a19c77b'

causal_testing_framework-14.3.0/causal_testing/discovery/abstract_discovery.py

@@ -0,0 +1,236 @@
+"""
+This module implements the abstract Discovery class to infer causal DAGs from data.
+"""
+
+import random
+import re
+import warnings
+from abc import ABC, abstractmethod
+from enum import Enum
+from itertools import permutations
+
+import networkx as nx
+import pandas as pd
+import rustworkx as rx
+
+from causal_testing.main import CausalTestingFramework
+from causal_testing.specification.causal_dag import CausalDAG
+from causal_testing.specification.scenario import Scenario
+from causal_testing.testing.causal_effect import Negative, Positive
+from causal_testing.testing.causal_test_result import CausalTestResult
+from causal_testing.testing.metamorphic_relation import generate_metamorphic_relations
+
+TestResult = Enum("TestResult", [("PASS", 2), ("FAIL", 0), ("INESTIMABLE", 1)])
+
+# Ignore warnings from statsmodels when we try to evaluate test cases
+warnings.simplefilter("ignore")
+
+
+def simple_cycle(causal_dag: CausalDAG):
+    """
+    Find a cycle in the given CausalDAG, if one exists, returns the first found.
+
+    :param causal_dag: The CausalDAG to check for cycles.
+    :returns: A list of edges in the cycle, or an empty list if there are no cycles.
+    """
+    rx_graph = rx.networkx_converter(causal_dag)
+    return [(rx_graph[i], rx_graph[j]) for i, j in rx.digraph_find_cycle(rx_graph)]
+
+
+def effect_direction(result: CausalTestResult) -> str:
+    """
+    Check whether the estimated causal effect is negative or positive.
+
+    :param result: The causal test result object.
+    :returns: Whether the estimated causal test is positive or negative (or no effect).
+    """
+    if pd.api.types.is_numeric_dtype(
+        result.estimator.df[result.estimator.base_test_case.treatment_variable.name]
+    ) and pd.api.types.is_numeric_dtype(result.estimator.df[result.estimator.base_test_case.outcome_variable.name]):
+        if Negative().apply(result):
+            return "negative"
+        if Positive().apply(result):
+            return "positive"
+    return None
+
+
+def is_match(u: str, v: str, patterns: list[str]):
+    """
+    Check whether a given edge matches a given pattern.
+
+    :param u: The origin node of the edge.
+    :param v: The destination node of the edge.
+    :param patterns: A list of tuples containing the patterns to check against.
+    :returns: True if the edge matches the pattern, False otherwise.
+    """
+    return any(re.fullmatch(pat_u, u) and re.fullmatch(pat_v, v) for pat_u, pat_v in patterns)
+
+
+class Discovery(ABC):
+    """
+    Abstract class for causal discovery.
+    """
+
+    def __init__(
+        self,
+        df: pd.DataFrame,
+        random_seed: int = 0,
+        exclude_edges: str = None,
+        include_edges: str = None,
+        alpha: float = 0.05,
+    ):
+
+        random.seed(random_seed)
+        self.df = df
+        self.random_seed = int(random_seed)
+        self.alpha = float(alpha)
+
+        self.possible_edges = []
+        self.include_edges = []
+        self.exclude_edges = []
+
+        for u, v in permutations(df.columns, 2):
+            if exclude_edges and is_match(u, v, exclude_edges):
+                self.exclude_edges.append((u, v))
+            else:
+                self.possible_edges.append((u, v))
+
+            if include_edges and is_match(u, v, include_edges):
+                self.include_edges.append((u, v))
+
+        if self.include_edges:
+            # Check to make sure that the include edges don't specify a cycle
+            initial_dag = CausalDAG()
+            initial_dag.add_edges_from(self.include_edges)
+
+            if not initial_dag.is_acyclic():
+                raise ValueError(
+                    "Specified include edges include a cycle, making it impossible to infer a DAG. "
+                    "Please resolve this and try again."
+                )
+
+    @abstractmethod
+    def discover(self) -> CausalDAG:
+        """
+        Discover the causal DAG.
+
+        :returns: The inferred causal DAG.
+        """
+
+    def remove_cycles(self, causal_dag: CausalDAG):
+        """
+        Remove cycles from individuals by iteratively deleting a random edge from each cycle until there are no more
+        cycles.
+
+        :param causal_dag: The CausalDAG to be repaired.
+        """
+        nodes = causal_dag.nodes
+        cycle = simple_cycle(causal_dag)
+        while cycle:
+            idx = random.choice(range(len(cycle)))
+            while cycle[idx] in self.include_edges:
+                idx = (idx + 1) % len(cycle)
+            causal_dag.remove_edge(cycle[idx][0], cycle[idx][1])
+            cycle = simple_cycle(causal_dag)
+        causal_dag.add_nodes_from(nodes)
+
+    def write_dot(self, individual: CausalDAG, output_file: str):
+        """
+        Write the given individual to the given output file.
+
+        :param individual: The causal DAG to output.
+        :param output_file: The name of the file to write to.
+        """
+        if hasattr(individual, "test_results"):
+            for _, test in individual.test_results.iterrows():
+                if (test["treatment"], test["outcome"]) in individual.edges:
+                    if test["result"] == TestResult.PASS:
+                        individual[test["treatment"]][test["outcome"]]["color"] = "green"
+                    elif test["result"] == TestResult.INESTIMABLE:
+                        individual[test["treatment"]][test["outcome"]]["color"] = "orange"
+                    elif test["result"] == TestResult.FAIL:
+                        individual[test["treatment"]][test["outcome"]]["color"] = "red"
+                    else:
+                        raise ValueError(f"Invalid test outcome {test['result']}")
+                else:
+                    individual.add_edge(test["treatment"], test["outcome"], ignore_cycles=True)
+                    individual[test["treatment"]][test["outcome"]]["style"] = "dashed"
+                    if test["result"] == TestResult.PASS:
+                        individual[test["treatment"]][test["outcome"]]["style"] = "invis"
+                        individual[test["treatment"]][test["outcome"]]["constraint"] = False
+                    elif test["result"] == TestResult.INESTIMABLE:
+                        individual[test["treatment"]][test["outcome"]]["color"] = "orange"
+                    elif test["result"] == TestResult.FAIL:
+                        individual[test["treatment"]][test["outcome"]]["color"] = "red"
+                    else:
+                        raise ValueError(f"Invalid test outcome {test['result']}")
+
+        nx.drawing.nx_pydot.write_dot(individual, output_file)
+
+    def _json_stub_params(self, outcome: str) -> str:
+        if pd.api.types.is_bool_dtype(self.df[outcome]):
+            return {"estimator": "LogisticRegressionEstimator", "estimate_type": "unit_odds_ratio"}
+        if pd.api.types.is_categorical_dtype(self.df[outcome]) or pd.api.types.is_object_dtype(self.df[outcome]):
+            return {"estimator": "MultinomialRegressionEstimator", "estimate_type": "unit_odds_ratio"}
+        if pd.api.types.is_numeric_dtype(self.df[outcome]):
+            return {"estimator": "LinearRegressionEstimator", "estimate_type": "coefficient"}
+        raise ValueError(f"Invalid datatype {self.df.dtypes[outcome]}")
+
+    def evaluate_tests(self, causal_dag: CausalDAG) -> pd.DataFrame:
+        """
+        Generate and evaluate causal test cases from the supplied CausalDAG and return a list of edges for which the
+        corresponding causal test case failed.
+        These results are then assigned to a new attribute `test_results` within the individual for later reuse.
+
+        :param causal_dag: The CausalDAG to evaluate.
+        :returns: Pandas dataframe with test outcome details
+                  (result, expected effect, treatment, outcome, effect direction).
+        """
+
+        ctf = CausalTestingFramework(None)
+        ctf.dag = causal_dag
+        ctf.data = self.df
+        ctf.create_variables()
+        ctf.scenario = Scenario(list(ctf.variables["inputs"].values()) + list(ctf.variables["outputs"].values()))
+
+        ctf.test_cases = ctf.create_test_cases(
+            {
+                "tests": [
+                    relation.to_json_stub(
+                        alpha=self.alpha,
+                        **self._json_stub_params(relation.base_test_case.outcome_variable),
+                    )
+                    for relation in generate_metamorphic_relations(causal_dag)
+                ]
+            }
+        )
+
+        results = []
+
+        # We use "silent=False" here to allow for inestimable edges, but it'd be good to have a more stringent
+        # error catching strategy to catch "genuine" problems (e.g. to do with the structure of the data)
+        for test_case, result in zip(ctf.test_cases, ctf.run_tests(silent=False)):
+            if result.effect_estimate is None:
+                results.append(
+                    {
+                        "result": TestResult.INESTIMABLE,
+                        "expected_effect": test_case.expected_causal_effect.__class__.__name__,
+                        "treatment": test_case.base_test_case.treatment_variable.name,
+                        "outcome": test_case.base_test_case.outcome_variable.name,
+                    }
+                )
+            else:
+                results.append(
+                    {
+                        "result": (
+                            TestResult.PASS if test_case.expected_causal_effect.apply(result) else TestResult.FAIL
+                        ),
+                        "expected_effect": test_case.expected_causal_effect.__class__.__name__,
+                        "treatment": test_case.base_test_case.treatment_variable.name,
+                        "outcome": test_case.base_test_case.outcome_variable.name,
+                        "effect": effect_direction(result),
+                    }
+                )
+
+        causal_dag.test_results = pd.DataFrame(results)
+        return pd.DataFrame(results)