causal-testing-framework 14.2.0__tar.gz → 14.3.0__tar.gz

{causal_testing_framework-14.2.0 → causal_testing_framework-14.3.0}/.gitignore

@@ -67,3 +67,5 @@ temp/
 
 # log files
 *.log
+
+causal_test_results.json

{causal_testing_framework-14.2.0/causal_testing_framework.egg-info → causal_testing_framework-14.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: causal_testing_framework
-Version: 14.2.0
+Version: 14.3.0
 Summary: A framework for causal testing using causal directed acyclic graphs.
 Author: The CITCOM team
 License: MIT
@@ -27,6 +27,7 @@ Requires-Dist: sympy~=1.14.0
 Requires-Dist: pyarrow<25,>=19.0.1
 Requires-Dist: fastparquet>=2024.11.0
 Requires-Dist: tqdm~=4.67.1
+Requires-Dist: rustworkx>=0.18.0
 Provides-Extra: dev
 Requires-Dist: astroid==3.3.8; extra == "dev"
 Requires-Dist: isort; extra == "dev"

causal_testing_framework-14.3.0/causal_testing/__main__.py

@@ -0,0 +1,137 @@
+"""This module contains the main entrypoint functionality to the Causal Testing Framework."""
+
+import json
+import logging
+import os
+import tempfile
+from importlib.metadata import entry_points
+from pathlib import Path
+
+import networkx as nx
+import pandas as pd
+
+from causal_testing.testing.metamorphic_relation import generate_causal_tests
+
+from .main import CausalTestingFramework, CausalTestingPaths, Command, parse_args, setup_logging
+
+
+def main() -> None:
+    """
+
+    Main entry point for the Causal Testing Framework
+
+    """
+
+    # Parse arguments
+    args = parse_args()
+    # Setup logging
+    setup_logging(args.log_level)
+
+    match args.command:
+        case Command.GENERATE:
+            logging.info("Generating causal tests")
+            generate_causal_tests(
+                args.dag_path,
+                args.output,
+                args.ignore_cycles,
+                args.threads,
+                effect_type=args.effect_type,
+                estimate_type=args.estimate_type,
+                estimator=args.estimator,
+                skip=False,
+            )
+            logging.info("Causal test generation completed successfully.")
+
+        case Command.DISCOVER:
+            discover_map = {ff.name: ff for ff in entry_points(group="discovery")}
+            if args.technique not in discover_map:
+                raise ValueError(
+                    f"Unsupported technique {args.technique}. Supported: {sorted(discover_map)}. "
+                    "If you have implemented a custom technique, you will need to add this to your entrypoints via "
+                    "your pyproject.toml file."
+                )
+            kwargs = {}
+            for argument in args.technique_kwargs:
+                split = argument.split("=")
+                if len(split) != 2:
+                    raise ValueError(f"Malformed argument {argument}. Should be specified as `arg_name=arg_value`")
+                kwargs[split[0]] = split[1]
+
+            logging.info("Discovering causal structure")
+            # Need to reset index to allow for multiple files having the same index (i.e. starting at zero).
+            # Otherwise you end up with duplicate indices, which causes problems further down the line
+            df = pd.concat([pd.read_csv(path) for path in args.data_paths]).reset_index()
+            if args.variables:
+                df = df[args.variables]
+
+            discover_class = discover_map[args.technique].load()
+            discover = discover_class(
+                df=df,
+                exclude_edges=(
+                    list(nx.nx_pydot.read_dot(args.exclude_edges).edges()) if args.exclude_edges is not None else []
+                ),
+                include_edges=(
+                    list(nx.nx_pydot.read_dot(args.include_edges).edges()) if args.include_edges is not None else []
+                ),
+                alpha=args.alpha,
+                **kwargs,
+            )
+            evolved_dag = discover.discover()
+            discover.write_dot(evolved_dag, args.output)
+            logging.info("Causal structure discovery completed successfully.")
+        case Command.TEST:
+            # Create paths object
+            paths = CausalTestingPaths(
+                dag_path=args.dag_path,
+                data_paths=args.data_paths,
+                test_config_path=args.test_config,
+                output_path=args.output,
+            )
+
+            # Create and setup framework
+            framework = CausalTestingFramework(paths, ignore_cycles=args.ignore_cycles, query=args.query)
+            framework.setup()
+
+            # Load and run tests
+            framework.load_tests()
+
+            if args.batch_size > 0:
+                logging.info(f"Running tests in batches of size {args.batch_size}")
+                with tempfile.TemporaryDirectory() as tmpdir:
+                    output_files = []
+                    for i, results in enumerate(
+                        framework.run_tests_in_batches(
+                            batch_size=args.batch_size,
+                            silent=args.silent,
+                            adequacy=args.adequacy,
+                            bootstrap_size=args.bootstrap_size,
+                        )
+                    ):
+                        temp_file_path = os.path.join(tmpdir, f"output_{i}.json")
+                        framework.save_results(results, temp_file_path)
+                        output_files.append(temp_file_path)
+                        del results
+
+                    # Now stitch the results together from the temporary files
+                    all_results = []
+                    for file_path in output_files:
+                        with open(file_path, "r", encoding="utf-8") as f:
+                            all_results.extend(json.load(f))
+
+                    output_path = Path(args.output)
+                    output_path.parent.mkdir(parents=True, exist_ok=True)
+
+                    with open(args.output, "w", encoding="utf-8") as f:
+                        json.dump(all_results, f, indent=4)
+            else:
+                logging.info("Running tests in regular mode")
+                results = framework.run_tests(
+                    silent=args.silent, adequacy=args.adequacy, bootstrap_size=args.bootstrap_size
+                )
+                framework.save_results(results)
+
+            logging.info("Causal testing completed successfully.")
+
+
+if __name__ == "__main__":
+    main()

{causal_testing_framework-14.2.0 → causal_testing_framework-14.3.0}/causal_testing/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '14.2.0'
-__version_tuple__ = version_tuple = (14, 2, 0)
+__version__ = version = '14.3.0'
+__version_tuple__ = version_tuple = (14, 3, 0)
 
-__commit_id__ = commit_id = 'g9f06df212'
+__commit_id__ = commit_id = 'ge6a19c77b'

causal_testing_framework-14.3.0/causal_testing/discovery/abstract_discovery.py

@@ -0,0 +1,236 @@
+"""
+This module implements the abstract Discovery class to infer causal DAGs from data.
+"""
+
+import random
+import re
+import warnings
+from abc import ABC, abstractmethod
+from enum import Enum
+from itertools import permutations
+
+import networkx as nx
+import pandas as pd
+import rustworkx as rx
+
+from causal_testing.main import CausalTestingFramework
+from causal_testing.specification.causal_dag import CausalDAG
+from causal_testing.specification.scenario import Scenario
+from causal_testing.testing.causal_effect import Negative, Positive
+from causal_testing.testing.causal_test_result import CausalTestResult
+from causal_testing.testing.metamorphic_relation import generate_metamorphic_relations
+
+TestResult = Enum("TestResult", [("PASS", 2), ("FAIL", 0), ("INESTIMABLE", 1)])
+
+# Ignore warnings from statsmodels when we try to evaluate test cases
+warnings.simplefilter("ignore")
+
+
+def simple_cycle(causal_dag: CausalDAG):
+    """
+    Find a cycle in the given CausalDAG, if one exists, returns the first found.
+
+    :param causal_dag: The CausalDAG to check for cycles.
+    :returns: A list of edges in the cycle, or an empty list if there are no cycles.
+    """
+    rx_graph = rx.networkx_converter(causal_dag)
+    return [(rx_graph[i], rx_graph[j]) for i, j in rx.digraph_find_cycle(rx_graph)]
+
+
+def effect_direction(result: CausalTestResult) -> str:
+    """
+    Check whether the estimated causal effect is negative or positive.
+
+    :param result: The causal test result object.
+    :returns: Whether the estimated causal test is positive or negative (or no effect).
+    """
+    if pd.api.types.is_numeric_dtype(
+        result.estimator.df[result.estimator.base_test_case.treatment_variable.name]
+    ) and pd.api.types.is_numeric_dtype(result.estimator.df[result.estimator.base_test_case.outcome_variable.name]):
+        if Negative().apply(result):
+            return "negative"
+        if Positive().apply(result):
+            return "positive"
+    return None
+
+
+def is_match(u: str, v: str, patterns: list[str]):
+    """
+    Check whether a given edge matches a given pattern.
+
+    :param u: The origin node of the edge.
+    :param v: The destination node of the edge.
+    :param patterns: A list of tuples containing the patterns to check against.
+    :returns: True if the edge matches the pattern, False otherwise.
+    """
+    return any(re.fullmatch(pat_u, u) and re.fullmatch(pat_v, v) for pat_u, pat_v in patterns)
+
+
+class Discovery(ABC):
+    """
+    Abstract class for causal discovery.
+    """
+
+    def __init__(
+        self,
+        df: pd.DataFrame,
+        random_seed: int = 0,
+        exclude_edges: str = None,
+        include_edges: str = None,
+        alpha: float = 0.05,
+    ):
+
+        random.seed(random_seed)
+        self.df = df
+        self.random_seed = int(random_seed)
+        self.alpha = float(alpha)
+
+        self.possible_edges = []
+        self.include_edges = []
+        self.exclude_edges = []
+
+        for u, v in permutations(df.columns, 2):
+            if exclude_edges and is_match(u, v, exclude_edges):
+                self.exclude_edges.append((u, v))
+            else:
+                self.possible_edges.append((u, v))
+
+            if include_edges and is_match(u, v, include_edges):
+                self.include_edges.append((u, v))
+
+        if self.include_edges:
+            # Check to make sure that the include edges don't specify a cycle
+            initial_dag = CausalDAG()
+            initial_dag.add_edges_from(self.include_edges)
+
+            if not initial_dag.is_acyclic():
+                raise ValueError(
+                    "Specified include edges include a cycle, making it impossible to infer a DAG. "
+                    "Please resolve this and try again."
+                )
+
+    @abstractmethod
+    def discover(self) -> CausalDAG:
+        """
+        Discover the causal DAG.
+
+        :returns: The inferred causal DAG.
+        """
+
+    def remove_cycles(self, causal_dag: CausalDAG):
+        """
+        Remove cycles from individuals by iteratively deleting a random edge from each cycle until there are no more
+        cycles.
+
+        :param causal_dag: The CausalDAG to be repaired.
+        """
+        nodes = causal_dag.nodes
+        cycle = simple_cycle(causal_dag)
+        while cycle:
+            idx = random.choice(range(len(cycle)))
+            while cycle[idx] in self.include_edges:
+                idx = (idx + 1) % len(cycle)
+            causal_dag.remove_edge(cycle[idx][0], cycle[idx][1])
+            cycle = simple_cycle(causal_dag)
+        causal_dag.add_nodes_from(nodes)
+
+    def write_dot(self, individual: CausalDAG, output_file: str):
+        """
+        Write the given individual to the given output file.
+
+        :param individual: The causal DAG to output.
+        :param output_file: The name of the file to write to.
+        """
+        if hasattr(individual, "test_results"):
+            for _, test in individual.test_results.iterrows():
+                if (test["treatment"], test["outcome"]) in individual.edges:
+                    if test["result"] == TestResult.PASS:
+                        individual[test["treatment"]][test["outcome"]]["color"] = "green"
+                    elif test["result"] == TestResult.INESTIMABLE:
+                        individual[test["treatment"]][test["outcome"]]["color"] = "orange"
+                    elif test["result"] == TestResult.FAIL:
+                        individual[test["treatment"]][test["outcome"]]["color"] = "red"
+                    else:
+                        raise ValueError(f"Invalid test outcome {test['result']}")
+                else:
+                    individual.add_edge(test["treatment"], test["outcome"], ignore_cycles=True)
+                    individual[test["treatment"]][test["outcome"]]["style"] = "dashed"
+                    if test["result"] == TestResult.PASS:
+                        individual[test["treatment"]][test["outcome"]]["style"] = "invis"
+                        individual[test["treatment"]][test["outcome"]]["constraint"] = False
+                    elif test["result"] == TestResult.INESTIMABLE:
+                        individual[test["treatment"]][test["outcome"]]["color"] = "orange"
+                    elif test["result"] == TestResult.FAIL:
+                        individual[test["treatment"]][test["outcome"]]["color"] = "red"
+                    else:
+                        raise ValueError(f"Invalid test outcome {test['result']}")
+
+        nx.drawing.nx_pydot.write_dot(individual, output_file)
+
+    def _json_stub_params(self, outcome: str) -> str:
+        if pd.api.types.is_bool_dtype(self.df[outcome]):
+            return {"estimator": "LogisticRegressionEstimator", "estimate_type": "unit_odds_ratio"}
+        if pd.api.types.is_categorical_dtype(self.df[outcome]) or pd.api.types.is_object_dtype(self.df[outcome]):
+            return {"estimator": "MultinomialRegressionEstimator", "estimate_type": "unit_odds_ratio"}
+        if pd.api.types.is_numeric_dtype(self.df[outcome]):
+            return {"estimator": "LinearRegressionEstimator", "estimate_type": "coefficient"}
+        raise ValueError(f"Invalid datatype {self.df.dtypes[outcome]}")
+
+    def evaluate_tests(self, causal_dag: CausalDAG) -> pd.DataFrame:
+        """
+        Generate and evaluate causal test cases from the supplied CausalDAG and return a list of edges for which the
+        corresponding causal test case failed.
+        These results are then assigned to a new attribute `test_results` within the individual for later reuse.
+
+        :param causal_dag: The CausalDAG to evaluate.
+        :returns: Pandas dataframe with test outcome details
+                  (result, expected effect, treatment, outcome, effect direction).
+        """
+
+        ctf = CausalTestingFramework(None)
+        ctf.dag = causal_dag
+        ctf.data = self.df
+        ctf.create_variables()
+        ctf.scenario = Scenario(list(ctf.variables["inputs"].values()) + list(ctf.variables["outputs"].values()))
+
+        ctf.test_cases = ctf.create_test_cases(
+            {
+                "tests": [
+                    relation.to_json_stub(
+                        alpha=self.alpha,
+                        **self._json_stub_params(relation.base_test_case.outcome_variable),
+                    )
+                    for relation in generate_metamorphic_relations(causal_dag)
+                ]
+            }
+        )
+
+        results = []
+
+        # We use "silent=False" here to allow for inestimable edges, but it'd be good to have a more stringent
+        # error catching strategy to catch "genuine" problems (e.g. to do with the structure of the data)
+        for test_case, result in zip(ctf.test_cases, ctf.run_tests(silent=False)):
+            if result.effect_estimate is None:
+                results.append(
+                    {
+                        "result": TestResult.INESTIMABLE,
+                        "expected_effect": test_case.expected_causal_effect.__class__.__name__,
+                        "treatment": test_case.base_test_case.treatment_variable.name,
+                        "outcome": test_case.base_test_case.outcome_variable.name,
+                    }
+                )
+            else:
+                results.append(
+                    {
+                        "result": (
+                            TestResult.PASS if test_case.expected_causal_effect.apply(result) else TestResult.FAIL
+                        ),
+                        "expected_effect": test_case.expected_causal_effect.__class__.__name__,
+                        "treatment": test_case.base_test_case.treatment_variable.name,
+                        "outcome": test_case.base_test_case.outcome_variable.name,
+                        "effect": effect_direction(result),
+                    }
+                )
+
+        causal_dag.test_results = pd.DataFrame(results)
+        return pd.DataFrame(results)

causal_testing_framework-14.3.0/causal_testing/discovery/hill_climber_discovery.py

@@ -0,0 +1,155 @@
+"""
+This module implements a hill climbing algorithm to optimise causal DAGs based on the tests that pass/fail.
+"""
+
+import random
+import time
+
+import numpy as np
+import pandas as pd
+
+from causal_testing.discovery.abstract_discovery import Discovery, TestResult
+from causal_testing.specification.causal_dag import CausalDAG
+
+
+class HillClimberDiscovery(Discovery):
+    """
+    Simple hill climber evolution of cauasl DAGs via 1+1EA.
+    Attempts to maximise the number of passing tests and minimise the number of failing tests.
+    """
+
+    def __init__(
+        self,
+        df: pd.DataFrame,
+        random_seed: int = 0,
+        include_edges: str = None,
+        exclude_edges: str = None,
+        alpha: float = 0.05,
+        max_iterations: int = 100,
+        max_iterations_without_improvement: int = 10,
+    ):
+        super().__init__(
+            df=df, random_seed=random_seed, include_edges=include_edges, exclude_edges=exclude_edges, alpha=alpha
+        )
+        self.max_iterations = int(max_iterations)
+        self.max_iterations_without_improvement = int(max_iterations_without_improvement)
+
+    def sum_test_outcomes(self, test_results: pd.DataFrame) -> dict:
+        """
+        Aggregate the number of passing, failing, and inestimable tests
+        :param test_results: Dataframe containing the raw pass/fail/inestimable outcome of each test case.
+        :returns: Dictionary containing the number of pass/fail/inestimable outcomes.
+        """
+        counts = pd.concat(
+            [
+                pd.DataFrame(np.sort(test_results[["treatment", "outcome"]], axis=1), columns=["treatment", "outcome"]),
+                pd.get_dummies(test_results["result"]).astype(int),
+            ],
+            axis=1,
+        )
+        # Ensure every column is initialised - Test outcomes that never occurred won't be in the dataframe otherwise
+        for col in TestResult:
+            if col not in counts.columns:
+                counts[col] = 0
+        counts = counts.groupby(["treatment", "outcome"]).sum().reset_index()[list(TestResult)]
+        # The below line normalises by the number of tests *for each edge*
+        # Independence tests X _||_ Y get two tests (X _||_ Y and Y _||_ X) because we don't know which way the
+        # causality flows. We need to normalise this (e.g. if X _||_ Y and Y _||_ X both pass, then the score should be
+        # 1 rather than 2) otherwise we end up unintentionally optimising for more independences.
+        counts = counts.apply(lambda col: col / counts.sum(axis=1))
+
+        return counts.sum(axis=0).to_dict()
+
+    def evaluate_fitness(
+        self,
+        individual: CausalDAG,
+    ) -> tuple[tuple[float, float, float], list[tuple[str, str]]]:
+        """
+        Evaluate the fitness of a given causal DAG by evaluating the corresponding test cases using a tier based
+        fitness metric.
+        lexicographical order (max pass, minimise failure, minimise unknown)
+        e.g. (X pass, Y fail, Z+1 unknown) is better than (X pass, Y+1 fail, Z unknown)
+
+        :param individual: The candidate individual to evaluate.
+        :returns: Tuple of the form (X, Y), where X is a triple containing the number of passing, failing, and
+                  inestimable tests respectively, and Y is a list of failing edges.
+        """
+        self.evaluate_tests(individual)
+        counts = self.sum_test_outcomes(individual.test_results)
+
+        # Add extra "var1" and "var2" columns to serve as order independent "treatment" and "outcome"
+        query_df = pd.concat(
+            [
+                individual.test_results,
+                pd.DataFrame(
+                    np.sort(individual.test_results[["treatment", "outcome"]], axis=1), columns=["var1", "var2"]
+                ),
+            ],
+            axis=1,
+        )
+        problem_tests = query_df.groupby(["var1", "var2"]).filter(
+            # Groups are problematic if at least one test fails or no test passes
+            lambda group: (group["result"] == TestResult.FAIL).any()
+            or ~(group["result"] == TestResult.PASS).any()
+        )
+        problem_edges = problem_tests[["treatment", "outcome"]].apply(tuple, axis=1).tolist()
+
+        fitness_values = (
+            counts.get(TestResult.PASS, 0),
+            -counts.get(TestResult.FAIL, 0),
+            -counts.get(TestResult.INESTIMABLE, 0),
+        )
+        return fitness_values, problem_edges
+
+    def discover(self) -> CausalDAG:
+        """
+        Discover the causal DAG.
+
+        :returns: The inferred causal DAG.
+        """
+
+        start_time = time.time()
+        individual = CausalDAG()
+        individual.add_nodes_from(self.df.columns)
+        individual.add_edges_from(self.possible_edges)
+        self.remove_cycles(individual)
+        fitness_values, problem_edges = self.evaluate_fitness(individual)
+
+        iterations = self.max_iterations
+        iterations_without_improvement = 0
+
+        while problem_edges and iterations:
+            iterations -= 1
+
+            new_individual = individual.copy()
+            for origin, dest in random.sample(
+                # If we've gone over the maximum iterations without improvement
+                problem_edges
+                + (
+                    self.possible_edges
+                    if iterations_without_improvement > self.max_iterations_without_improvement
+                    else []
+                ),
+                random.randint(1, len(problem_edges)),
+            ):
+                if new_individual.has_edge(origin, dest) and (origin, dest) not in self.include_edges:
+                    new_individual.remove_edge(origin, dest)
+                elif not new_individual.has_edge(origin, dest) and (origin, dest) not in self.exclude_edges:
+                    # Want to bypass the cycle check of CausalDAG as we remove the cycles afterwards
+                    new_individual.add_edge(origin, dest, ignore_cycles=True)
+            self.remove_cycles(new_individual)
+            new_fitness_values, new_problem_edges = self.evaluate_fitness(new_individual)
+
+            if new_fitness_values > fitness_values:
+                fitness_values = new_fitness_values
+                problem_edges = new_problem_edges
+                individual = new_individual
+                iterations_without_improvement = 0
+            else:
+                iterations_without_improvement += 1
+
+        end_time = time.time()
+        individual.graph["fitness"] = fitness_values
+        individual.graph["time"] = round(end_time - start_time)
+
+        return individual