causal-testing-framework 14.3.0__tar.gz → 15.0.0__tar.gz

causal_testing_framework-15.0.0/.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,8 @@
+## Checklist
+Before you mark your PR as ready for review, please ensure you have completed the following.
+
+- [ ] Added unit tests for new functionality
+- [ ] Properly documented any new code
+- [ ] Updated Sphinx documentation in `docs/` and checked for updated configuration/runtime options
+- [ ] Added a jupyter notebook tutorial in `docs/source/tutorials` for any new functionality
+- [ ] Verified local docs build with `sphinx-build -W source/ build`

causal_testing_framework-15.0.0/.github/workflows/build-docs.yaml

@@ -0,0 +1,29 @@
+name: Build docs
+
+on:
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  build:
+    name: MegaLinter
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Set up Pandoc
+        uses: pandoc/actions/setup@v1
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+         python-version: 3.12
+      - name: Install dependencies
+        run: |
+            python --version
+            python -m pip install --upgrade pip
+            pip install -e .[dev]
+      - name: "Build docs"
+        working-directory: "docs"
+        run: |
+           sphinx-build -W source/ build

{causal_testing_framework-14.3.0 → causal_testing_framework-15.0.0}/.github/workflows/ci-tests-drafts.yaml

@@ -25,7 +25,7 @@ jobs:
           python --version
           python -m pip install --upgrade pip
           pip install -e .
-          pip install -e .[test]
+          pip install -e .[dev]
           pip install pytest pytest-cov
     - name: Register Jupyter Kernel
       run: |

{causal_testing_framework-14.3.0 → causal_testing_framework-15.0.0}/.github/workflows/ci-tests.yaml

@@ -30,7 +30,7 @@ jobs:
           python --version
           python -m pip install --upgrade pip
           pip install -e .
-          pip install -e .[test]
+          pip install -e .[dev]
           pip install pytest pytest-cov
     - name: Register Jupyter Kernel
       run: |

{causal_testing_framework-14.3.0 → causal_testing_framework-15.0.0}/.pre-commit-config.yaml

@@ -3,31 +3,31 @@ repos:
     rev: v6.0.0
     hooks:
       - id: trailing-whitespace
-        files: ^causal_testing/
+        files: ^(causal_testing|tests)/
       - id: end-of-file-fixer
-        files: ^causal_testing/
+        files: ^(causal_testing|tests)/
       - id: check-added-large-files
-        files: ^causal_testing/
+        files: ^(causal_testing|tests)/
       - id: check-merge-conflict
-        files: ^causal_testing/
+        files: ^(causal_testing|tests)/
       - id: debug-statements
-        files: ^causal_testing/
+        files: ^(causal_testing|tests)/
       - id: mixed-line-ending
-        files: ^causal_testing/
+        files: ^(causal_testing|tests)/
 
   - repo: https://github.com/psf/black
     rev: 25.9.0
     hooks:
       - id: black
         args: ['--line-length=120', '--target-version=py311']
-        files: ^causal_testing/
+        files: ^(causal_testing|tests)/
 
   - repo: https://github.com/pycqa/isort
     rev: 7.0.0
     hooks:
       - id: isort
         args: ['--profile', 'black', '--line-length', '120']
-        files: ^causal_testing/
+        files: ^(causal_testing|tests)/
 
   - repo: local
     hooks:

{causal_testing_framework-14.3.0 → causal_testing_framework-15.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: causal_testing_framework
-Version: 14.3.0
+Version: 15.0.0
 Summary: A framework for causal testing using causal directed acyclic graphs.
 Author: The CITCOM team
 License: MIT
@@ -20,7 +20,7 @@ Requires-Dist: scikit_learn~=1.4
 Requires-Dist: scipy<=1.17.1,>=1.12.0
 Requires-Dist: statsmodels~=0.14
 Requires-Dist: tabulate~=0.9
-Requires-Dist: pydot~=2.0
+Requires-Dist: pydot>=2.0
 Requires-Dist: pygad~=3.3
 Requires-Dist: deap~=1.4.1
 Requires-Dist: sympy~=1.14.0
@@ -44,10 +44,9 @@ Requires-Dist: ipykernel; extra == "dev"
 Requires-Dist: pandoc; extra == "dev"
 Requires-Dist: pre-commit; extra == "dev"
 Requires-Dist: tox; extra == "dev"
-Provides-Extra: test
-Requires-Dist: nbclient; extra == "test"
-Requires-Dist: nbformat; extra == "test"
-Requires-Dist: ipykernel; extra == "test"
+Requires-Dist: nbclient; extra == "dev"
+Requires-Dist: nbformat; extra == "dev"
+Requires-Dist: ipykernel; extra == "dev"
 Dynamic: license-file
 
 # Causal Testing Framework
@@ -73,8 +72,8 @@ the inputs and outputs of the system under test, supported by mathematical found
 enable causal inference. Each causal test case targets the causal effect of a specific intervention on the system under test--that is,
 a deliberate modification to the input configuration expected to produce a corresponding change in one or more outputs.
 
-![Causal Testing Workflow](images/schematic-dark.png#gh-dark-mode-only)
-![Causal Testing Workflow](images/schematic.png#gh-light-mode-only)
+![Causal Testing Workflow](docs/source/_static/images/testing-workflow-dark.png#gh-dark-mode-only)
+![Causal Testing Workflow](docs/source/_static/images/testing-workflow.png#gh-light-mode-only)
 
 ## Installation
 

{causal_testing_framework-14.3.0 → causal_testing_framework-15.0.0}/README.md

@@ -21,8 +21,8 @@ the inputs and outputs of the system under test, supported by mathematical found
 enable causal inference. Each causal test case targets the causal effect of a specific intervention on the system under test--that is,
 a deliberate modification to the input configuration expected to produce a corresponding change in one or more outputs.
 
-![Causal Testing Workflow](images/schematic-dark.png#gh-dark-mode-only)
-![Causal Testing Workflow](images/schematic.png#gh-light-mode-only)
+![Causal Testing Workflow](docs/source/_static/images/testing-workflow-dark.png#gh-dark-mode-only)
+![Causal Testing Workflow](docs/source/_static/images/testing-workflow.png#gh-light-mode-only)
 
 ## Installation
 

causal_testing_framework-15.0.0/causal_testing/__main__.py

@@ -0,0 +1,283 @@
+"""This module contains the main entrypoint functionality to the Causal Testing Framework."""
+
+import argparse
+import json
+import logging
+from enum import Enum
+from importlib.metadata import entry_points
+from typing import Optional, Sequence
+from warnings import warn
+
+import networkx as nx
+import pandas as pd
+
+from causal_testing.causal_testing_framework import CausalTestingFramework, read_dataframe
+from causal_testing.specification.causal_dag import CausalDAG
+
+logger = logging.getLogger(__name__)
+
+
+class Command(Enum):
+    """
+    Enum for supported CTF commands.
+    """
+
+    TEST = "test"
+    GENERATE = "generate"
+    DISCOVER = "discover"
+    EVALUATE = "evaluate"
+
+
+def setup_logging(level: str) -> None:
+    """Set up logging configuration."""
+    logging.basicConfig(
+        level=level, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", datefmt="%Y-%m-%d %H:%M:%S"
+    )
+
+
+def parse_args(args: Optional[Sequence[str]] = None) -> argparse.Namespace:
+    """Parse command line arguments."""
+    main_parser = argparse.ArgumentParser(
+        add_help=True,
+        description="Causal Testing Framework - "
+        "A causal inference-driven framework for functional black-box testing of complex software.",
+    )
+
+    subparsers = main_parser.add_subparsers(
+        help="The action you want to run - call `causal_testing {action} -h` for further details", dest="command"
+    )
+
+    # Generation
+    parser_generate = subparsers.add_parser(Command.GENERATE.value, help="Generate causal tests from a DAG")
+    parser_generate.add_argument("-D", "--dag-path", help="Path to the DAG file (.dot)", required=True)
+    parser_generate.add_argument("-o", "--output", help="Path for output file (.json)", required=True)
+    parser_generate.add_argument(
+        "-i", "--ignore-cycles", help="Ignore cycles in DAG", action="store_true", default=False
+    )
+    parser_generate.add_argument(
+        "--threads", "-t", type=int, help="The number of parallel threads to use.", required=False, default=0
+    )
+
+    # Testing
+    parser_test = subparsers.add_parser(Command.TEST.value, help="Run causal tests")
+    parser_test.add_argument("-D", "--dag-path", help="Path to the DAG file (.dot)", required=True)
+    parser_test.add_argument("-o", "--output", help="Path for output file (.json)", required=True)
+    parser_test.add_argument("-i", "--ignore-cycles", help="Ignore cycles in DAG", action="store_true", default=False)
+    parser_test.add_argument("-t", "--test-config", help="Path to test configuration file (.json)", required=True)
+    parser_test.add_argument("-q", "--query", help="Query string to filter data (e.g. 'age > 18')", type=str)
+    parser_test.add_argument(
+        "-A", "--adequacy", help="Calculate causal test adequacy for each test case", action="store_true", default=False
+    )
+    parser_test.add_argument(
+        "-b",
+        "--adequacy-bootstrap-size",
+        dest="bootstrap_size",
+        help="Number of bootstrap samples for causal test adequacy. Defaults to 100",
+        type=int,
+    )
+    parser_test.add_argument(
+        "-s",
+        "--silent",
+        action="store_true",
+        help="Do not crash on error. If set to true, errors are recorded as test results.",
+        default=False,
+    )
+
+    # DAG evaluation
+    parser_evaluate = subparsers.add_parser(
+        Command.EVALUATE.value, help="Evaluate how well a causal DAG fits a dataset"
+    )
+    parser_evaluate.add_argument("-D", "--dag-path", help="Path to the DAG file (.dot)", required=True)
+    parser_evaluate.add_argument("-o", "--output", help="Path for output file (.csv)", required=True)
+    parser_evaluate.add_argument(
+        "-i", "--ignore-cycles", help="Ignore cycles in DAG", action="store_true", default=False
+    )
+    parser_evaluate.add_argument("-q", "--query", help="Query string to filter data (e.g. 'age > 18')", type=str)
+    parser_evaluate.add_argument(
+        "-b",
+        "--adequacy-bootstrap-size",
+        dest="bootstrap_size",
+        help="Number of bootstrap samples for causal test adequacy. Defaults to 100",
+        type=int,
+        default=100,
+    )
+    parser_evaluate.add_argument(
+        "-s",
+        "--silent",
+        action="store_true",
+        help="Do not crash on error. If set to true, errors are recorded as test results.",
+        default=False,
+    )
+    parser_evaluate.add_argument("-t", "--test-config", help="Path to test configuration file (.json)")
+
+    # Discovery
+    parser_discover = subparsers.add_parser(Command.DISCOVER.value, help="Discover causal structures from data")
+    parser_discover.add_argument(
+        "-t",
+        "--technique",
+        default="HillClimberDiscovery",
+        help=(
+            "The name of the technique to use. Currently supported are 'HillClimberDiscovery' and 'NSGADiscovery'. "
+            "Defaults to HillClimberDiscovery."
+        ),
+    )
+    parser_discover.add_argument(
+        "-V",
+        "--variables",
+        help="The subset of variables from the data to consider. Defaults to all.",
+        nargs="*",
+        default=[],
+    )
+    parser_discover.add_argument("-o", "--output", help="Path for output DAG file (.dot)", required=True)
+    parser_discover.add_argument(
+        "-i", "--include-edges", help="Path to file containing edges to include", required=False
+    )
+    parser_discover.add_argument(
+        "-e", "--exclude-edges", help="Path to file containing edges to exclude", required=False
+    )
+    parser_discover.add_argument(
+        "--technique-kwargs",
+        help="Keywords for the discovery technique. These should be specified as `arg1=value1 arg2=value2...`.",
+        nargs="*",
+        default=[],
+    )
+
+    for parser in [parser_generate, parser_discover, parser_test, parser_evaluate]:
+        parser.add_argument(
+            "-l",
+            "--log_level",
+            default="WARNING",
+            type=str.upper,
+            choices=["NONE", "DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"],
+            help="Set the logging level (default: WARNING).",
+        )
+        parser.add_argument(
+            "-a",
+            "--alpha",
+            help=(
+                "The significance level of the confidence intervals used to determine causality. "
+                "This should be a value between 0 and 1. Defaults to 0.05 for 95%% confidence intervals."
+            ),
+            default=0.05,
+        )
+        parser.add_argument("-d", "--data-paths", help="Paths to data files (.csv)", nargs="+", required=True)
+
+    args = main_parser.parse_args(args)
+
+    # Assume the user wants test adequacy if they're setting bootstrap_size
+    if getattr(args, "bootstrap_size", None) is not None:
+        args.adequacy = True
+    if getattr(args, "adequacy", False) and getattr(args, "bootstrap_size", None) is None:
+        # Need this here rather than a default value because otherwise the above always sets adequacy to True
+        args.bootstrap_size = 100
+
+    args.command = Command(args.command)
+    return args
+
+
+def main() -> None:
+    """
+
+    Main entry point for the Causal Testing Framework
+
+    """
+
+    # Parse arguments
+    args = parse_args()
+    # Setup logging
+    setup_logging(args.log_level)
+
+    match args.command:
+        case Command.GENERATE:
+            logging.info("Generating causal tests")
+            df = pd.concat(read_dataframe(path) for path in args.data_paths)
+            causal_dag = CausalDAG(args.dag_path, ignore_cycles=args.ignore_cycles, datatypes=df.dtypes)
+            causal_tests = causal_dag.generate_causal_tests(
+                threads=args.threads,
+                skip=False,
+            )
+            with open(args.output, "w", encoding="utf-8") as f:
+                json.dump({"tests": [test.to_dict() for test in causal_tests]}, f)
+            logging.info("Causal test generation completed successfully.")
+
+        case Command.DISCOVER:
+            discover_map = {ff.name: ff for ff in entry_points(group="discovery")}
+            if args.technique not in discover_map:
+                raise ValueError(
+                    f"Unsupported technique {args.technique}. Supported: {sorted(discover_map)}. "
+                    "If you have implemented a custom technique, you will need to add this to your entrypoints via "
+                    "your pyproject.toml file."
+                )
+            kwargs = {}
+            for argument in args.technique_kwargs:
+                split = argument.split("=")
+                if len(split) != 2:
+                    raise ValueError(f"Malformed argument {argument}. Should be specified as `arg_name=arg_value`")
+                kwargs[split[0]] = split[1]
+
+            logging.info("Discovering causal structure")
+            # Need to reset index to allow for multiple files having the same index (i.e. starting at zero).
+            # Otherwise you end up with duplicate indices, which causes problems further down the line
+            df = pd.concat([read_dataframe(path) for path in args.data_paths]).reset_index(drop=True)
+            if args.variables:
+                df = df[args.variables]
+            # Drop unnamed columns
+            unnamed_columns = [c for c in df.columns if c.startswith("Unnamed: ")]
+            if unnamed_columns:
+                warn(f"Dropping unnamed columns: {unnamed_columns}")
+            df = df.drop(unnamed_columns, axis=1)
+
+            discover_class = discover_map[args.technique].load()
+            discover = discover_class(
+                df=df,
+                exclude_edges=(
+                    list(nx.nx_pydot.read_dot(args.exclude_edges).edges()) if args.exclude_edges is not None else []
+                ),
+                include_edges=(
+                    list(nx.nx_pydot.read_dot(args.include_edges).edges()) if args.include_edges is not None else []
+                ),
+                alpha=args.alpha,
+                **kwargs,
+            )
+            evolved_dag = discover.discover()
+            if args.output is not None:
+                nx.drawing.nx_pydot.write_dot(evolved_dag, args.output)
+            logging.info("Causal structure discovery completed successfully.")
+        case Command.TEST:
+            # Create and setup framework
+            framework = CausalTestingFramework()
+
+            framework.setup(
+                dag_path=args.dag_path,
+                data_paths=args.data_paths,
+                test_cases_path=args.test_config,
+                query=args.query,
+                ignore_cycles=args.ignore_cycles,
+            )
+
+            logging.info("Running tests")
+            framework.run_tests(silent=args.silent, adequacy=args.adequacy, bootstrap_size=args.bootstrap_size)
+            framework.save_results(args.output)
+
+            logging.info("Causal testing completed successfully.")
+        case Command.EVALUATE:
+            # Create and setup framework
+            framework = CausalTestingFramework()
+            framework.load_data(args.data_paths, query=args.query)
+            framework.load_dag(args.dag_path, args.ignore_cycles)
+            framework.dag.datatypes = framework.df.dtypes
+
+            if args.test_config:
+                framework.load_test_cases_from_json(args.test_config)
+            else:
+                framework.test_cases = framework.dag.generate_causal_tests()
+
+            logging.info("Running tests on entire dataset")
+            results = framework.evaluate_dag(alpha=args.alpha, bootstrap_size=args.bootstrap_size)
+            logging.info("Causal testing completed successfully.")
+            logging.info("Running tests on bootstrap samples")
+            results.to_csv(args.output)
+
+
+if __name__ == "__main__":
+    main()

{causal_testing_framework-14.3.0 → causal_testing_framework-15.0.0}/causal_testing/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '14.3.0'
-__version_tuple__ = version_tuple = (14, 3, 0)
+__version__ = version = '15.0.0'
+__version_tuple__ = version_tuple = (15, 0, 0)
 
-__commit_id__ = commit_id = 'ge6a19c77b'
+__commit_id__ = commit_id = 'g9ee3e7c92'