dataleaks 0.1.0__py3-none-any.whl

dataleaks/__init__.py

@@ -0,0 +1,3 @@
+from dataleaks.api import DataLeaks
+
+__all__ = ["DataLeaks"]

dataleaks/api.py

@@ -0,0 +1,67 @@
+from __future__ import annotations
+
+from typing import Any
+
+from dataleaks.engine.aggregator import FindingAggregator
+from dataleaks.engine.defaults import build_default_registry
+from dataleaks.engine.registry import DetectorRegistry
+from dataleaks.engine.runner import DetectorRunner
+from dataleaks.reporting.report import LeakageReport
+from dataleaks.schemas.config import DataLeaksConfig
+from dataleaks.schemas.dataset import DatasetContext
+from dataleaks.schemas.finding import Finding
+
+
+class DataLeaks:
+    """Public API for running DataLeaks analysis."""
+
+    def __init__(
+        self,
+        data,
+        *,
+        target: str | None = None,
+        train=None,
+        validation=None,
+        test=None,
+        config: DataLeaksConfig | None = None,
+        metadata: dict[str, Any] | None = None,
+        registry: DetectorRegistry | None = None,
+    ) -> None:
+        self.context = DatasetContext(
+            data=data,
+            target=target,
+            train=train if train is not None else data,
+            validation=validation,
+            test=test,
+            metadata=metadata or {},
+        )
+
+        self.config = config or DataLeaksConfig()
+
+        if registry is None:
+            self.registry = build_default_registry(
+                self.context,
+                self.config,
+            )
+        else:
+            self.registry = registry
+
+    def run(self) -> LeakageReport:
+        """Run detectors, aggregate findings, and build a report."""
+
+        runner = DetectorRunner(self.registry)
+
+        findings: list[Finding] = runner.run(
+            self.context
+        )
+
+        findings = FindingAggregator().aggregate(
+            findings
+        )
+
+        return LeakageReport.from_findings(
+            findings,
+            metadata=self.context.metadata,
+            execution=runner.execution,
+            schema_validation=self.context.schema_validation,
+        )

dataleaks/cli.py

@@ -0,0 +1,276 @@
+from __future__ import annotations
+
+import argparse
+import ast
+import sys
+
+import pandas as pd
+
+from dataleaks.api import DataLeaks
+from dataleaks.reporting.console import ConsoleReporter
+from dataleaks.reporting.json import JSONReporter
+
+
+def build_parser() -> argparse.ArgumentParser:
+    """Build the DataLeaks command-line argument parser."""
+
+    parser = argparse.ArgumentParser(
+        prog="dataleaks",
+        description="Detect data leakage in machine learning datasets.",
+    )
+
+    parser.add_argument(
+        "file",
+        help="Path to the training CSV dataset.",
+    )
+
+    parser.add_argument(
+        "--target",
+        help="Target column used for leakage analysis.",
+        default=None,
+    )
+
+    parser.add_argument(
+        "--test",
+        help="Path to the test CSV dataset.",
+        default=None,
+    )
+
+    parser.add_argument(
+        "--validation",
+        help="Path to the validation CSV dataset.",
+        default=None,
+    )
+
+    parser.add_argument(
+        "--time-column",
+        help=(
+            "Time column used for chronological train/test "
+            "split validation."
+        ),
+        default=None,
+    )
+
+    parser.add_argument(
+        "--prediction-time-column",
+        help=(
+            "Prediction timestamp column used to detect "
+            "future feature timestamps."
+        ),
+        default=None,
+    )
+
+    parser.add_argument(
+        "--feature-time-columns",
+        nargs="+",
+        help=(
+            "One or more feature timestamp columns to compare "
+            "against the prediction timestamp."
+        ),
+        default=None,
+    )
+
+    parser.add_argument(
+        "--conditional-time-column",
+        help=(
+            "Temporal column whose expected presence depends "
+            "on a target condition."
+        ),
+        default=None,
+    )
+
+    parser.add_argument(
+        "--conditional-target-column",
+        help=(
+            "Target column controlling conditional temporal "
+            "missingness."
+        ),
+        default=None,
+    )
+
+    parser.add_argument(
+        "--conditional-present-when",
+        help=(
+            "Target value for which the conditional temporal "
+            "column is expected to be present."
+        ),
+        default=None,
+    )
+
+    parser.add_argument(
+        "--output",
+        choices=["console", "json"],
+        default="console",
+        help="Output format. Defaults to console.",
+    )
+
+    return parser
+
+
+def _parse_cli_value(value: str) -> object:
+    """Parse a CLI scalar into a Python value when possible."""
+
+    try:
+        return ast.literal_eval(value)
+    except (ValueError, SyntaxError):
+        return value
+
+
+def _build_temporal_metadata(
+    *,
+    time_column: str | None,
+    prediction_time_column: str | None,
+    feature_time_columns: list[str] | None,
+    conditional_time_column: str | None = None,
+    conditional_target_column: str | None = None,
+    conditional_present_when: str | None = None,
+) -> dict:
+    """Build temporal metadata from CLI arguments."""
+
+    temporal: dict[str, object] = {}
+
+    if time_column is not None:
+        temporal["time_column"] = time_column
+
+    if prediction_time_column is not None:
+        temporal["prediction_time_column"] = (
+            prediction_time_column
+        )
+
+    if feature_time_columns is not None:
+        temporal["feature_time_columns"] = (
+            feature_time_columns
+        )
+
+    conditional_args = (
+        conditional_time_column,
+        conditional_target_column,
+        conditional_present_when,
+    )
+
+    conditional_count = sum(
+        value is not None
+        for value in conditional_args
+    )
+
+    if conditional_count not in (0, 3):
+        raise ValueError(
+            "--conditional-time-column, "
+            "--conditional-target-column, and "
+            "--conditional-present-when must be provided together"
+        )
+
+    if conditional_count == 3:
+        temporal["conditional_columns"] = {
+            conditional_time_column: {
+                "target_column": conditional_target_column,
+                "present_when": _parse_cli_value(
+                    conditional_present_when
+                ),
+            }
+        }
+
+    return temporal
+
+
+def main() -> int:
+    """Run the DataLeaks CLI."""
+
+    parser = build_parser()
+    args = parser.parse_args()
+
+    try:
+        data = pd.read_csv(args.file)
+
+        test = (
+            pd.read_csv(args.test)
+            if args.test is not None
+            else None
+        )
+
+        validation = (
+            pd.read_csv(args.validation)
+            if args.validation is not None
+            else None
+        )
+
+        temporal_metadata = _build_temporal_metadata(
+            time_column=args.time_column,
+            prediction_time_column=(
+                args.prediction_time_column
+            ),
+            feature_time_columns=args.feature_time_columns,
+            conditional_time_column=(
+                args.conditional_time_column
+            ),
+            conditional_target_column=(
+                args.conditional_target_column
+            ),
+            conditional_present_when=(
+                args.conditional_present_when
+            ),
+        )
+
+        metadata = {}
+
+        if temporal_metadata:
+            metadata["temporal"] = temporal_metadata
+
+        report = DataLeaks(
+            data,
+            target=args.target,
+            test=test,
+            validation=validation,
+            metadata=metadata,
+        ).run()
+
+        if args.output == "json":
+            output = JSONReporter().render(report)
+        else:
+            output = ConsoleReporter().render(report)
+
+        print(output)
+        return 0
+
+    except FileNotFoundError as exc:
+        missing_file = args.file
+
+        if args.test is not None and args.test in str(exc):
+            missing_file = args.test
+
+        elif (
+            args.validation is not None
+            and args.validation in str(exc)
+        ):
+            missing_file = args.validation
+
+        print(
+            f"Error: dataset file not found: {missing_file}",
+            file=sys.stderr,
+        )
+        return 1
+
+    except pd.errors.EmptyDataError as exc:
+        print(
+            f"Error: dataset file is empty: {exc}",
+            file=sys.stderr,
+        )
+        return 1
+
+    except pd.errors.ParserError as exc:
+        print(
+            f"Error: could not parse CSV file: {exc}",
+            file=sys.stderr,
+        )
+        return 1
+
+    except (TypeError, ValueError) as exc:
+        print(
+            f"Error: {exc}",
+            file=sys.stderr,
+        )
+        return 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

dataleaks/detectors/cross_dataset/overlap.py

@@ -0,0 +1,137 @@
+from __future__ import annotations
+
+import pandas as pd
+
+from dataleaks.engine.detector import BaseDetector
+from dataleaks.schemas.dataset import DatasetContext
+from dataleaks.schemas.finding import Finding
+
+
+class CrossDatasetOverlapDetector(BaseDetector):
+    """Detect exact overlap between the current dataset and an external dataset."""
+
+    name = "cross_dataset_overlap"
+    category = "cross_dataset_leakage"
+
+    def __init__(
+        self,
+        reference_data: pd.DataFrame,
+        columns: list[str] | None = None,
+    ) -> None:
+        if not isinstance(reference_data, pd.DataFrame):
+            raise TypeError(
+                "reference_data must be a pandas DataFrame"
+            )
+
+        if columns is not None and not columns:
+            raise ValueError(
+                "columns must contain at least one column"
+            )
+
+        self.reference_data = reference_data
+        self.columns = columns
+
+    def detect(self, context: DatasetContext) -> list[Finding]:
+        data = context.data
+        reference = self.reference_data
+
+        if data.empty or reference.empty:
+            return []
+
+        columns = self._resolve_columns(
+            data,
+            reference,
+        )
+
+        if not columns:
+            return []
+
+        current_rows = (
+            data[columns]
+            .drop_duplicates()
+            .reset_index(drop=True)
+        )
+
+        reference_rows = (
+            reference[columns]
+            .drop_duplicates()
+            .reset_index(drop=True)
+        )
+
+        current_index = pd.MultiIndex.from_frame(current_rows)
+        reference_index = pd.MultiIndex.from_frame(reference_rows)
+
+        overlap = current_index.intersection(reference_index)
+        overlap_count = len(overlap)
+
+        if overlap_count == 0:
+            return []
+
+        current_unique_count = len(current_rows)
+
+        overlap_ratio = (
+            overlap_count / current_unique_count
+            if current_unique_count
+            else 0.0
+        )
+
+        if overlap_ratio >= 0.5:
+            severity = "critical"
+        elif overlap_ratio >= 0.1:
+            severity = "high"
+        else:
+            severity = "medium"
+
+        return [
+            Finding(
+                detector=self.name,
+                category=self.category,
+                severity=severity,
+                confidence=overlap_ratio,
+                explanation=(
+                    f"{overlap_count} unique row(s) from the current "
+                    "dataset also occur in the reference dataset."
+                ),
+                recommendation=(
+                    "Verify whether the reference dataset was used during "
+                    "training, preprocessing, augmentation, or evaluation. "
+                    "Ensure that shared observations cannot transfer "
+                    "target or evaluation information into the model."
+                ),
+                affected_columns=columns,
+                evidence={
+                    "type": "cross_dataset_exact_overlap",
+                    "overlap_count": overlap_count,
+                    "current_unique_rows": current_unique_count,
+                    "reference_unique_rows": len(reference_rows),
+                    "overlap_ratio": overlap_ratio,
+                    "columns": columns,
+                },
+            )
+        ]
+
+    def _resolve_columns(
+        self,
+        data: pd.DataFrame,
+        reference: pd.DataFrame,
+    ) -> list[str]:
+        if self.columns is not None:
+            missing = [
+                column
+                for column in self.columns
+                if column not in data.columns
+                or column not in reference.columns
+            ]
+
+            if missing:
+                raise ValueError(
+                    f"Configured overlap columns are missing: {missing}"
+                )
+
+            return self.columns
+
+        return [
+            column
+            for column in data.columns
+            if column in reference.columns
+        ]