exadata-validator 0.0.1__py3-none-any.whl

data_validator/__init__.py

@@ -0,0 +1 @@
+"""Standalone validator package."""

data_validator/__main__.py

@@ -0,0 +1,4 @@
+from data_validator.commands import entry
+
+if __name__ == "__main__":
+    entry()

data_validator/commands.py

@@ -0,0 +1,99 @@
+from __future__ import annotations
+
+import logging
+import sys
+from pathlib import Path
+from uuid import uuid4
+
+import click as cl
+
+from data_validator.exceptions import ExitCode
+from data_validator.exceptions import InvalidDatasetError
+from data_validator.exceptions import handle_errors
+from data_validator.reporting import render_report
+from data_validator.usecases import validate_data_model_folder
+
+logging.basicConfig(
+    stream=sys.stderr, level=logging.INFO, format="%(levelname)s: %(message)s"
+)
+LOGGER = logging.getLogger(__name__)
+
+
+@cl.group()
+def cli():
+    """exadata-validator command line interface."""
+
+
+def _format_report_link(output_path: Path) -> str:
+    resolved = output_path.resolve()
+    uri = resolved.as_uri()
+    label = resolved.name
+    if cl.get_text_stream("stdout").isatty():
+        # OSC 8 hyperlink sequence for terminals that support clickable links.
+        return f"\033]8;;{uri}\033\\{label}\033]8;;\033\\"
+    return uri
+
+
+def _default_html_report_path() -> Path:
+    return Path("/tmp") / f"exadata-validator-report-{uuid4().hex}.html"
+
+
+@cli.command("validate-data-model")
+@cl.argument("folder", type=cl.Path(exists=True, file_okay=False, path_type=Path))
+@cl.option(
+    "--threads",
+    type=cl.IntRange(min=1),
+    default=None,
+    help="DuckDB worker threads (defaults to CPU count).",
+)
+@cl.option(
+    "--report-all/--fail-fast",
+    default=True,
+    help="Collect and report all validation errors instead of failing at the first one.",
+)
+@cl.option(
+    "--format",
+    "output_format",
+    type=cl.Choice(["text", "json", "ndjson", "html"], case_sensitive=False),
+    default="text",
+    show_default=True,
+    help="Output format when collecting all validation errors.",
+)
+@cl.option(
+    "--output",
+    "output_path",
+    type=cl.Path(dir_okay=False, writable=True, path_type=Path),
+    default=None,
+    help="Optional output file path when collecting all validation errors.",
+)
+@handle_errors
+def validate_data_model(
+    folder: Path,
+    threads: int | None,
+    report_all: bool,
+    output_format: str,
+    output_path: Path | None,
+):
+    report = validate_data_model_folder(folder, threads=threads, report_all=report_all)
+
+    if report_all:
+        selected_format = output_format.lower()
+        rendered = render_report(report, selected_format)
+        target_output_path = output_path
+        if selected_format == "html" and target_output_path is None:
+            target_output_path = _default_html_report_path()
+
+        if target_output_path:
+            target_output_path.write_text(rendered + "\n", encoding="utf-8")
+            cl.echo(f"Report written to {_format_report_link(target_output_path)}")
+        else:
+            cl.echo(rendered)
+        if report.has_errors:
+            raise SystemExit(ExitCode.FILE_ERROR)
+    elif report.has_errors:
+        raise InvalidDatasetError(report.issues[0].message)
+
+    LOGGER.info("Validation completed successfully for %s", folder)
+
+
+entry = cli

data_validator/dataelements.py

@@ -0,0 +1,175 @@
+import json
+from dataclasses import dataclass
+
+from data_validator.exceptions import InvalidDataModelError
+from data_validator.exceptions import UserInputError
+
+
+@dataclass
+class CommonDataElement:
+    code: str
+    metadata: str
+
+    @classmethod
+    def from_metadata(cls, metadata: dict):
+        code = metadata["code"]
+        if not code.isidentifier():
+            raise UserInputError(f"CDE: {code} is not a valid python identifier")
+
+        validate_metadata(code, metadata)
+        return cls(code=code, metadata=json.dumps(metadata))
+
+    def get_enumerations(self):
+        parsed = json.loads(self.metadata)
+        return parsed["enumerations"] if "enumerations" in parsed else {}
+
+
+def flatten_cdes(schema_data):
+    cdes = []
+
+    if "variables" in schema_data:
+        for metadata in schema_data["variables"]:
+            metadata = reformat_metadata(metadata)
+            cdes.append(CommonDataElement.from_metadata(metadata))
+
+    if "groups" in schema_data:
+        for group_data in schema_data["groups"]:
+            cdes.extend(flatten_cdes(group_data))
+
+    return cdes
+
+
+def get_sql_type_per_column(cdes):
+    return {code: json.loads(cde.metadata)["sql_type"] for code, cde in cdes.items()}
+
+
+def get_cdes_with_min_max(cdes, columns):
+    cdes_with_min_max = {}
+    for code, cde in cdes.items():
+        if code not in columns:
+            continue
+
+        metadata = json.loads(cde.metadata)
+        min_value = metadata.get("min")
+        max_value = metadata.get("max")
+
+        if min_value is not None or max_value is not None:
+            cdes_with_min_max[code] = (min_value, max_value)
+
+    return cdes_with_min_max
+
+
+def get_cdes_with_enumerations(cdes, columns):
+    cdes_with_enumerations = {}
+    for code, cde in cdes.items():
+        if code not in columns:
+            continue
+
+        metadata = json.loads(cde.metadata)
+        if metadata["is_categorical"]:
+            cdes_with_enumerations[code] = list(metadata["enumerations"].keys())
+
+    return cdes_with_enumerations
+
+
+def get_dataset_enums(cdes):
+    return json.loads(cdes["dataset"].metadata)["enumerations"]
+
+
+def validate_dataset_present_on_cdes_with_proper_format(cdes):
+    dataset_cde = [cde for cde in cdes if cde.code == "dataset"]
+    if not dataset_cde:
+        raise InvalidDataModelError("There is no 'dataset' CDE in the data model.")
+
+    dataset_metadata = json.loads(dataset_cde[0].metadata)
+    if not dataset_metadata["is_categorical"]:
+        raise InvalidDataModelError(
+            "CDE 'dataset' must have the 'isCategorical' property equal to 'true'."
+        )
+
+    if dataset_metadata["sql_type"] != "text":
+        raise InvalidDataModelError(
+            "CDE 'dataset' must have the 'sql_type' property equal to 'text'."
+        )
+
+
+def validate_longitudinal_data_model(cdes):
+    subject_id_metadata = None
+    visit_id_metadata = None
+
+    for cde in cdes:
+        if cde.code == "subjectid":
+            subject_id_metadata = json.loads(cde.metadata)
+        elif cde.code == "visitid":
+            visit_id_metadata = json.loads(cde.metadata)
+
+    if not subject_id_metadata:
+        raise InvalidDataModelError(
+            "There is no 'subjectid' CDE in the longitudinal data model."
+        )
+
+    if not visit_id_metadata:
+        raise InvalidDataModelError(
+            "There is no 'visitid' CDE in the longitudinal data model."
+        )
+
+    validate_visitid_cde(visit_id_metadata)
+
+
+def validate_visitid_cde(metadata):
+    if not metadata["is_categorical"]:
+        raise InvalidDataModelError(
+            "CDE 'visitid' must have the 'isCategorical' property equal to 'true'."
+        )
+
+    if metadata["sql_type"] != "text":
+        raise InvalidDataModelError(
+            "CDE 'visitid' must have the 'sql_type' property equal to 'text'."
+        )
+
+    if "enumerations" not in metadata:
+        raise InvalidDataModelError(
+            "CDE 'visitid' must contain the 'enumerations' property."
+        )
+
+
+def reformat_metadata(metadata):
+    new_key_assign = {
+        "isCategorical": "is_categorical",
+        "minValue": "min",
+        "maxValue": "max",
+    }
+
+    for old_key, new_key in new_key_assign.items():
+        if old_key in metadata:
+            metadata[new_key] = metadata.pop(old_key)
+
+    if "enumerations" in metadata:
+        metadata["enumerations"] = {
+            enumeration["code"]: enumeration["label"]
+            for enumeration in metadata["enumerations"]
+        }
+
+    return metadata
+
+
+def validate_metadata(code, metadata):
+    for element in ["is_categorical", "code", "sql_type", "label", "type"]:
+        if element not in metadata:
+            raise InvalidDataModelError(
+                f"Element: {element} is missing from the CDE {code}"
+            )
+
+    if metadata["is_categorical"] and "enumerations" not in metadata:
+        raise InvalidDataModelError(
+            f"The CDE {code} has 'is_categorical' set to True but there are no enumerations."
+        )
+
+    if {"min", "max"} <= set(metadata) and metadata["min"] >= metadata["max"]:
+        raise InvalidDataModelError(f"The CDE {code} has min greater than the max.")
+
+    valid_metadata_types = ["nominal", "real", "integer", "text"]
+    if metadata["type"] not in valid_metadata_types:
+        raise InvalidDataModelError(
+            f"The CDE {code} has an 'type' the only valid types are:{valid_metadata_types} "
+        )

data_validator/duckdb_validator.py

@@ -0,0 +1,400 @@
+from __future__ import annotations
+
+import copy
+import difflib
+import os
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Callable
+
+import duckdb
+
+from data_validator.dataelements import flatten_cdes
+from data_validator.dataelements import get_cdes_with_enumerations
+from data_validator.dataelements import get_cdes_with_min_max
+from data_validator.dataelements import get_dataset_enums
+from data_validator.dataelements import get_sql_type_per_column
+from data_validator.exceptions import InvalidDatasetError
+from data_validator.reporting import ValidationIssue
+
+
+def _quote_identifier(value: str) -> str:
+    return '"' + value.replace('"', '""') + '"'
+
+
+def _quote_literal(value: str) -> str:
+    return "'" + value.replace("'", "''") + "'"
+
+
+def _format_in_list(values: list[str]) -> str:
+    return ", ".join(_quote_literal(value) for value in values)
+
+
+def _format_allowed_values(values: list[str], max_items: int = 8) -> str:
+    shown = values[:max_items]
+    formatted = ", ".join(repr(value) for value in shown)
+    if len(values) > max_items:
+        return f"{formatted}, ... (total={len(values)})"
+    return formatted
+
+
+@dataclass(frozen=True)
+class _FusedCheck:
+    key: str
+    rule: str
+    column: str | None
+    condition_sql: str
+    sample_expression_sql: str
+    message_factory: Callable[[int, str | None], str]
+
+
+class DuckDBDatasetValidator:
+    """Validate CSV files against a data model using direct DuckDB CSV queries."""
+
+    def __init__(self, data_model_metadata: dict, threads: int | None = None) -> None:
+        cdes_list = flatten_cdes(copy.deepcopy(data_model_metadata))
+        self._cdes = {cde.code: cde for cde in cdes_list}
+        self._sql_type_per_column = get_sql_type_per_column(self._cdes)
+        self._dataset_enumerations = list(get_dataset_enums(self._cdes).keys())
+        self._longitudinal = bool(data_model_metadata.get("longitudinal", False))
+        self._threads = threads if threads is not None else (os.cpu_count() or 1)
+
+    def validate_csv(
+        self, csv_path: Path, report_all: bool = False
+    ) -> list[ValidationIssue]:
+        issues: list[ValidationIssue] = []
+        conn = duckdb.connect(database=":memory:")
+        try:
+            conn.execute(f"PRAGMA threads={self._threads}")
+
+            columns = self._get_columns(conn, csv_path)
+            issues.extend(self._validate_columns(columns, csv_path))
+
+            if issues and not report_all:
+                raise InvalidDatasetError(issues[0].message)
+
+            fused_checks = self._build_fused_checks(columns)
+            issues.extend(self._run_fused_checks(conn, csv_path, fused_checks))
+
+            if self._longitudinal and {"subjectid", "visitid"} <= set(columns):
+                issues.extend(self._validate_longitudinal_pairs(conn, csv_path))
+        except duckdb.Error as exc:
+            message = f"Unable to validate csv '{csv_path.name}'. {exc}"
+            if report_all:
+                issues.append(
+                    ValidationIssue(
+                        rule="csv.duckdb_error",
+                        message=message,
+                        file=str(csv_path),
+                    )
+                )
+                return issues
+            raise InvalidDatasetError(message) from exc
+        finally:
+            conn.close()
+
+        if issues and not report_all:
+            raise InvalidDatasetError(issues[0].message)
+        return issues
+
+    def _get_columns(
+        self, conn: duckdb.DuckDBPyConnection, csv_path: Path
+    ) -> list[str]:
+        info = conn.execute(
+            (
+                "DESCRIBE SELECT * "
+                "FROM read_csv_auto(?, header=true, all_varchar=true, nullstr=[''])"
+            ),
+            [str(csv_path)],
+        ).fetchall()
+        return [str(row[0]) for row in info]
+
+    def _validate_columns(
+        self, columns: list[str], csv_path: Path
+    ) -> list[ValidationIssue]:
+        issues: list[ValidationIssue] = []
+
+        if "dataset" not in columns:
+            issues.append(
+                ValidationIssue(
+                    rule="columns.dataset_required",
+                    message="The 'dataset' column is required to exist in the csv.",
+                    file=str(csv_path),
+                    column="dataset",
+                )
+            )
+
+        unknown_columns = (
+            set(columns) - set(self._sql_type_per_column.keys()) - {"row_id"}
+        )
+        for column in sorted(unknown_columns):
+            suggestion = self._get_column_suggestion(column)
+            message = f"Column '{column}' is not present in the CDEs."
+            if suggestion:
+                message += f" Did you mean '{suggestion}'?"
+            issues.append(
+                ValidationIssue(
+                    rule="columns.unknown",
+                    message=message,
+                    file=str(csv_path),
+                    column=column,
+                )
+            )
+
+        if self._longitudinal:
+            for required in ("subjectid", "visitid"):
+                if required not in columns:
+                    issues.append(
+                        ValidationIssue(
+                            rule="longitudinal.required_column",
+                            message=(
+                                "The "
+                                f"'{required}' column is required for longitudinal data models."
+                            ),
+                            file=str(csv_path),
+                            column=required,
+                        )
+                    )
+
+        return issues
+
+    def _get_column_suggestion(self, unknown_column: str) -> str | None:
+        candidates = [
+            column for column in self._sql_type_per_column.keys() if column != "row_id"
+        ]
+        matches = difflib.get_close_matches(unknown_column, candidates, n=1, cutoff=0.7)
+        return matches[0] if matches else None
+
+    def _build_fused_checks(self, columns: list[str]) -> list[_FusedCheck]:
+        checks: list[_FusedCheck] = []
+
+        type_mapping = {"int": "BIGINT", "real": "DOUBLE"}
+        for column in columns:
+            sql_type = self._sql_type_per_column.get(column)
+            cast_type = type_mapping.get(sql_type)
+            if not cast_type:
+                continue
+
+            quoted_column = _quote_identifier(column)
+            checks.append(
+                _FusedCheck(
+                    key=f"type_{column}",
+                    rule="types.invalid",
+                    column=column,
+                    condition_sql=(
+                        f"{quoted_column} IS NOT NULL "
+                        f"AND TRY_CAST({quoted_column} AS {cast_type}) IS NULL"
+                    ),
+                    sample_expression_sql=quoted_column,
+                    message_factory=lambda count, sample, c=column, t=sql_type: (
+                        f"Column '{c}' has invalid {t} values "
+                        f"(count={count}, example={sample!r})."
+                    ),
+                )
+            )
+
+        cdes_with_min_max = get_cdes_with_min_max(self._cdes, columns)
+        for column, (min_value, max_value) in cdes_with_min_max.items():
+            quoted_column = _quote_identifier(column)
+            if min_value is not None:
+                checks.append(
+                    _FusedCheck(
+                        key=f"min_{column}",
+                        rule="range.min",
+                        column=column,
+                        condition_sql=(
+                            f"{quoted_column} IS NOT NULL "
+                            f"AND TRY_CAST({quoted_column} AS DOUBLE) < {min_value}"
+                        ),
+                        sample_expression_sql=quoted_column,
+                        message_factory=lambda count, sample, c=column, v=min_value: (
+                            f"Column '{c}' has values below minimum {v} "
+                            f"(count={count}, example={sample!r})."
+                        ),
+                    )
+                )
+            if max_value is not None:
+                checks.append(
+                    _FusedCheck(
+                        key=f"max_{column}",
+                        rule="range.max",
+                        column=column,
+                        condition_sql=(
+                            f"{quoted_column} IS NOT NULL "
+                            f"AND TRY_CAST({quoted_column} AS DOUBLE) > {max_value}"
+                        ),
+                        sample_expression_sql=quoted_column,
+                        message_factory=lambda count, sample, c=column, v=max_value: (
+                            f"Column '{c}' has values above maximum {v} "
+                            f"(count={count}, example={sample!r})."
+                        ),
+                    )
+                )
+
+        cdes_with_enumerations = get_cdes_with_enumerations(self._cdes, columns)
+        for column, allowed_values in cdes_with_enumerations.items():
+            if not allowed_values:
+                continue
+            quoted_column = _quote_identifier(column)
+            in_list = _format_in_list(allowed_values)
+            allowed_display = _format_allowed_values(allowed_values)
+            checks.append(
+                _FusedCheck(
+                    key=f"enum_{column}",
+                    rule="enum.invalid",
+                    column=column,
+                    condition_sql=(
+                        f"{quoted_column} IS NOT NULL AND {quoted_column} NOT IN ({in_list})"
+                    ),
+                    sample_expression_sql=quoted_column,
+                    message_factory=lambda count,
+                    sample,
+                    c=column,
+                    allowed=allowed_display: (
+                        f"Column '{c}' has invalid categorical value(s). "
+                        f"Allowed values: [{allowed}]. "
+                        f"Found {count} invalid row(s), example invalid value: {sample!r}."
+                    ),
+                )
+            )
+
+        if "dataset" in columns:
+            in_list = _format_in_list(self._dataset_enumerations)
+            dataset_allowed_display = _format_allowed_values(self._dataset_enumerations)
+            checks.append(
+                _FusedCheck(
+                    key="dataset_enum",
+                    rule="dataset.enum",
+                    column="dataset",
+                    condition_sql=f"dataset IS NOT NULL AND dataset NOT IN ({in_list})",
+                    sample_expression_sql="dataset",
+                    message_factory=lambda count,
+                    sample,
+                    allowed=dataset_allowed_display: (
+                        "Column 'dataset' has value(s) not declared in CDEsMetadata "
+                        f"enumerations. Allowed values: [{allowed}]. "
+                        f"Found {count} invalid row(s), example invalid value: {sample!r}."
+                    ),
+                )
+            )
+
+        if self._longitudinal and "subjectid" in columns:
+            checks.append(
+                _FusedCheck(
+                    key="longitudinal_subjectid_null",
+                    rule="longitudinal.subjectid_null",
+                    column="subjectid",
+                    condition_sql="subjectid IS NULL",
+                    sample_expression_sql=_quote_literal("NULL"),
+                    message_factory=lambda count, _: (
+                        "Column 'subjectid' should never contain null values "
+                        f"(count={count})."
+                    ),
+                )
+            )
+        if self._longitudinal and "visitid" in columns:
+            checks.append(
+                _FusedCheck(
+                    key="longitudinal_visitid_null",
+                    rule="longitudinal.visitid_null",
+                    column="visitid",
+                    condition_sql="visitid IS NULL",
+                    sample_expression_sql=_quote_literal("NULL"),
+                    message_factory=lambda count, _: (
+                        "Column 'visitid' should never contain null values "
+                        f"(count={count})."
+                    ),
+                )
+            )
+
+        return checks
+
+    def _run_fused_checks(
+        self,
+        conn: duckdb.DuckDBPyConnection,
+        csv_path: Path,
+        checks: list[_FusedCheck],
+    ) -> list[ValidationIssue]:
+        if not checks:
+            return []
+
+        projections = []
+        for check in checks:
+            count_alias = _quote_identifier(f"{check.key}__count")
+            sample_alias = _quote_identifier(f"{check.key}__sample")
+            row_alias = _quote_identifier(f"{check.key}__row")
+            projections.append(
+                f"SUM(CASE WHEN {check.condition_sql} THEN 1 ELSE 0 END) AS {count_alias}"
+            )
+            projections.append(
+                (
+                    "MIN(CASE WHEN "
+                    f"{check.condition_sql} THEN {check.sample_expression_sql} "
+                    f"ELSE NULL END) AS {sample_alias}"
+                )
+            )
+            projections.append(
+                f"MIN(CASE WHEN {check.condition_sql} THEN _rownum ELSE NULL END) AS {row_alias}"
+            )
+
+        query = (
+            "WITH csv_data AS ("
+            "  SELECT row_number() OVER () AS _rownum, * "
+            "  FROM read_csv_auto(?, header=true, all_varchar=true, nullstr=[''])"
+            ") "
+            "SELECT " + ", ".join(projections) + " FROM csv_data"
+        )
+        row = conn.execute(query, [str(csv_path)]).fetchone()
+        if row is None:
+            return []
+
+        issues: list[ValidationIssue] = []
+        for index, check in enumerate(checks):
+            count = int(row[index * 3] or 0)
+            sample = row[index * 3 + 1]
+            first_data_row = row[index * 3 + 2]
+            if count <= 0:
+                continue
+            line_number = (
+                int(first_data_row) + 1 if first_data_row is not None else None
+            )
+            message = check.message_factory(count, sample)
+            if line_number is not None:
+                message += f" First seen at line {line_number}."
+            issues.append(
+                ValidationIssue(
+                    rule=check.rule,
+                    message=message,
+                    file=str(csv_path),
+                    column=check.column,
+                    row=line_number,
+                )
+            )
+        return issues
+
+    def _validate_longitudinal_pairs(
+        self, conn: duckdb.DuckDBPyConnection, csv_path: Path
+    ) -> list[ValidationIssue]:
+        duplicates = conn.execute(
+            (
+                "SELECT subjectid, visitid, COUNT(*) AS duplicate_count "
+                "FROM read_csv_auto(?, header=true, all_varchar=true, nullstr=['']) "
+                "GROUP BY subjectid, visitid "
+                "HAVING COUNT(*) > 1 "
+                "LIMIT 5"
+            ),
+            [str(csv_path)],
+        ).fetchall()
+        if not duplicates:
+            return []
+
+        return [
+            ValidationIssue(
+                rule="longitudinal.duplicate_pair",
+                message=(
+                    "Invalid csv: duplicate (visitid, subjectid) pairs detected: "
+                    f"{[(subject, visit) for subject, visit, _ in duplicates]}"
+                ),
+                file=str(csv_path),
+            )
+        ]

data_validator/exceptions.py

@@ -0,0 +1,80 @@
+import sys
+from contextlib import contextmanager
+from enum import IntEnum
+from functools import wraps
+
+
+class DataBaseError(Exception):
+    """Legacy DB error type."""
+
+    def __init__(self, message) -> None:
+        self.message = message
+        super().__init__(message)
+
+
+class UserInputError(Exception):
+    def __init__(self, message) -> None:
+        self.message = message
+        super().__init__(message)
+
+
+class FileContentError(Exception):
+    def __init__(self, message) -> None:
+        self.message = message
+        super().__init__(message)
+
+
+class InvalidDatasetError(Exception):
+    def __init__(self, message) -> None:
+        self.message = message
+        super().__init__(message)
+
+
+class InvalidDataModelError(Exception):
+    def __init__(self, message) -> None:
+        self.message = message
+        super().__init__(message)
+
+
+class ForeignKeyError(Exception):
+    """Legacy FK error type."""
+
+    def __init__(self, message) -> None:
+        self.message = message
+        super().__init__(message)
+
+
+class ExitCode(IntEnum):
+    OK = 0
+    USER_ERROR = 64
+    DB_ERROR = 65
+    FILE_ERROR = 66
+
+
+def handle_errors(func):
+    @contextmanager
+    def _handle_errors():
+        try:
+            yield
+        except UserInputError as exc:
+            print("User input error:\n")
+            print(f"\t{exc.message}")
+            sys.exit(ExitCode.USER_ERROR)
+        except DataBaseError as exc:
+            print("Database error:\n")
+            print(f"\t{exc.message}")
+            sys.exit(ExitCode.DB_ERROR)
+        except (FileContentError, InvalidDatasetError, InvalidDataModelError) as exc:
+            print(f"\nValidation error: {exc.message}")
+            sys.exit(ExitCode.FILE_ERROR)
+        except ForeignKeyError as exc:
+            print("Foreign key error:\n")
+            print(f"\t{exc.message}")
+            sys.exit(ExitCode.USER_ERROR)
+
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+        with _handle_errors():
+            return func(*args, **kwargs)
+
+    return wrapper

data_validator/reporting.py

@@ -0,0 +1,342 @@
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from dataclasses import field
+from datetime import datetime
+from datetime import timezone
+from html import escape
+from pathlib import Path
+
+
+@dataclass
+class ValidationIssue:
+    rule: str
+    message: str
+    file: str | None = None
+    column: str | None = None
+    row: int | None = None
+    severity: str = "error"
+
+    def to_dict(self) -> dict:
+        return {
+            "severity": self.severity,
+            "rule": self.rule,
+            "message": self.message,
+            "file": self.file,
+            "column": self.column,
+            "row": self.row,
+        }
+
+
+@dataclass
+class ValidationReport:
+    folder: str
+    files_checked: list[str] = field(default_factory=list)
+    issues: list[ValidationIssue] = field(default_factory=list)
+    generated_at: str = field(
+        default_factory=lambda: datetime.now(timezone.utc).isoformat()
+    )
+
+    @property
+    def has_errors(self) -> bool:
+        return bool(self.issues)
+
+    def to_dict(self) -> dict:
+        issues_by_file: dict[str, list[dict]] = {}
+        for file_key, grouped_issues in _group_issues_by_file(self):
+            key = file_key if file_key is not None else "__folder__"
+            issues_by_file[key] = [issue.to_dict() for issue in grouped_issues]
+
+        return {
+            "folder": self.folder,
+            "generated_at": self.generated_at,
+            "files_checked": self.files_checked,
+            "error_count": len(self.issues),
+            "issues": [issue.to_dict() for issue in self.issues],
+            "issues_by_file": issues_by_file,
+        }
+
+
+def render_report(report: ValidationReport, output_format: str) -> str:
+    if output_format == "text":
+        return _render_text(report)
+    if output_format == "json":
+        return json.dumps(report.to_dict(), indent=2)
+    if output_format == "ndjson":
+        return _render_ndjson(report)
+    if output_format == "html":
+        return _render_html(report)
+    raise ValueError(f"Unsupported report format: {output_format}")
+
+
+def _group_issues_by_file(
+    report: ValidationReport,
+) -> list[tuple[str | None, list[ValidationIssue]]]:
+    grouped: dict[str | None, list[ValidationIssue]] = {}
+    for issue in report.issues:
+        grouped.setdefault(issue.file, []).append(issue)
+
+    ordered: list[tuple[str | None, list[ValidationIssue]]] = []
+    for file_path in report.files_checked:
+        issues = grouped.pop(file_path, None)
+        if issues:
+            ordered.append((file_path, issues))
+
+    for file_path, issues in grouped.items():
+        ordered.append((file_path, issues))
+
+    return ordered
+
+
+def _render_text(report: ValidationReport) -> str:
+    lines = [
+        f"Validation report for: {report.folder}",
+        f"Files checked: {len(report.files_checked)}",
+        f"Errors: {len(report.issues)}",
+    ]
+
+    grouped_issues = _group_issues_by_file(report)
+    if not grouped_issues:
+        lines.append("No validation errors were found.")
+        return "\n".join(lines)
+
+    index = 1
+    for file_path, issues in grouped_issues:
+        if file_path is None:
+            lines.append("Folder-level issues:")
+        else:
+            lines.append(f"CSV: {file_path}")
+        for issue in issues:
+            location_parts = []
+            if issue.column:
+                location_parts.append(f"column={issue.column}")
+            if issue.row is not None:
+                location_parts.append(f"line={issue.row}")
+            location = ", ".join(location_parts) if location_parts else "global"
+            lines.append(f"{index}. [{issue.rule}] {location}: {issue.message}")
+            index += 1
+    return "\n".join(lines)
+
+
+def _render_ndjson(report: ValidationReport) -> str:
+    lines = [
+        json.dumps(
+            {
+                "type": "summary",
+                "folder": report.folder,
+                "generated_at": report.generated_at,
+                "files_checked": len(report.files_checked),
+                "error_count": len(report.issues),
+            }
+        )
+    ]
+    for issue in report.issues:
+        lines.append(
+            json.dumps(
+                {
+                    "type": "issue",
+                    "folder": report.folder,
+                    **issue.to_dict(),
+                }
+            )
+        )
+    return "\n".join(lines)
+
+
+def _render_html(report: ValidationReport) -> str:
+    grouped_sections: list[str] = []
+    grouped_issues = _group_issues_by_file(report)
+    for file_path, issues in grouped_issues:
+        section_title = "Folder-level issues"
+        title_tooltip = "Folder-level validation issues"
+        if file_path is not None:
+            file_name = Path(file_path).name
+            section_title = file_name
+            title_tooltip = file_path
+
+        rows = []
+        for issue in issues:
+            severity_class = f"severity-{escape(issue.severity).lower()}"
+            rows.append(
+                "<tr>"
+                f"<td><span class='severity {severity_class}'>{escape(issue.severity)}</span></td>"
+                f"<td>{escape(issue.rule)}</td>"
+                f"<td>{escape(issue.column or '')}</td>"
+                f"<td>{'' if issue.row is None else issue.row}</td>"
+                f"<td>{escape(issue.message)}</td>"
+                "</tr>"
+            )
+
+        table_rows = "\n".join(rows)
+        grouped_sections.append(
+            "<section class='file-section'>"
+            "<div class='file-header'>"
+            f"<h2 class='file-title' title='{escape(title_tooltip)}'>{escape(section_title)}</h2>"
+            f"<p class='file-count'>{len(issues)} error(s)</p>"
+            "</div>"
+            "<div class='table-wrap'>"
+            "<table>"
+            "<colgroup>"
+            "<col style='width:90px'>"
+            "<col style='width:150px'>"
+            "<col style='width:22%'>"
+            "<col style='width:70px'>"
+            "<col>"
+            "</colgroup>"
+            "<thead><tr>"
+            "<th>Severity</th><th>Rule</th><th>Column</th><th>Line</th><th>Message</th>"
+            "</tr></thead><tbody>"
+            f"{table_rows}"
+            "</tbody></table></div></section>"
+        )
+
+    sections_html = (
+        "".join(grouped_sections)
+        if grouped_sections
+        else (
+            "<div class='table-wrap'><table><thead><tr>"
+            "<th>Severity</th><th>Rule</th><th>Column</th><th>Line</th><th>Message</th>"
+            "</tr></thead><tbody><tr><td colspan='5'><div class='empty-state'>"
+            "No validation errors were found."
+            "</div></td></tr></tbody></table></div>"
+        )
+    )
+    return (
+        "<!doctype html>"
+        "<html><head><meta charset='utf-8'>"
+        "<meta name='viewport' content='width=device-width, initial-scale=1'>"
+        "<title>Data Validation Report</title>"
+        "<style>"
+        ":root{"
+        "--bg:#f4f7fb;"
+        "--card:#ffffff;"
+        "--text:#12263a;"
+        "--muted:#5a6b7c;"
+        "--border:#d7e0ea;"
+        "--header:#e8eef6;"
+        "--accent:#0c6d9a;"
+        "--error-bg:#ffe9e8;"
+        "--error-text:#8f1e18;"
+        "}"
+        "body{"
+        "margin:0;"
+        "font-family:'Avenir Next','Trebuchet MS','Gill Sans',sans-serif;"
+        "background:linear-gradient(160deg,#f7fbff 0%,#eef4fa 100%);"
+        "color:var(--text);"
+        "}"
+        ".container{"
+        "max-width:1120px;"
+        "margin:32px auto;"
+        "padding:0 20px;"
+        "}"
+        ".card{"
+        "background:var(--card);"
+        "border:1px solid var(--border);"
+        "border-radius:16px;"
+        "box-shadow:0 12px 30px rgba(8,39,64,.08);"
+        "overflow:hidden;"
+        "}"
+        ".hero{"
+        "padding:24px 28px 18px 28px;"
+        "background:linear-gradient(135deg,#edf4fb 0%,#e4edf7 100%);"
+        "border-bottom:1px solid var(--border);"
+        "}"
+        ".title{"
+        "margin:0;"
+        "font-size:28px;"
+        "letter-spacing:.3px;"
+        "}"
+        ".subtitle{"
+        "margin:6px 0 0 0;"
+        "color:var(--muted);"
+        "font-size:14px;"
+        "}"
+        ".summary{"
+        "display:grid;"
+        "grid-template-columns:repeat(auto-fit,minmax(180px,1fr));"
+        "gap:12px;"
+        "padding:18px 24px 22px 24px;"
+        "}"
+        ".stat{"
+        "background:#f8fbff;"
+        "border:1px solid var(--border);"
+        "border-radius:12px;"
+        "padding:12px 14px;"
+        "}"
+        ".stat-label{"
+        "margin:0 0 6px 0;"
+        "font-size:12px;"
+        "text-transform:uppercase;"
+        "letter-spacing:.08em;"
+        "color:var(--muted);"
+        "}"
+        ".stat-value{"
+        "margin:0;"
+        "font-size:16px;"
+        "font-weight:600;"
+        "line-height:1.3;"
+        "word-break:break-word;"
+        "}"
+        ".file-section{margin:0 16px 18px 16px;border:1px solid var(--border);border-radius:12px;overflow:hidden;background:#fff;}"
+        ".file-header{display:flex;justify-content:space-between;gap:12px;align-items:center;padding:12px 14px;background:#f3f8fe;border-bottom:1px solid var(--border);}"
+        ".file-title{margin:0;font-size:16px;font-weight:700;cursor:help;word-break:break-word;}"
+        ".file-count{margin:0;font-size:12px;color:var(--muted);font-weight:600;text-transform:uppercase;letter-spacing:.06em;}"
+        ".table-wrap{padding:0;overflow-x:auto;}"
+        "table{border-collapse:collapse;width:100%;min-width:920px;background:#fff;table-layout:fixed;}"
+        "th,td{padding:10px 12px;text-align:left;vertical-align:top;border-bottom:1px solid var(--border);}"
+        "th{"
+        "background:var(--header);"
+        "font-size:12px;"
+        "text-transform:uppercase;"
+        "letter-spacing:.06em;"
+        "color:#27445d;"
+        "position:sticky;"
+        "top:0;"
+        "}"
+        "tbody tr:nth-child(even){background:#fbfdff;}"
+        "tbody tr:hover{background:#f2f8fe;}"
+        ".severity{"
+        "display:inline-block;"
+        "padding:3px 8px;"
+        "border-radius:999px;"
+        "font-size:11px;"
+        "font-weight:700;"
+        "text-transform:uppercase;"
+        "letter-spacing:.05em;"
+        "}"
+        ".severity-error{background:var(--error-bg);color:var(--error-text);}"
+        ".empty-state{padding:18px 0;color:var(--muted);font-style:italic;}"
+        "@media (max-width:700px){"
+        ".container{margin:18px auto;padding:0 10px;}"
+        ".hero{padding:18px 16px;}"
+        ".summary{padding:12px 14px 16px 14px;}"
+        ".title{font-size:22px;}"
+        "}"
+        "</style></head><body>"
+        "<div class='container'><section class='card'>"
+        "<header class='hero'>"
+        "<h1 class='title'>Data Validation Report</h1>"
+        "<p class='subtitle'>Generated from exadata-validator</p>"
+        "</header>"
+        "<section class='summary'>"
+        "<article class='stat'>"
+        "<p class='stat-label'>Folder</p>"
+        f"<p class='stat-value'>{escape(report.folder)}</p>"
+        "</article>"
+        "<article class='stat'>"
+        "<p class='stat-label'>Generated</p>"
+        f"<p class='stat-value'>{escape(report.generated_at)}</p>"
+        "</article>"
+        "<article class='stat'>"
+        "<p class='stat-label'>Files Checked</p>"
+        f"<p class='stat-value'>{len(report.files_checked)}</p>"
+        "</article>"
+        "<article class='stat'>"
+        "<p class='stat-label'>Errors</p>"
+        f"<p class='stat-value'>{len(report.issues)}</p>"
+        "</article>"
+        "</section>"
+        f"{sections_html}"
+        "</section></div></body></html>"
+    )

data_validator/usecases.py

@@ -0,0 +1,159 @@
+from __future__ import annotations
+
+import copy
+import json
+from pathlib import Path
+
+import duckdb
+
+from data_validator.dataelements import flatten_cdes
+from data_validator.dataelements import (
+    validate_dataset_present_on_cdes_with_proper_format,
+)
+from data_validator.dataelements import validate_longitudinal_data_model
+from data_validator.duckdb_validator import DuckDBDatasetValidator
+from data_validator.exceptions import FileContentError
+from data_validator.exceptions import InvalidDataModelError
+from data_validator.exceptions import InvalidDatasetError
+from data_validator.exceptions import UserInputError
+from data_validator.reporting import ValidationIssue
+from data_validator.reporting import ValidationReport
+
+LONGITUDINAL = "longitudinal"
+
+
+def _quote_literal(value: str) -> str:
+    return "'" + value.replace("'", "''") + "'"
+
+
+def _read_json_file(path: Path) -> dict:
+    with path.open("r", encoding="utf-8") as stream:
+        try:
+            return json.load(stream)
+        except json.JSONDecodeError as exc:
+            raise FileContentError(
+                f"Unable to decode json file. {exc.args[0]}"
+            ) from exc
+
+
+def validate_data_model_metadata(data_model_metadata: dict) -> None:
+    if "version" not in data_model_metadata:
+        raise UserInputError("You need to include a version on the CDEsMetadata.json")
+
+    cdes = flatten_cdes(copy.deepcopy(data_model_metadata))
+    validate_dataset_present_on_cdes_with_proper_format(cdes)
+
+    if LONGITUDINAL in data_model_metadata:
+        longitudinal = data_model_metadata[LONGITUDINAL]
+        if not isinstance(longitudinal, bool):
+            raise UserInputError(
+                f"Longitudinal flag should be boolean, value given: {longitudinal}"
+            )
+        if longitudinal:
+            validate_longitudinal_data_model(cdes)
+
+
+def _validate_dataset_uniqueness_with_sql(
+    csv_files: list[Path],
+) -> list[ValidationIssue]:
+    if len(csv_files) < 2:
+        return []
+
+    file_list_sql = ", ".join(_quote_literal(str(path)) for path in csv_files)
+    query = (
+        "WITH csv_rows AS ("
+        "  SELECT filename, dataset "
+        f"  FROM read_csv_auto([{file_list_sql}], "
+        "       header=true, all_varchar=true, nullstr=[''], "
+        "       filename=true, union_by_name=true)"
+        "), normalized AS ("
+        "  SELECT filename, NULLIF(LOWER(TRIM(dataset)), '') AS dataset_normalized "
+        "  FROM csv_rows "
+        "  WHERE dataset IS NOT NULL"
+        ") "
+        "SELECT dataset_normalized, list(DISTINCT filename) AS files "
+        "FROM normalized "
+        "WHERE dataset_normalized IS NOT NULL "
+        "GROUP BY dataset_normalized "
+        "HAVING COUNT(DISTINCT filename) > 1 "
+        "ORDER BY dataset_normalized"
+    )
+
+    conn = duckdb.connect(database=":memory:")
+    try:
+        rows = conn.execute(query).fetchall()
+    except duckdb.Error as exc:
+        raise InvalidDatasetError(
+            f"Unable to validate folder-level dataset uniqueness: {exc}"
+        ) from exc
+    finally:
+        conn.close()
+
+    issues: list[ValidationIssue] = []
+    for dataset_normalized, files in rows:
+        file_names = sorted(Path(str(path)).name for path in files)
+        issues.append(
+            ValidationIssue(
+                rule="folder.dataset_uniqueness",
+                message=(
+                    "Dataset code collision after normalization (trim+lower) for "
+                    f"'{dataset_normalized}' across files: {file_names}"
+                ),
+            )
+        )
+    return issues
+
+
+def validate_data_model_folder(
+    folder: Path, *, threads: int | None = None, report_all: bool = False
+) -> ValidationReport:
+    report = ValidationReport(folder=str(folder))
+
+    metadata_path = folder / "CDEsMetadata.json"
+    if not metadata_path.exists():
+        raise UserInputError(f"Missing metadata file: {metadata_path}")
+
+    data_model_metadata = _read_json_file(metadata_path)
+    try:
+        validate_data_model_metadata(data_model_metadata)
+    except (InvalidDataModelError, UserInputError) as exc:
+        if not report_all:
+            raise
+        report.issues.append(
+            ValidationIssue(
+                rule="metadata.invalid",
+                message=str(exc),
+                file=str(metadata_path),
+            )
+        )
+        return report
+
+    csv_files = sorted(folder.glob("*.csv"))
+    if not csv_files:
+        raise UserInputError(
+            f"No CSV files found in {folder}. Expected at least one dataset csv."
+        )
+    report.files_checked = [str(csv_file) for csv_file in csv_files]
+
+    validator = DuckDBDatasetValidator(data_model_metadata, threads=threads)
+    for csv_file in csv_files:
+        csv_issues = validator.validate_csv(csv_file, report_all=report_all)
+        report.issues.extend(csv_issues)
+
+    try:
+        duplicate_issues = _validate_dataset_uniqueness_with_sql(csv_files)
+        report.issues.extend(duplicate_issues)
+    except InvalidDatasetError as exc:
+        if not report_all:
+            raise
+        report.issues.append(
+            ValidationIssue(
+                rule="folder.dataset_uniqueness",
+                message=str(exc),
+            )
+        )
+
+    if report.issues and not report_all:
+        raise InvalidDatasetError(report.issues[0].message)
+
+    return report

exadata_validator-0.0.1.dist-info/METADATA

@@ -0,0 +1,61 @@
+Metadata-Version: 2.4
+Name: exadata-validator
+Version: 0.0.1
+Summary: Validate data-model folders using DuckDB.
+Author: Exaflow Team
+Requires-Python: >=3.10,<3.11
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Requires-Dist: click (>=8.1,<8.2)
+Requires-Dist: duckdb (>=1.1,<1.2)
+Description-Content-Type: text/markdown
+
+# exadata-validator
+
+`exadata-validator` validates data-model folders using DuckDB.
+
+## Install With pip
+
+```bash
+python -m venv .venv
+source .venv/bin/activate
+pip install exadata-validator
+```
+
+Validate a data model folder:
+
+```bash
+exadata-validator validate-data-model /path/to/data_model_folder
+```
+
+## Develop With Poetry
+
+From the repository root:
+
+```bash
+cd data-validator/exaflow-data-validator
+poetry install
+```
+
+Then run the same CLI through Poetry:
+
+```bash
+poetry run exadata-validator validate-data-model /path/to/data_model_folder
+```
+
+Use `exadata-validator validate-data-model --help` for reporting, output, and threading options.
+
+## Folder Layout
+
+```text
+/path/to/data_model_folder/
+  CDEsMetadata.json
+  dataset1.csv
+  dataset2.csv
+```
+
+## Validation Notes
+
+- CSV validation queries files directly with DuckDB and uses fused aggregate checks to reduce scan overhead.
+- Folder-level dataset uniqueness is enforced across all CSV files via SQL using normalized codes (`trim + lower`).
+

exadata_validator-0.0.1.dist-info/RECORD

@@ -0,0 +1,12 @@
+data_validator/__init__.py,sha256=Tu8Zffh4uJ1ryN2xbmt6NH3Rmtq7iCHkvurX-bWo8uY,36
+data_validator/__main__.py,sha256=roEHUIHJaAQBK-dtqzNdaE9kUG6qaC-ZOoI4M-vuA8A,82
+data_validator/commands.py,sha256=PHWSAfGllMCmK1xsT7oUy8v8jTcVGke6gp79kcWIt-c,2971
+data_validator/dataelements.py,sha256=sElw3t-MvgeTwqC5G0RGFuFioTYIiiu6eKAZkCqEA-0,5392
+data_validator/duckdb_validator.py,sha256=OBqK8ATi4zsAl7g1idv2CBlOLdDvRCtZciq9isdF8mc,15446
+data_validator/exceptions.py,sha256=hEWv45Y_eIPJKwIuK3HJ9Cefay4PINfVcUfjgMFiODU,2030
+data_validator/reporting.py,sha256=pdeJ-ad_fOuYKt1jZqY0iFD433UWh8I5w5m011n6Moo,11622
+data_validator/usecases.py,sha256=NGvAmnxngFiMiiCY_3bz2zOQWAT1Oa3HO-F5ALm4H9o,5353
+exadata_validator-0.0.1.dist-info/METADATA,sha256=xTI6S5nld1kObK9jyCjFSIVLcQ3FI8aPPVzED-UjuP8,1379
+exadata_validator-0.0.1.dist-info/WHEEL,sha256=kJCRJT_g0adfAJzTx2GUMmS80rTJIVHRCfG0DQgLq3o,88
+exadata_validator-0.0.1.dist-info/entry_points.txt,sha256=50_GAf8IaFjGTrft9Iy0tso4G3v-R2ITd4C8Sf7gmMI,67
+exadata_validator-0.0.1.dist-info/RECORD,,

exadata_validator-0.0.1.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: poetry-core 2.3.1
+Root-Is-Purelib: true
+Tag: py3-none-any

exadata_validator-0.0.1.dist-info/entry_points.txt

@@ -0,0 +1,3 @@
+[console_scripts]
+exadata-validator=data_validator.commands:entry
+