exadata-validator 0.0.1__tar.gz

exadata_validator-0.0.1/PKG-INFO

@@ -0,0 +1,61 @@
+Metadata-Version: 2.4
+Name: exadata-validator
+Version: 0.0.1
+Summary: Validate data-model folders using DuckDB.
+Author: Exaflow Team
+Requires-Python: >=3.10,<3.11
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Requires-Dist: click (>=8.1,<8.2)
+Requires-Dist: duckdb (>=1.1,<1.2)
+Description-Content-Type: text/markdown
+
+# exadata-validator
+
+`exadata-validator` validates data-model folders using DuckDB.
+
+## Install With pip
+
+```bash
+python -m venv .venv
+source .venv/bin/activate
+pip install exadata-validator
+```
+
+Validate a data model folder:
+
+```bash
+exadata-validator validate-data-model /path/to/data_model_folder
+```
+
+## Develop With Poetry
+
+From the repository root:
+
+```bash
+cd data-validator/exaflow-data-validator
+poetry install
+```
+
+Then run the same CLI through Poetry:
+
+```bash
+poetry run exadata-validator validate-data-model /path/to/data_model_folder
+```
+
+Use `exadata-validator validate-data-model --help` for reporting, output, and threading options.
+
+## Folder Layout
+
+```text
+/path/to/data_model_folder/
+  CDEsMetadata.json
+  dataset1.csv
+  dataset2.csv
+```
+
+## Validation Notes
+
+- CSV validation queries files directly with DuckDB and uses fused aggregate checks to reduce scan overhead.
+- Folder-level dataset uniqueness is enforced across all CSV files via SQL using normalized codes (`trim + lower`).
+

exadata_validator-0.0.1/README.md

@@ -0,0 +1,48 @@
+# exadata-validator
+
+`exadata-validator` validates data-model folders using DuckDB.
+
+## Install With pip
+
+```bash
+python -m venv .venv
+source .venv/bin/activate
+pip install exadata-validator
+```
+
+Validate a data model folder:
+
+```bash
+exadata-validator validate-data-model /path/to/data_model_folder
+```
+
+## Develop With Poetry
+
+From the repository root:
+
+```bash
+cd data-validator/exaflow-data-validator
+poetry install
+```
+
+Then run the same CLI through Poetry:
+
+```bash
+poetry run exadata-validator validate-data-model /path/to/data_model_folder
+```
+
+Use `exadata-validator validate-data-model --help` for reporting, output, and threading options.
+
+## Folder Layout
+
+```text
+/path/to/data_model_folder/
+  CDEsMetadata.json
+  dataset1.csv
+  dataset2.csv
+```
+
+## Validation Notes
+
+- CSV validation queries files directly with DuckDB and uses fused aggregate checks to reduce scan overhead.
+- Folder-level dataset uniqueness is enforced across all CSV files via SQL using normalized codes (`trim + lower`).

exadata_validator-0.0.1/data_validator/__init__.py

@@ -0,0 +1 @@
+"""Standalone validator package."""

exadata_validator-0.0.1/data_validator/__main__.py

@@ -0,0 +1,4 @@
+from data_validator.commands import entry
+
+if __name__ == "__main__":
+    entry()

exadata_validator-0.0.1/data_validator/commands.py

@@ -0,0 +1,99 @@
+from __future__ import annotations
+
+import logging
+import sys
+from pathlib import Path
+from uuid import uuid4
+
+import click as cl
+
+from data_validator.exceptions import ExitCode
+from data_validator.exceptions import InvalidDatasetError
+from data_validator.exceptions import handle_errors
+from data_validator.reporting import render_report
+from data_validator.usecases import validate_data_model_folder
+
+logging.basicConfig(
+    stream=sys.stderr, level=logging.INFO, format="%(levelname)s: %(message)s"
+)
+LOGGER = logging.getLogger(__name__)
+
+
+@cl.group()
+def cli():
+    """exadata-validator command line interface."""
+
+
+def _format_report_link(output_path: Path) -> str:
+    resolved = output_path.resolve()
+    uri = resolved.as_uri()
+    label = resolved.name
+    if cl.get_text_stream("stdout").isatty():
+        # OSC 8 hyperlink sequence for terminals that support clickable links.
+        return f"\033]8;;{uri}\033\\{label}\033]8;;\033\\"
+    return uri
+
+
+def _default_html_report_path() -> Path:
+    return Path("/tmp") / f"exadata-validator-report-{uuid4().hex}.html"
+
+
+@cli.command("validate-data-model")
+@cl.argument("folder", type=cl.Path(exists=True, file_okay=False, path_type=Path))
+@cl.option(
+    "--threads",
+    type=cl.IntRange(min=1),
+    default=None,
+    help="DuckDB worker threads (defaults to CPU count).",
+)
+@cl.option(
+    "--report-all/--fail-fast",
+    default=True,
+    help="Collect and report all validation errors instead of failing at the first one.",
+)
+@cl.option(
+    "--format",
+    "output_format",
+    type=cl.Choice(["text", "json", "ndjson", "html"], case_sensitive=False),
+    default="text",
+    show_default=True,
+    help="Output format when collecting all validation errors.",
+)
+@cl.option(
+    "--output",
+    "output_path",
+    type=cl.Path(dir_okay=False, writable=True, path_type=Path),
+    default=None,
+    help="Optional output file path when collecting all validation errors.",
+)
+@handle_errors
+def validate_data_model(
+    folder: Path,
+    threads: int | None,
+    report_all: bool,
+    output_format: str,
+    output_path: Path | None,
+):
+    report = validate_data_model_folder(folder, threads=threads, report_all=report_all)
+
+    if report_all:
+        selected_format = output_format.lower()
+        rendered = render_report(report, selected_format)
+        target_output_path = output_path
+        if selected_format == "html" and target_output_path is None:
+            target_output_path = _default_html_report_path()
+
+        if target_output_path:
+            target_output_path.write_text(rendered + "\n", encoding="utf-8")
+            cl.echo(f"Report written to {_format_report_link(target_output_path)}")
+        else:
+            cl.echo(rendered)
+        if report.has_errors:
+            raise SystemExit(ExitCode.FILE_ERROR)
+    elif report.has_errors:
+        raise InvalidDatasetError(report.issues[0].message)
+
+    LOGGER.info("Validation completed successfully for %s", folder)
+
+
+entry = cli

exadata_validator-0.0.1/data_validator/dataelements.py

@@ -0,0 +1,175 @@
+import json
+from dataclasses import dataclass
+
+from data_validator.exceptions import InvalidDataModelError
+from data_validator.exceptions import UserInputError
+
+
+@dataclass
+class CommonDataElement:
+    code: str
+    metadata: str
+
+    @classmethod
+    def from_metadata(cls, metadata: dict):
+        code = metadata["code"]
+        if not code.isidentifier():
+            raise UserInputError(f"CDE: {code} is not a valid python identifier")
+
+        validate_metadata(code, metadata)
+        return cls(code=code, metadata=json.dumps(metadata))
+
+    def get_enumerations(self):
+        parsed = json.loads(self.metadata)
+        return parsed["enumerations"] if "enumerations" in parsed else {}
+
+
+def flatten_cdes(schema_data):
+    cdes = []
+
+    if "variables" in schema_data:
+        for metadata in schema_data["variables"]:
+            metadata = reformat_metadata(metadata)
+            cdes.append(CommonDataElement.from_metadata(metadata))
+
+    if "groups" in schema_data:
+        for group_data in schema_data["groups"]:
+            cdes.extend(flatten_cdes(group_data))
+
+    return cdes
+
+
+def get_sql_type_per_column(cdes):
+    return {code: json.loads(cde.metadata)["sql_type"] for code, cde in cdes.items()}
+
+
+def get_cdes_with_min_max(cdes, columns):
+    cdes_with_min_max = {}
+    for code, cde in cdes.items():
+        if code not in columns:
+            continue
+
+        metadata = json.loads(cde.metadata)
+        min_value = metadata.get("min")
+        max_value = metadata.get("max")
+
+        if min_value is not None or max_value is not None:
+            cdes_with_min_max[code] = (min_value, max_value)
+
+    return cdes_with_min_max
+
+
+def get_cdes_with_enumerations(cdes, columns):
+    cdes_with_enumerations = {}
+    for code, cde in cdes.items():
+        if code not in columns:
+            continue
+
+        metadata = json.loads(cde.metadata)
+        if metadata["is_categorical"]:
+            cdes_with_enumerations[code] = list(metadata["enumerations"].keys())
+
+    return cdes_with_enumerations
+
+
+def get_dataset_enums(cdes):
+    return json.loads(cdes["dataset"].metadata)["enumerations"]
+
+
+def validate_dataset_present_on_cdes_with_proper_format(cdes):
+    dataset_cde = [cde for cde in cdes if cde.code == "dataset"]
+    if not dataset_cde:
+        raise InvalidDataModelError("There is no 'dataset' CDE in the data model.")
+
+    dataset_metadata = json.loads(dataset_cde[0].metadata)
+    if not dataset_metadata["is_categorical"]:
+        raise InvalidDataModelError(
+            "CDE 'dataset' must have the 'isCategorical' property equal to 'true'."
+        )
+
+    if dataset_metadata["sql_type"] != "text":
+        raise InvalidDataModelError(
+            "CDE 'dataset' must have the 'sql_type' property equal to 'text'."
+        )
+
+
+def validate_longitudinal_data_model(cdes):
+    subject_id_metadata = None
+    visit_id_metadata = None
+
+    for cde in cdes:
+        if cde.code == "subjectid":
+            subject_id_metadata = json.loads(cde.metadata)
+        elif cde.code == "visitid":
+            visit_id_metadata = json.loads(cde.metadata)
+
+    if not subject_id_metadata:
+        raise InvalidDataModelError(
+            "There is no 'subjectid' CDE in the longitudinal data model."
+        )
+
+    if not visit_id_metadata:
+        raise InvalidDataModelError(
+            "There is no 'visitid' CDE in the longitudinal data model."
+        )
+
+    validate_visitid_cde(visit_id_metadata)
+
+
+def validate_visitid_cde(metadata):
+    if not metadata["is_categorical"]:
+        raise InvalidDataModelError(
+            "CDE 'visitid' must have the 'isCategorical' property equal to 'true'."
+        )
+
+    if metadata["sql_type"] != "text":
+        raise InvalidDataModelError(
+            "CDE 'visitid' must have the 'sql_type' property equal to 'text'."
+        )
+
+    if "enumerations" not in metadata:
+        raise InvalidDataModelError(
+            "CDE 'visitid' must contain the 'enumerations' property."
+        )
+
+
+def reformat_metadata(metadata):
+    new_key_assign = {
+        "isCategorical": "is_categorical",
+        "minValue": "min",
+        "maxValue": "max",
+    }
+
+    for old_key, new_key in new_key_assign.items():
+        if old_key in metadata:
+            metadata[new_key] = metadata.pop(old_key)
+
+    if "enumerations" in metadata:
+        metadata["enumerations"] = {
+            enumeration["code"]: enumeration["label"]
+            for enumeration in metadata["enumerations"]
+        }
+
+    return metadata
+
+
+def validate_metadata(code, metadata):
+    for element in ["is_categorical", "code", "sql_type", "label", "type"]:
+        if element not in metadata:
+            raise InvalidDataModelError(
+                f"Element: {element} is missing from the CDE {code}"
+            )
+
+    if metadata["is_categorical"] and "enumerations" not in metadata:
+        raise InvalidDataModelError(
+            f"The CDE {code} has 'is_categorical' set to True but there are no enumerations."
+        )
+
+    if {"min", "max"} <= set(metadata) and metadata["min"] >= metadata["max"]:
+        raise InvalidDataModelError(f"The CDE {code} has min greater than the max.")
+
+    valid_metadata_types = ["nominal", "real", "integer", "text"]
+    if metadata["type"] not in valid_metadata_types:
+        raise InvalidDataModelError(
+            f"The CDE {code} has an 'type' the only valid types are:{valid_metadata_types} "
+        )