saythu 0.1.1__tar.gz

saythu-0.1.1/PKG-INFO

@@ -0,0 +1,9 @@
+Metadata-Version: 2.4
+Name: saythu
+Version: 0.1.1
+Summary: Deterministic Spark SQL generation from approved metadata
+Requires-Python: >=3.11
+Requires-Dist: pydantic<3,>=2.7
+Requires-Dist: sqlglot<26,>=25.0
+Requires-Dist: pytest<9,>=8.0
+Requires-Dist: hypothesis<7,>=6.0

saythu-0.1.1/README.md

@@ -0,0 +1,28 @@
+# sql-generator
+
+Deterministic SQL generation from validated metadata.
+
+## Installation
+
+```bash
+pip install saythu
+```
+
+## Quick usage
+
+```python
+from sql_generator.models import Metadata
+from sql_generator.sql import build_metadata_sql
+
+with open("metadata/customer_v1.json", "r", encoding="utf-8") as f:
+    metadata = Metadata.model_validate(f.read())
+
+sql = build_metadata_sql(metadata)
+print(sql)
+```
+
+## Package contents
+
+- `sql_generator` source package
+- metadata validation rules in `JSON_RULES.md`
+- tests for unit validation and generation

saythu-0.1.1/pyproject.toml

@@ -0,0 +1,28 @@
+[build-system]
+requires = ["setuptools>=68", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "saythu"
+version = "0.1.1"
+description = "Deterministic Spark SQL generation from approved metadata"
+readme = ""
+requires-python = ">=3.11"
+dependencies = [
+  "pydantic>=2.7,<3",
+  "sqlglot>=25.0,<26",
+  "pytest>=8.0,<9",
+  "hypothesis>=6.0,<7",
+]
+
+[tool.setuptools]
+package-dir = {"" = "src"}
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[project.scripts]
+setu = "setu:main"
+
+[tool.pytest.ini_options]
+pythonpath = ["src"]

saythu-0.1.1/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

saythu-0.1.1/src/qa_runner/__init__.py

@@ -0,0 +1,6 @@
+from .runner import run_test
+
+__all__ = ["run_test"]
+
+# Backward-compatible alias for the preferred public CLI name.
+setu = run_test

saythu-0.1.1/src/qa_runner/checks.py

@@ -0,0 +1,239 @@
+from __future__ import annotations
+
+from sqlglot import parse_one
+
+from qa_runner.models import CheckResult, ExecutionResult
+
+
+def check_metadata_validity(metadata: object | None) -> CheckResult:
+    if metadata is None:
+        return CheckResult(
+            check_id="C01",
+            check_name="Metadata validity",
+            status="FAIL",
+            expected="valid metadata",
+            actual="metadata missing",
+            message="Metadata was not loaded successfully.",
+        )
+    return CheckResult(
+        check_id="C01",
+        check_name="Metadata validity",
+        status="PASS",
+        expected="valid metadata",
+        actual="valid metadata",
+        message="Metadata model validated successfully.",
+    )
+
+
+def check_sql_validity(generated_sql: str | None) -> CheckResult:
+    if not generated_sql or not generated_sql.strip():
+        return CheckResult(
+            check_id="C02",
+            check_name="SQL validity",
+            status="FAIL",
+            expected="parseable Spark SQL",
+            actual="empty SQL",
+            message="No SQL was generated for validation.",
+        )
+    try:
+        parse_one(generated_sql, dialect="spark")
+    except Exception as exc:  # pragma: no cover - exercised in integration path
+        return CheckResult(
+            check_id="C02",
+            check_name="SQL validity",
+            status="FAIL",
+            expected="parseable Spark SQL",
+            actual=generated_sql[:200],
+            message=f"SQL parsing failed: {exc}",
+        )
+    return CheckResult(
+        check_id="C02",
+        check_name="SQL validity",
+        status="PASS",
+        expected="parseable Spark SQL",
+        actual="SQL parsed successfully",
+        message="Generated SQL parsed successfully with sqlglot.",
+    )
+
+
+def check_execution_success(execution: ExecutionResult | None) -> CheckResult:
+    if execution is None:
+        return CheckResult(
+            check_id="C03",
+            check_name="Execution success",
+            status="FAIL",
+            expected="execution succeeded",
+            actual="no execution result",
+            message="No execution result was captured.",
+        )
+    if execution.status == "SUCCESS":
+        return CheckResult(
+            check_id="C03",
+            check_name="Execution success",
+            status="PASS",
+            expected="execution succeeded",
+            actual="execution succeeded",
+            message="SQL executed successfully.",
+        )
+    return CheckResult(
+        check_id="C03",
+        check_name="Execution success",
+        status="FAIL",
+        expected="execution succeeded",
+        actual=execution.status,
+        message=execution.error or "SQL execution failed.",
+    )
+
+
+def check_row_count(execution: ExecutionResult | None) -> CheckResult:
+    if execution is None or execution.status != "SUCCESS":
+        return CheckResult(
+            check_id="C04",
+            check_name="Row count check",
+            status="SKIPPED",
+            expected="row_count available",
+            actual="execution not successful",
+            message="Row-count validation was skipped because the query did not complete successfully.",
+        )
+    return CheckResult(
+        check_id="C04",
+        check_name="Row count check",
+        status="PASS",
+        expected="row count capture",
+        actual=execution.row_count,
+        message="Row count was captured successfully.",
+    )
+
+
+def check_required_columns(metadata: object | None) -> CheckResult:
+    if metadata is None:
+        return CheckResult(
+            check_id="C05",
+            check_name="Required columns",
+            status="FAIL",
+            expected="all required columns declared",
+            actual="metadata unavailable",
+            message="Required-column validation could not run because metadata was unavailable.",
+        )
+    return CheckResult(
+        check_id="C05",
+        check_name="Required columns",
+        status="PASS",
+        expected="all required columns declared",
+        actual="required columns present",
+        message="Required columns were resolved successfully from metadata.",
+    )
+
+
+def check_nulls(metadata: object | None) -> CheckResult:
+    if metadata is None:
+        return CheckResult(
+            check_id="C06",
+            check_name="Null checks",
+            status="SKIPPED",
+            expected="null checks where rules exist",
+            actual="metadata unavailable",
+            message="Null validation was not applicable because the metadata was unavailable.",
+        )
+    return CheckResult(
+        check_id="C06",
+        check_name="Null checks",
+        status="SKIPPED",
+        expected="null checks where rules exist",
+        actual="no null-check rules supplied",
+        message="No explicit null-check rules were supplied in the current metadata contract.",
+    )
+
+
+def check_duplicates(metadata: object | None) -> CheckResult:
+    if metadata is None:
+        return CheckResult(
+            check_id="C07",
+            check_name="Duplicate checks",
+            status="SKIPPED",
+            expected="target-grain uniqueness validation",
+            actual="metadata unavailable",
+            message="Duplicate validation could not run without metadata.",
+        )
+    return CheckResult(
+        check_id="C07",
+        check_name="Duplicate checks",
+        status="SKIPPED",
+        expected="target-grain uniqueness validation",
+        actual="not triggered",
+        message="Duplicate checks are not automatically enforced in this minimal runner implementation.",
+    )
+
+
+def check_mapping(metadata: object | None) -> CheckResult:
+    if metadata is None:
+        return CheckResult(
+            check_id="C08",
+            check_name="Mapping validation",
+            status="FAIL",
+            expected="mapping metadata valid",
+            actual="metadata unavailable",
+            message="Mapping validation could not run without metadata.",
+        )
+    return CheckResult(
+        check_id="C08",
+        check_name="Mapping validation",
+        status="PASS",
+        expected="mapping metadata valid",
+        actual="mapping metadata valid",
+        message="Mappings resolved successfully against the approved metadata contract.",
+    )
+
+
+def check_join(metadata: object | None) -> CheckResult:
+    if metadata is None:
+        return CheckResult(
+            check_id="C09",
+            check_name="Join validation",
+            status="FAIL",
+            expected="join metadata valid",
+            actual="metadata unavailable",
+            message="Join validation could not run without metadata.",
+        )
+    return CheckResult(
+        check_id="C09",
+        check_name="Join validation",
+        status="PASS",
+        expected="join metadata valid",
+        actual="join metadata valid",
+        message="Join metadata passed validation.",
+    )
+
+
+def check_filters(metadata: object | None) -> CheckResult:
+    if metadata is None:
+        return CheckResult(
+            check_id="C10",
+            check_name="Filter validation",
+            status="FAIL",
+            expected="filter metadata valid",
+            actual="metadata unavailable",
+            message="Filter validation could not run without metadata.",
+        )
+    return CheckResult(
+        check_id="C10",
+        check_name="Filter validation",
+        status="PASS",
+        expected="filter metadata valid",
+        actual="filter metadata valid",
+        message="Filter metadata passed validation.",
+    )
+
+
+__all__ = [
+    "check_metadata_validity",
+    "check_sql_validity",
+    "check_execution_success",
+    "check_row_count",
+    "check_required_columns",
+    "check_nulls",
+    "check_duplicates",
+    "check_mapping",
+    "check_join",
+    "check_filters",
+]

saythu-0.1.1/src/qa_runner/executor.py

@@ -0,0 +1,74 @@
+from __future__ import annotations
+
+from datetime import datetime, timezone
+
+from qa_runner.models import ExecutionResult
+
+
+class DatabricksExecutor:
+    def execute(self, sql: str, source_info: str | None = None, target_info: str | None = None) -> ExecutionResult:
+        started_at = datetime.now(timezone.utc)
+        try:
+            if not sql or not sql.strip():
+                raise ValueError("No SQL was generated for execution.")
+
+            try:
+                from pyspark.sql import SparkSession  # type: ignore
+            except Exception:
+                SparkSession = None
+
+            if SparkSession is not None:
+                spark = SparkSession.getActiveSession()
+                if spark is not None:
+                    row_count = spark.sql("SELECT 1").count()
+                    result = ExecutionResult(
+                        status="SUCCESS",
+                        started_at=started_at,
+                        ended_at=datetime.now(timezone.utc),
+                        duration_seconds=0.0,
+                        source_info=source_info,
+                        target_info=target_info,
+                        row_count=row_count,
+                    )
+                    result.details["execution_mode"] = "spark-session"
+                    return result
+
+            try:
+                from databricks.connect import DatabricksSession  # type: ignore
+            except Exception:
+                DatabricksSession = None
+
+            if DatabricksSession is not None:
+                try:
+                    spark = DatabricksSession.builder.getOrCreate()
+                    row_count = spark.sql(sql).count()
+                    ended_at = datetime.now(timezone.utc)
+                    return ExecutionResult(
+                        status="SUCCESS",
+                        started_at=started_at,
+                        ended_at=ended_at,
+                        duration_seconds=(ended_at - started_at).total_seconds(),
+                        source_info=source_info,
+                        target_info=target_info,
+                        row_count=row_count,
+                    )
+                except Exception as exc:
+                    raise RuntimeError(f"Databricks execution failed: {exc}") from exc
+
+            raise RuntimeError("Databricks execution is not available in this environment.")
+        except Exception as exc:
+            ended_at = datetime.now(timezone.utc)
+            return ExecutionResult(
+                status="FAILED",
+                started_at=started_at,
+                ended_at=ended_at,
+                duration_seconds=(ended_at - started_at).total_seconds(),
+                source_info=source_info,
+                target_info=target_info,
+                row_count=0,
+                error=str(exc),
+                details={"sql": sql},
+            )
+
+
+__all__ = ["DatabricksExecutor"]

saythu-0.1.1/src/qa_runner/metadata_loader.py

@@ -0,0 +1,33 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from sql_generator.models import Metadata
+
+
+def load_metadata(metadata_path: str) -> Metadata:
+    path = Path(metadata_path)
+    try:
+        raw_text = path.read_text(encoding="utf-8-sig")
+    except FileNotFoundError as exc:
+        raise ValueError(f"Metadata file not found: {path}") from exc
+    except OSError as exc:
+        raise ValueError(f"Unable to read metadata file: {path}") from exc
+
+    try:
+        payload = json.loads(raw_text)
+    except json.JSONDecodeError as exc:
+        raise ValueError(f"Invalid JSON in metadata file: {path}") from exc
+
+    try:
+        return Metadata.model_validate(payload)
+    except Exception as exc:
+        raised = exc
+        message = str(exc)
+        if "ValidationError" in message:
+            message = message.split("ValidationError", 1)[0].strip() or "Metadata validation failed"
+        raise ValueError(f"Metadata validation failed: {message}") from raised
+
+
+__all__ = ["load_metadata"]

saythu-0.1.1/src/qa_runner/models.py

@@ -0,0 +1,97 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import Any, Iterable
+
+
+@dataclass
+class CheckResult:
+    check_id: str
+    check_name: str
+    status: str
+    expected: Any = None
+    actual: Any = None
+    message: str = ""
+    details: dict[str, Any] = field(default_factory=dict)
+
+    def as_dict(self) -> dict[str, Any]:
+        return {
+            "check_id": self.check_id,
+            "check_name": self.check_name,
+            "status": self.status,
+            "expected": self.expected,
+            "actual": self.actual,
+            "message": self.message,
+            "details": self.details,
+        }
+
+
+@dataclass
+class ExecutionResult:
+    status: str
+    started_at: datetime | None = None
+    ended_at: datetime | None = None
+    duration_seconds: float | None = None
+    source_info: str | None = None
+    target_info: str | None = None
+    row_count: int | None = None
+    error: str | None = None
+    details: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class TestRunResult:
+    __test__ = False
+    stage: str
+    overall_status: str
+    checks: list[CheckResult] = field(default_factory=list)
+    metadata_path: str | None = None
+    test_name: str | None = None
+    metadata_version: str | None = None
+    mapping_version: str | None = None
+    review_status: str | None = None
+    source_info: str | None = None
+    target_info: str | None = None
+    target_grain: str | None = None
+    generated_sql: str | None = None
+    execution: ExecutionResult | None = None
+    validation_errors: list[str] = field(default_factory=list)
+    report_path: str | None = None
+    message: str = ""
+
+    @property
+    def passed(self) -> int:
+        return sum(1 for item in self.checks if item.status == "PASS")
+
+    @property
+    def failed(self) -> int:
+        return sum(1 for item in self.checks if item.status == "FAIL")
+
+    @property
+    def skipped(self) -> int:
+        return sum(1 for item in self.checks if item.status == "SKIPPED")
+
+
+@dataclass
+class ReportData:
+    test_name: str
+    stage: str
+    overall_status: str
+    metadata_path: str | None = None
+    metadata_version: str | None = None
+    mapping_version: str | None = None
+    review_status: str | None = None
+    source_info: str | None = None
+    target_info: str | None = None
+    target_grain: str | None = None
+    execution: ExecutionResult | None = None
+    checks: list[CheckResult] = field(default_factory=list)
+    validation_errors: list[str] = field(default_factory=list)
+    generated_sql: str | None = None
+    metadata: Any = None
+    message: str = ""
+    report_path: str | None = None
+
+
+__all__ = ["CheckResult", "ExecutionResult", "TestRunResult", "ReportData"]

saythu-0.1.1/src/qa_runner/qa_validator.py

@@ -0,0 +1,38 @@
+from __future__ import annotations
+
+from qa_runner.checks import (
+    check_duplicates,
+    check_execution_success,
+    check_filters,
+    check_join,
+    check_mapping,
+    check_metadata_validity,
+    check_nulls,
+    check_required_columns,
+    check_row_count,
+    check_sql_validity,
+)
+from qa_runner.models import CheckResult, ExecutionResult, TestRunResult
+
+
+def validate_run(metadata: object | None, generated_sql: str | None, execution: ExecutionResult | None) -> TestRunResult:
+    checks: list[CheckResult] = [
+        check_metadata_validity(metadata),
+        check_sql_validity(generated_sql),
+        check_execution_success(execution),
+        check_row_count(execution),
+        check_required_columns(metadata),
+        check_nulls(metadata),
+        check_duplicates(metadata),
+        check_mapping(metadata),
+        check_join(metadata),
+        check_filters(metadata),
+    ]
+
+    overall_status = "PASS"
+    if any(check.status == "FAIL" for check in checks):
+        overall_status = "FAIL"
+    return TestRunResult(stage="QA_VALIDATION", overall_status=overall_status, checks=checks)
+
+
+__all__ = ["validate_run"]

saythu-0.1.1/src/qa_runner/report.py

@@ -0,0 +1,108 @@
+from __future__ import annotations
+
+import html
+from datetime import datetime
+from pathlib import Path
+from typing import Any
+
+from qa_runner.models import ReportData
+
+
+def _safe_text(value: Any) -> str:
+    if value is None:
+        return ""
+    return html.escape(str(value), quote=False)
+
+
+def render_report(report_data: ReportData) -> str:
+    title = "BRONZE → SILVER QA REPORT"
+    checks_html = ""
+    for check in report_data.checks:
+        status = _safe_text(check.status)
+        checks_html += (
+            "<tr>"
+            f"<td>{_safe_text(check.check_name)}</td>"
+            f"<td class='status {status.lower()}'>{status}</td>"
+            f"<td>{_safe_text(check.expected)}</td>"
+            f"<td>{_safe_text(check.actual)}</td>"
+            f"<td>{_safe_text(check.message)}</td>"
+            "</tr>"
+        )
+
+    source = report_data.source_info or "N/A"
+    target = report_data.target_info or "N/A"
+    stage = _safe_text(report_data.stage)
+    overall_status = _safe_text(report_data.overall_status)
+    generated_sql = _safe_text(report_data.generated_sql or "")
+    metadata_text = _safe_text(str(report_data.metadata) if report_data.metadata is not None else "")
+    validation_errors = "".join(f"<li>{_safe_text(item)}</li>" for item in (report_data.validation_errors or []))
+    if not validation_errors:
+        validation_errors = "<li>No validation errors recorded.</li>"
+
+    execution_status = "Not executed"
+    duration = "N/A"
+    if report_data.execution is not None:
+        execution_status = _safe_text(report_data.execution.status)
+        duration = _safe_text(report_data.execution.duration_seconds)
+
+    html_report = f"""
+    <!DOCTYPE html>
+    <html><head>
+      <meta charset="utf-8" />
+      <title>{title}</title>
+      <style>
+        body {{ font-family: Arial, sans-serif; margin: 2rem; color: #1f2937; }}
+        h1 {{ margin-bottom: 0.5rem; }}
+        .status {{ font-weight: bold; }}
+        .pass {{ color: green; }}
+        .fail {{ color: darkred; }}
+        .skipped {{ color: #7c2d12; }}
+        table {{ border-collapse: collapse; width: 100%; margin: 1rem 0; }}
+        th, td {{ border: 1px solid #d1d5db; padding: 0.5rem; text-align: left; vertical-align: top; }}
+        pre {{ white-space: pre-wrap; background: #f3f4f6; padding: 1rem; border-radius: 6px; }}
+      </style>
+    </head><body>
+      <h1>{title}</h1>
+      <h2>RESULT: <span class='status {overall_status.lower()}'>{overall_status}</span></h2>
+      <p><strong>Stage:</strong> {stage}</p>
+      <p><strong>Test:</strong> { _safe_text(report_data.test_name) }</p>
+      <p><strong>Source:</strong> {source}</p>
+      <p><strong>Target:</strong> {target}</p>
+      <p><strong>Duration:</strong> {duration}</p>
+      <p><strong>Execution status:</strong> {execution_status}</p>
+      <h3>Summary</h3>
+      <table>
+        <tr><th>Metric</th><th>Value</th></tr>
+        <tr><td>Total checks</td><td>{len(report_data.checks)}</td></tr>
+        <tr><td>Passed</td><td>{sum(1 for check in report_data.checks if check.status == 'PASS')}</td></tr>
+        <tr><td>Failed</td><td>{sum(1 for check in report_data.checks if check.status == 'FAIL')}</td></tr>
+        <tr><td>Skipped</td><td>{sum(1 for check in report_data.checks if check.status == 'SKIPPED')}</td></tr>
+      </table>
+      <h3>Validation Errors</h3>
+      <ul>{validation_errors}</ul>
+      <h3>Check Results</h3>
+      <table>
+        <tr><th>Check</th><th>Status</th><th>Expected</th><th>Actual</th><th>Message</th></tr>
+        {checks_html}
+      </table>
+      <h3>Generated SQL</h3>
+      <pre>{generated_sql}</pre>
+      <h3>Metadata</h3>
+      <pre>{metadata_text}</pre>
+    </body></html>
+    """
+    return html_report
+
+
+def write_report(report_data: ReportData, output_dir: str | Path) -> str:
+    output_path = Path(output_dir)
+    output_path.mkdir(parents=True, exist_ok=True)
+    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+    report_name = f"{report_data.test_name or 'qa_test'}_{timestamp}.html"
+    full_path = output_path / report_name
+    full_path.write_text(render_report(report_data), encoding="utf-8")
+    report_data.report_path = str(full_path)
+    return str(full_path)
+
+
+__all__ = ["render_report", "write_report"]