datacontract-cli 0.10.0__py3-none-any.whl → 0.10.37__py3-none-any.whl

datacontract/model/run.py

@@ -1,21 +1,38 @@
 import logging
 from datetime import datetime, timezone
-from typing import List, Optional
+from enum import Enum
+from typing import List
 from uuid import UUID, uuid4
 
 from pydantic import BaseModel
 
 
+class ResultEnum(str, Enum):
+    passed = "passed"
+    warning = "warning"
+    failed = "failed"
+    error = "error"
+    info = "info"
+    unknown = "unknown"
+
+
 class Check(BaseModel):
+    id: str | None = None
+    key: str | None = None
+    category: str | None = None
     type: str
-    name: Optional[str]
-    result: str  # passed, failed, warning, unknown
-    engine: str
-    reason: Optional[str] = None
-    model: Optional[str] = None
-    field: Optional[str] = None
-    details: Optional[str] = None
-    diagnostics: Optional[dict] = None
+    name: str | None = None
+    model: str | None = None
+    field: str | None = None
+
+    engine: str | None = None
+    language: str | None = None
+    implementation: str | None = None
+
+    result: ResultEnum | None = None
+    reason: str | None = None
+    details: str | None = None
+    diagnostics: dict | None = None
 
 
 class Log(BaseModel):
@@ -26,36 +43,36 @@ class Log(BaseModel):
 
 class Run(BaseModel):
     runId: UUID
-    dataContractId: Optional[str] = None
-    dataContractVersion: Optional[str] = None
-    dataProductId: Optional[str] = None
-    outputPortId: Optional[str] = None
-    server: Optional[str] = None
-    timestampStart: datetime
-    timestampEnd: datetime
-    result: str = "unknown"  # passed, warning, failed, error, unknown
-    checks: List[Check]
-    logs: List[Log]
+    dataContractId: str | None = None
+    dataContractVersion: str | None = None
+    dataProductId: str | None = None
+    outputPortId: str | None = None
+    server: str | None = None
+    timestampStart: datetime | None
+    timestampEnd: datetime | None
+    result: ResultEnum = ResultEnum.unknown
+    checks: List[Check] | None
+    logs: List[Log] | None
 
     def has_passed(self):
         self.calculate_result()
-        return self.result == "passed"
+        return self.result == ResultEnum.passed
 
     def finish(self):
         self.timestampEnd = datetime.now(timezone.utc)
         self.calculate_result()
 
     def calculate_result(self):
-        if any(check.result == "error" for check in self.checks):
-            self.result = "error"
-        elif any(check.result == "failed" for check in self.checks):
-            self.result = "failed"
-        elif any(check.result == "warning" for check in self.checks):
-            self.result = "warning"
-        elif any(check.result == "passed" for check in self.checks):
-            self.result = "passed"
+        if any(check.result == ResultEnum.error for check in self.checks):
+            self.result = ResultEnum.error
+        elif any(check.result == ResultEnum.failed for check in self.checks):
+            self.result = ResultEnum.failed
+        elif any(check.result == ResultEnum.warning for check in self.checks):
+            self.result = ResultEnum.warning
+        elif any(check.result == ResultEnum.passed for check in self.checks):
+            self.result = ResultEnum.passed
         else:
-            self.result = "unknown"
+            self.result = ResultEnum.unknown
 
     def log_info(self, message: str):
         logging.info(message)
@@ -72,6 +89,9 @@ class Run(BaseModel):
     def pretty(self):
         return self.model_dump_json(indent=2)
 
+    def pretty_logs(self) -> str:
+        return "\n".join(f"[{log.timestamp.isoformat()}] {log.level}: {log.message}" for log in self.logs)
+
     @staticmethod
     def create_run():
         """

datacontract/output/junit_test_results.py

@@ -0,0 +1,135 @@
+import xml.etree.ElementTree as ET
+from pathlib import Path
+from xml.dom import minidom
+
+import yaml
+
+from datacontract.model.run import ResultEnum, Run
+
+
+def write_junit_test_results(run: Run, console, output_path: Path):
+    if not output_path:
+        console.print("No output path specified for JUnit test results. Skip writing JUnit test results.")
+        return
+
+    testsuite = ET.Element(
+        "testsuite",
+        id=str(run.runId),
+        name=run.dataContractId if run.dataContractId else "Data Contract",
+        tests=str(len(run.checks)),
+        errors=str(count_errors(run)),
+        failures=str(count_failed(run)),
+        skipped=str(count_skipped(run)),
+        timestamp=run.timestampStart.replace(tzinfo=None).isoformat(),
+        time=str((run.timestampEnd - run.timestampStart).total_seconds()),
+    )
+
+    testsuiteProperties = ET.SubElement(testsuite, "properties")
+    if run.dataContractId is not None:
+        ET.SubElement(testsuiteProperties, "property", name="dataContractId", value=run.dataContractId)
+    if run.dataContractVersion is not None:
+        ET.SubElement(testsuiteProperties, "property", name="dataContractVersion", value=run.dataContractVersion)
+    if run.dataProductId is not None:
+        ET.SubElement(testsuiteProperties, "property", name="dataProductId", value=run.dataProductId)
+    if run.outputPortId is not None:
+        ET.SubElement(testsuiteProperties, "property", name="outputPortId", value=run.outputPortId)
+    if run.server is not None:
+        ET.SubElement(testsuiteProperties, "property", name="server", value=run.server)
+
+    for check in run.checks:
+        testcase = ET.SubElement(testsuite, "testcase", classname=to_class_name(check), name=to_testcase_name(check))
+        if check.result == ResultEnum.passed:
+            pass
+        elif check.result == ResultEnum.failed:
+            failure = ET.SubElement(
+                testcase,
+                "failure",
+                message=check.reason if check.reason else "Failed",
+                type=check.category if check.category else "General",
+            )
+            failure.text = to_failure_text(check)
+        elif check.result == ResultEnum.error:
+            error = ET.SubElement(
+                testcase,
+                "error",
+                message=check.reason if check.reason else "Error",
+                type=check.category if check.category else "General",
+            )
+            error.text = to_failure_text(check)
+        elif check.result == ResultEnum.warning:
+            skipped = ET.SubElement(
+                testcase,
+                "skipped",
+                message=check.reason if check.reason else "Warning",
+                type=check.category if check.category else "General",
+            )
+            skipped.text = to_failure_text(check)
+        else:
+            ET.SubElement(
+                testcase,
+                "skipped",
+                message=check.reason if check.reason else "Skipped",
+                type=check.category if check.category else "General",
+            )
+
+    if run.logs:
+        system_out = ET.SubElement(testsuite, "system-out")
+        system_out.text = logs_to_system_out(run)
+
+    xml_str: str = ET.tostring(testsuite, xml_declaration=True, encoding="utf-8")
+    xml_str_pretty = minidom.parseString(xml_str).toprettyxml(indent="  ")
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    with open(output_path, "w", encoding="utf-8") as f:
+        f.write(xml_str_pretty)
+    console.print(f"JUnit test results written to {output_path}")
+
+
+def to_testcase_name(check):
+    if check.key:
+        return check.key
+    if check.name:
+        return check.name
+    else:
+        return "unknown"
+
+
+def logs_to_system_out(run):
+    result = ""
+    for log in run.logs:
+        result += f"{log.timestamp} {log.level}: {log.message}\n"
+    return result
+
+
+def to_class_name(check):
+    if check.model and check.field:
+        return f"{check.model}.{check.field}"
+    elif check.model:
+        return check.model
+    elif check.field:
+        return check.field
+    else:
+        return "general"
+
+
+def to_failure_text(check):
+    return (
+        f"Name: {check.name}\n"
+        f"Engine: {check.engine}\n"
+        f"Implementation:\n{check.implementation}\n\n"
+        f"Result: {check.result.value if check.result is not None else ''}\n"
+        f"Reason: {check.reason}\n"
+        f"Details: {check.details}\n"
+        f"Diagnostics:\n{yaml.dump(check.diagnostics, default_flow_style=False)}"
+    )
+
+
+def count_errors(run):
+    return sum(1 for check in run.checks if check.result == ResultEnum.error)
+
+
+def count_failed(run):
+    return sum(1 for check in run.checks if check.result == ResultEnum.failed)
+
+
+def count_skipped(run):
+    return sum(1 for check in run.checks if check.result is None)

datacontract/output/output_format.py

@@ -0,0 +1,10 @@
+from enum import Enum
+
+
+class OutputFormat(str, Enum):
+    # json = "json" # coming soon
+    junit = "junit"
+
+    @classmethod
+    def get_supported_formats(cls):
+        return list(map(lambda c: c.value, cls))

datacontract/output/test_results_writer.py

@@ -0,0 +1,79 @@
+from pathlib import Path
+
+import typer
+from rich import box
+from rich.console import Console
+from rich.table import Table
+
+from datacontract.model.run import Run
+from datacontract.output.junit_test_results import write_junit_test_results
+from datacontract.output.output_format import OutputFormat
+
+
+def write_test_result(run: Run, console: Console, output_format: OutputFormat, output_path: Path):
+    if output_format == OutputFormat.junit:
+        write_junit_test_results(run, console, output_path)
+
+    _print_table(run, console)
+    if run.result == "passed":
+        console.print(
+            f"🟢 data contract is valid. Run {len(run.checks)} checks. Took {(run.timestampEnd - run.timestampStart).total_seconds()} seconds."
+        )
+    elif run.result == "warning":
+        console.print("🟠 data contract has warnings. Found the following warnings:")
+        i = 1
+        for check in run.checks:
+            if check.result != "passed":
+                field = to_field(run, check)
+                if field:
+                    field = field + " "
+                else:
+                    field = ""
+                console.print(f"{i}) {field}{check.name}: {check.reason}")
+                i += 1
+    else:
+        console.print("🔴 data contract is invalid, found the following errors:")
+        i = 1
+        for check in run.checks:
+            if check.result != "passed":
+                field = to_field(run, check)
+                if field:
+                    field = field + " "
+                else:
+                    field = ""
+                console.print(f"{i}) {field}{check.name}: {check.reason}")
+                i += 1
+        raise typer.Exit(code=1)
+
+
+def _print_table(run, console):
+    table = Table(box=box.ROUNDED)
+    table.add_column("Result", no_wrap=True)
+    table.add_column("Check", max_width=100)
+    table.add_column("Field", max_width=32)
+    table.add_column("Details", max_width=50)
+    for check in sorted(run.checks, key=lambda c: (c.result or "", c.model or "", c.field or "")):
+        table.add_row(with_markup(check.result), check.name, to_field(run, check), check.reason)
+    console.print(table)
+
+
+def to_field(run, check):
+    models = [c.model for c in run.checks]
+    if len(set(models)) > 1:
+        if check.field is None:
+            return check.model
+        return check.model + "." + check.field
+    else:
+        return check.field
+
+
+def with_markup(result):
+    if result == "passed":
+        return "[green]passed[/green]"
+    if result == "warning":
+        return "[yellow]warning[/yellow]"
+    if result == "failed":
+        return "[red]failed[/red]"
+    if result == "error":
+        return "[red]error[/red]"
+    return result

datacontract/schemas/datacontract-1.1.0.init.yaml

@@ -0,0 +1,91 @@
+dataContractSpecification: 1.2.1
+id: my-data-contract-id
+info:
+  title: My Data Contract
+  version: 0.0.1
+#  description:
+#  owner:
+#  contact:
+#    name:
+#    url:
+#    email:
+
+
+### servers
+
+#servers:
+#  production:
+#    type: s3
+#    location: s3://
+#    format: parquet
+#    delimiter: new_line
+
+### terms
+
+#terms:
+#  usage:
+#  limitations:
+#  billing:
+#  noticePeriod:
+
+
+### models
+
+# models:
+#   my_model:
+#     description:
+#     type:
+#     fields:
+#       my_field:
+#         type:
+#         description:
+
+
+### definitions
+
+# definitions:
+#   my_field:
+#     domain:
+#     name:
+#     title:
+#     type:
+#     description:
+#     example:
+#     pii:
+#     classification:
+
+
+### servicelevels
+
+#servicelevels:
+#  availability:
+#    description: The server is available during support hours
+#    percentage: 99.9%
+#  retention:
+#    description: Data is retained for one year because!
+#    period: P1Y
+#    unlimited: false
+#  latency:
+#    description: Data is available within 25 hours after the order was placed
+#    threshold: 25h
+#    sourceTimestampField: orders.order_timestamp
+#    processedTimestampField: orders.processed_timestamp
+#  freshness:
+#    description: The age of the youngest row in a table.
+#    threshold: 25h
+#    timestampField: orders.order_timestamp
+#  frequency:
+#    description: Data is delivered once a day
+#    type: batch # or streaming
+#    interval: daily # for batch, either or cron
+#    cron: 0 0 * * * # for batch, either or interval
+#  support:
+#    description: The data is available during typical business hours at headquarters
+#    time: 9am to 5pm in EST on business days
+#    responseTime: 1h
+#  backup:
+#    description: Data is backed up once a week, every Sunday at 0:00 UTC.
+#    interval: weekly
+#    cron: 0 0 * * 0
+#    recoveryTime: 24 hours
+#    recoveryPoint: 1 week