dataproduct-cli 0.0.1__py3-none-any.whl

dataproduct/__init__.py

@@ -0,0 +1,5 @@
+"""dataproduct-cli — a CLI for the Open Data Product Standard (ODPS).
+
+Sister project of datacontract-cli. See the ``specs/`` directory for the
+behavioral specification of each command.
+"""

dataproduct/cli.py

@@ -0,0 +1,81 @@
+import logging
+import sys
+from importlib import metadata
+from pathlib import Path
+from typing import Iterable, Optional
+
+import typer
+from click import Context
+from dotenv import find_dotenv, load_dotenv
+from rich.console import Console
+from typer.core import TyperGroup
+from typing_extensions import Annotated
+
+from dataproduct.output.output_format import OutputFormat
+
+console = Console()
+
+debug_option = Annotated[bool, typer.Option(help="Enable debug logging")]
+
+# Order in which top-level commands appear in `dataproduct --help`.
+COMMAND_ORDER = ["init", "lint", "publish"]
+
+
+class OrderedCommands(TyperGroup):
+    def list_commands(self, ctx: Context) -> Iterable[str]:
+        known = set(COMMAND_ORDER)
+        return [c for c in COMMAND_ORDER if c in self.commands] + [c for c in self.commands if c not in known]
+
+
+app = typer.Typer(
+    cls=OrderedCommands,
+    no_args_is_help=True,
+    add_completion=False,
+    pretty_exceptions_show_locals=False,
+    help="CLI for data products following the Open Data Product Standard (ODPS).",
+)
+
+
+def version_callback(value: bool) -> None:
+    if value:
+        try:
+            version = metadata.version("dataproduct-cli")
+        except metadata.PackageNotFoundError:
+            version = "0.0.0"
+        console.print(version)
+        raise typer.Exit()
+
+
+@app.callback()
+def common(
+    ctx: typer.Context,
+    version: Annotated[
+        Optional[bool],
+        typer.Option("--version", callback=version_callback, is_eager=True, help="Print the version and exit."),
+    ] = None,
+) -> None:
+    """dataproduct CLI."""
+    load_dotenv(find_dotenv(usecwd=True))
+
+
+def enable_debug_logging(debug: bool, otherwise_disable_stderr: bool = False) -> None:
+    if debug:
+        logging.basicConfig(level=logging.DEBUG, stream=sys.stderr, format="%(asctime)s %(levelname)s %(message)s")
+    elif otherwise_disable_stderr:
+        logging.disable(logging.CRITICAL)
+
+
+def resolve_output_format(output_format: Optional[OutputFormat], output: Optional[Path]) -> Optional[OutputFormat]:
+    if output_format is not None:
+        return output_format
+    if output is not None:
+        return OutputFormat.junit if str(output).endswith(".xml") else OutputFormat.json
+    return None
+
+
+# Register the commands (each module attaches itself to `app`).
+from dataproduct import command_init, command_lint, command_publish  # noqa: E402, F401
+
+
+def main() -> None:
+    app()

dataproduct/command_init.py

@@ -0,0 +1,33 @@
+import os
+
+import typer
+from typing_extensions import Annotated
+
+from dataproduct.cli import app, console, debug_option, enable_debug_logging
+from dataproduct.init.init_template import get_init_template
+
+
+@app.command(
+    name="init",
+    epilog="Example: dataproduct init dataproduct.odps.yaml",
+)
+def init(
+    location: Annotated[
+        str, typer.Argument(help="The location of the data product file to create.")
+    ] = "dataproduct.odps.yaml",
+    template: Annotated[str, typer.Option(help="URL or path of a template or data product")] = None,
+    overwrite: Annotated[bool, typer.Option(help="Replace the existing data product file")] = False,
+    debug: debug_option = None,
+):
+    """
+    Create a new data product file.
+    """
+    enable_debug_logging(debug)
+
+    if not overwrite and os.path.exists(location):
+        console.print("File already exists, use --overwrite to overwrite")
+        raise typer.Exit(code=1)
+    template_str = get_init_template(template)
+    with open(location, "w") as f:
+        f.write(template_str)
+    console.print("📄 data product written to " + location)

dataproduct/command_lint.py

@@ -0,0 +1,58 @@
+from pathlib import Path
+
+import typer
+from typing_extensions import Annotated
+
+from dataproduct.cli import app, console, debug_option, enable_debug_logging, resolve_output_format
+from dataproduct.config import cli_config
+from dataproduct.data_product import DataProduct
+from dataproduct.output.output_format import OutputFormat
+from dataproduct.output.result_writer import write_result
+
+
+@app.command(
+    name="lint",
+    epilog="Example: dataproduct lint dataproduct.odps.yaml",
+)
+def lint(
+    location: Annotated[
+        str,
+        typer.Argument(help="The location (url or local path) of the data product yaml."),
+    ] = "dataproduct.odps.yaml",
+    schema: Annotated[
+        str,
+        typer.Option("--json-schema", help="The location (url or path) of the ODPS JSON Schema"),
+    ] = None,
+    output: Annotated[
+        Path,
+        typer.Option(
+            help="File path to write the results to (e.g. './TEST-dataproduct.xml'). "
+            "If omitted, results are printed to stdout."
+        ),
+    ] = None,
+    output_format: Annotated[
+        OutputFormat,
+        typer.Option(help="The result format. Accepted values: json, junit."),
+    ] = None,
+    all_errors: Annotated[
+        bool,
+        typer.Option(
+            "--all-errors",
+            help="Report all JSON Schema validation errors instead of stopping after the first one.",
+        ),
+    ] = False,
+    debug: debug_option = None,
+):
+    """
+    Validate that the data product is correctly formatted (against the ODPS JSON Schema).
+    """
+    enable_debug_logging(debug, otherwise_disable_stderr=True)
+
+    output_format = resolve_output_format(output_format, output)
+    run = DataProduct(
+        config=cli_config(),
+        data_product_file=location,
+        schema_location=schema,
+        all_errors=all_errors,
+    ).lint()
+    write_result(run, console, output_format, output)

dataproduct/command_publish.py

@@ -0,0 +1,47 @@
+import typer
+from typing_extensions import Annotated
+
+from dataproduct.cli import app, console, debug_option, enable_debug_logging
+from dataproduct.config import cli_config
+from dataproduct.integration.entropy_data import (
+    DataProductPublishError,
+    publish_data_product_to_entropy_data,
+)
+from dataproduct.lint.resolve import resolve_data_product_dict
+from dataproduct.model.exceptions import DataProductException
+
+
+@app.command(
+    name="publish",
+    epilog="Example: dataproduct publish dataproduct.odps.yaml",
+)
+def publish(
+    location: Annotated[
+        str,
+        typer.Argument(help="The location (url or local path) of the data product yaml."),
+    ] = "dataproduct.odps.yaml",
+    schema: Annotated[
+        str,
+        typer.Option("--json-schema", help="The location (url or path) of the ODPS JSON Schema"),
+    ] = None,
+    ssl_verification: Annotated[
+        bool,
+        typer.Option(help="SSL verification when publishing the data product."),
+    ] = True,
+    debug: debug_option = None,
+):
+    """
+    Publish the data product to Entropy Data.
+    """
+    enable_debug_logging(debug)
+
+    try:
+        data_product_dict = resolve_data_product_dict(location, config=cli_config())
+        publish_data_product_to_entropy_data(
+            data_product_dict=data_product_dict,
+            ssl_verification=ssl_verification,
+            config=cli_config(),
+        )
+    except (DataProductPublishError, DataProductException) as e:
+        console.print(f"[red]Failed publishing data product. Error: {e}[/red]")
+        raise typer.Exit(code=1)

dataproduct/config/__init__.py

@@ -0,0 +1,67 @@
+"""Programmatic configuration for hosts and API keys.
+
+Every value the CLI reads from ``ENTROPY_DATA_*`` (and the legacy
+``DATAMESH_MANAGER_*`` / ``DATACONTRACT_MANAGER_*``) environment variables can
+also be provided programmatically via :class:`Config` (or a plain dict keyed by
+the env var names). Reads fall back to the process environment.
+"""
+
+import os
+from typing import Optional, Union
+
+
+class Config:
+    def __init__(self, values: Optional[dict] = None):
+        self._values = dict(values or {})
+
+    def getenv(self, key: str) -> Optional[str]:
+        value = self._values.get(key)
+        if value is not None:
+            return value
+        return os.environ.get(key)
+
+    @classmethod
+    def resolve(cls, config: "Optional[Union[Config, dict]]" = None) -> "Config":
+        if config is None:
+            return cls()
+        if isinstance(config, Config):
+            return config
+        if isinstance(config, dict):
+            return cls(config)
+        raise TypeError(f"Unsupported config type: {type(config)!r}")
+
+    # Hosts
+    def get_entropy_data_host(self) -> Optional[str]:
+        return self.getenv("ENTROPY_DATA_HOST")
+
+    def get_datamesh_manager_host(self) -> Optional[str]:
+        return self.getenv("DATAMESH_MANAGER_HOST")
+
+    def get_datacontract_manager_host(self) -> Optional[str]:
+        return self.getenv("DATACONTRACT_MANAGER_HOST")
+
+    # API keys
+    def get_entropy_data_api_key(self) -> Optional[str]:
+        return self.getenv("ENTROPY_DATA_API_KEY")
+
+    def get_datamesh_manager_api_key(self) -> Optional[str]:
+        return self.getenv("DATAMESH_MANAGER_API_KEY")
+
+    def get_datacontract_manager_api_key(self) -> Optional[str]:
+        return self.getenv("DATACONTRACT_MANAGER_API_KEY")
+
+
+# Simple process-wide "current" config set from the CLI entry point.
+_cli_config = Config()
+
+
+def set_cli_config(config: Config) -> None:
+    global _cli_config
+    _cli_config = config
+
+
+def cli_config() -> Config:
+    return _cli_config
+
+
+__all__ = ["Config", "cli_config", "set_cli_config"]

dataproduct/data_product.py

@@ -0,0 +1,93 @@
+"""Core library entry point, parallel to datacontract-cli's ``DataContract``.
+
+from dataproduct.data_product import DataProduct
+
+run = DataProduct(data_product_file="dataproduct.odps.yaml").lint()
+DataProduct(data_product_file="dataproduct.odps.yaml").publish()
+"""
+
+from typing import Optional, Union
+
+from dataproduct.config import Config
+from dataproduct.integration.entropy_data import publish_data_product_to_entropy_data
+from dataproduct.lint.files import read_resource
+from dataproduct.lint.schema import fetch_schema
+from dataproduct.lint.validate import parse_yaml, validate_against_schema
+from dataproduct.model.exceptions import DataProductException
+from dataproduct.model.run import Check, ResultEnum, Run
+
+
+class DataProduct:
+    def __init__(
+        self,
+        data_product_file: Optional[str] = None,
+        data_product_str: Optional[str] = None,
+        schema_location: Optional[str] = None,
+        all_errors: bool = False,
+        config: "Optional[Union[Config, dict]]" = None,
+    ):
+        self._data_product_file = data_product_file
+        self._data_product_str = data_product_str
+        self._schema_location = schema_location
+        self._all_errors = all_errors
+        self._config = Config.resolve(config)
+
+    def _load_dict(self) -> dict:
+        if self._data_product_file is not None:
+            content = read_resource(self._data_product_file, self._config)
+        elif self._data_product_str is not None:
+            content = self._data_product_str
+        else:
+            raise DataProductException(
+                type="lint",
+                name="Load data product",
+                reason="No data product provided (file or string).",
+            )
+        return parse_yaml(content)
+
+    def lint(self) -> Run:
+        """Validate the data product against the ODPS JSON Schema (schema-only)."""
+        run = Run.create_run()
+        run.log_info("Linting data product")
+        try:
+            data = self._load_dict()
+            run.dataProductId = data.get("id")
+            run.dataProductVersion = data.get("version")
+            schema = fetch_schema(self._schema_location)
+            checks = validate_against_schema(data, schema, self._all_errors)
+            if checks:
+                run.checks.extend(checks)
+                for check in checks:
+                    run.log_error(str(check.reason))
+            else:
+                run.checks.append(
+                    Check(
+                        type="lint",
+                        result=ResultEnum.passed,
+                        name="Data product is syntactically valid",
+                    )
+                )
+        except DataProductException as e:
+            run.checks.append(Check(type=e.type, result=e.result, name=e.name, reason=e.reason, engine=e.engine))
+            run.log_error(str(e))
+        except Exception as e:
+            run.checks.append(
+                Check(
+                    type="general",
+                    result=ResultEnum.error,
+                    name="Check Data Product",
+                    reason=str(e),
+                )
+            )
+            run.log_error(str(e))
+        run.finish()
+        return run
+
+    def publish(self, ssl_verification: bool = True) -> None:
+        """Publish the data product to Entropy Data (no client-side lint in 0.1)."""
+        data = self._load_dict()
+        publish_data_product_to_entropy_data(
+            data_product_dict=data,
+            ssl_verification=ssl_verification,
+            config=self._config,
+        )

dataproduct/init/init_template.py

@@ -0,0 +1,26 @@
+import importlib.resources as resources
+import logging
+
+import requests
+
+DEFAULT_DATA_PRODUCT_INIT_TEMPLATE = "odps-1.0.0.init.yaml"
+
+
+def get_init_template(location: str = None) -> str:
+    """Return the contents of an init template.
+
+    - ``None`` -> the bundled default template.
+    - an ``http(s)://`` URL -> the fetched body.
+    - anything else -> read as a local file path.
+    """
+    if location is None:
+        logging.info("Use default bundled template " + DEFAULT_DATA_PRODUCT_INIT_TEMPLATE)
+        schemas = resources.files("dataproduct")
+        template = schemas.joinpath("schemas", DEFAULT_DATA_PRODUCT_INIT_TEMPLATE)
+        with template.open("r") as file:
+            return file.read()
+    elif location.startswith("http://") or location.startswith("https://"):
+        return requests.get(location).text
+    else:
+        with open(location, "r") as file:
+            return file.read()

dataproduct/integration/entropy_data.py

@@ -0,0 +1,87 @@
+"""Publish a data product to Entropy Data.
+
+Confirmed API contract (see specs/003-publish.md):
+``PUT {host}/api/dataproducts/{id}`` with ``x-api-key`` +
+``content-type: application/json`` and a JSON ODPS body; ``200 OK`` on success.
+Structurally identical to datacontract-cli's data-contract publish, with the
+path changed from ``datacontracts`` to ``dataproducts``.
+"""
+
+from urllib.parse import urlparse
+
+import requests
+
+from dataproduct.config import Config
+
+# Response header carrying the HTML location of the published data product.
+RESPONSE_HEADER_LOCATION_HTML = "location-html"
+
+
+def publish_data_product_to_entropy_data(
+    data_product_dict: dict, ssl_verification: bool, config: Config | None = None
+) -> None:
+    config = Config.resolve(config)
+    api_key = _get_api_key(config)
+    host = _get_host(config)
+    headers = {"Content-Type": "application/json", "x-api-key": api_key}
+
+    id = data_product_dict.get("id")
+    if not id:
+        raise DataProductPublishError("The data product has no top-level 'id'; cannot publish.")
+
+    url = f"{host}/api/dataproducts/{id}"
+    response = requests.put(
+        url=url,
+        json=data_product_dict,
+        headers=headers,
+        verify=ssl_verification,
+    )
+    if response.status_code != 200:
+        display_host = _extract_hostname(host)
+        raise DataProductPublishError(f"Error publishing data product to {display_host}: {response.text}")
+
+    print("✅ Published data product successfully")
+
+    location_html = response.headers.get(RESPONSE_HEADER_LOCATION_HTML)
+    if location_html:
+        print(f"🚀 Open {location_html}")
+
+
+class DataProductPublishError(Exception):
+    """Raised when a publish attempt fails (missing key, bad id, non-200)."""
+
+
+def _get_api_key(config: Config) -> str:
+    """API key with fallback priority:
+
+    1. ``ENTROPY_DATA_API_KEY``
+    2. ``DATAMESH_MANAGER_API_KEY``
+    3. ``DATACONTRACT_MANAGER_API_KEY``
+    """
+    api_key = (
+        config.get_entropy_data_api_key()
+        or config.get_datamesh_manager_api_key()
+        or config.get_datacontract_manager_api_key()
+    )
+    if api_key is None:
+        raise DataProductPublishError(
+            "Cannot publish, as neither ENTROPY_DATA_API_KEY, DATAMESH_MANAGER_API_KEY, "
+            "nor DATACONTRACT_MANAGER_API_KEY is set"
+        )
+    return api_key
+
+
+def _get_host(config: Config) -> str:
+    """Host with fallback priority: ENTROPY_DATA_HOST, DATAMESH_MANAGER_HOST,
+    DATACONTRACT_MANAGER_HOST, then the default ``https://api.entropy-data.com``."""
+    return (
+        config.get_entropy_data_host()
+        or config.get_datamesh_manager_host()
+        or config.get_datacontract_manager_host()
+        or "https://api.entropy-data.com"
+    )
+
+
+def _extract_hostname(url: str) -> str:
+    parsed = urlparse(url)
+    return parsed.netloc.split(":")[0] if parsed.netloc else url

dataproduct/lint/files.py

@@ -0,0 +1,38 @@
+import os
+from pathlib import Path
+
+import requests
+
+from dataproduct.config import Config
+from dataproduct.model.exceptions import DataProductException
+from dataproduct.model.run import ResultEnum
+
+
+def read_resource(location: str, config: Config | None = None) -> str:
+    """Read the raw text of a data product from a URL or local path.
+
+    Raises :class:`DataProductException` (an ``error`` result, no traceback) if a
+    local file is missing or a URL cannot be fetched.
+    """
+    location_str = str(location)
+    if location_str.startswith("http://") or location_str.startswith("https://"):
+        try:
+            response = requests.get(location_str)
+            response.raise_for_status()
+        except requests.RequestException as e:
+            raise DataProductException(
+                type="lint",
+                name=f"Reading data product from {location_str}",
+                reason=f"Failed to fetch '{location_str}': {e}",
+                result=ResultEnum.error,
+            )
+        return response.text
+
+    if not os.path.exists(location_str):
+        raise DataProductException(
+            type="lint",
+            name=f"Reading data product from {location_str}",
+            reason=f"The file '{location_str}' does not exist.",
+            result=ResultEnum.error,
+        )
+    return Path(location_str).read_text()

dataproduct/lint/resolve.py

@@ -0,0 +1,11 @@
+from typing import Any, Dict
+
+from dataproduct.config import Config
+from dataproduct.lint.files import read_resource
+from dataproduct.lint.validate import parse_yaml
+
+
+def resolve_data_product_dict(location: str, config: "Config | None" = None) -> Dict[str, Any]:
+    """Read and parse the data product at ``location`` into a plain dict."""
+    content = read_resource(location, config)
+    return parse_yaml(content)

dataproduct/lint/schema.py

@@ -0,0 +1,44 @@
+import importlib.resources as resources
+import json
+import logging
+import os
+from pathlib import Path
+from typing import Any, Dict, Union
+
+import requests
+
+from dataproduct.model.exceptions import DataProductException
+from dataproduct.model.run import ResultEnum
+
+DEFAULT_DATA_PRODUCT_SCHEMA = "odps-1.0.0.schema.json"
+
+
+def fetch_schema(location: Union[str, Path] = None) -> Dict[str, Any]:
+    """Fetch the ODPS JSON Schema to validate against.
+
+    ``None`` uses the bundled ODPS v1.0.0 schema; otherwise ``location`` is a URL
+    or local path.
+    """
+    if location is None:
+        logging.info("Use default bundled schema " + DEFAULT_DATA_PRODUCT_SCHEMA)
+        schemas = resources.files("dataproduct")
+        schema_file = schemas.joinpath("schemas", DEFAULT_DATA_PRODUCT_SCHEMA)
+        with schema_file.open("r") as file:
+            return json.load(file)
+
+    location_str = str(location)
+    if location_str.startswith("http://") or location_str.startswith("https://"):
+        logging.debug(f"Downloading schema from {location_str}")
+        response = requests.get(location_str)
+        return response.json()
+
+    if not os.path.exists(location_str):
+        raise DataProductException(
+            type="lint",
+            name=f"Reading schema from {location_str}",
+            reason=f"The file '{location_str}' does not exist.",
+            result=ResultEnum.error,
+        )
+    logging.debug(f"Loading JSON schema locally at {location_str}")
+    with open(location_str, "r") as file:
+        return json.load(file)

dataproduct/lint/validate.py

@@ -0,0 +1,61 @@
+from typing import Any, Dict, List
+
+import yaml
+from jsonschema.validators import validator_for
+
+from dataproduct.model.exceptions import DataProductException
+from dataproduct.model.run import Check, ResultEnum
+
+
+def parse_yaml(content: str) -> Dict[str, Any]:
+    """Parse a data product YAML string into a mapping.
+
+    Raises :class:`DataProductException` (single ``error``, no traceback) on
+    malformed YAML or a non-mapping document.
+    """
+    try:
+        data = yaml.safe_load(content)
+    except yaml.YAMLError as e:
+        raise DataProductException(
+            type="lint",
+            name="Parsing data product YAML",
+            reason=f"The data product is not valid YAML: {e}",
+            result=ResultEnum.error,
+        )
+    if not isinstance(data, dict):
+        raise DataProductException(
+            type="lint",
+            name="Parsing data product YAML",
+            reason="The data product must be a YAML mapping (object).",
+            result=ResultEnum.error,
+        )
+    return data
+
+
+def validate_against_schema(data: Dict[str, Any], schema: Dict[str, Any], all_errors: bool = False) -> List[Check]:
+    """Validate ``data`` against the ODPS JSON Schema.
+
+    Returns a list of ``error`` checks — empty when the document is valid. With
+    ``all_errors=False`` (default) only the first violation is reported.
+    """
+    validator_cls = validator_for(schema)
+    validator_cls.check_schema(schema)
+    validator = validator_cls(schema)
+
+    errors = sorted(validator.iter_errors(data), key=lambda e: list(e.absolute_path))
+    if not all_errors:
+        errors = errors[:1]
+
+    checks: List[Check] = []
+    for error in errors:
+        path = "/".join(str(p) for p in error.absolute_path) or "(root)"
+        checks.append(
+            Check(
+                type="lint",
+                result=ResultEnum.error,
+                name=f"Schema validation failed at '{path}'",
+                reason=error.message,
+                field=path,
+            )
+        )
+    return checks

dataproduct/model/__init__.py

@@ -0,0 +1,4 @@
+from dataproduct.model.exceptions import DataProductException
+from dataproduct.model.run import Check, ResultEnum, Run
+
+__all__ = ["Check", "DataProductException", "ResultEnum", "Run"]