fluvv 1.0.0__py3-none-any.whl

fluvv/__init__.py

@@ -0,0 +1,23 @@
+"""Fluvv: source + pipeline config on top of vanilla ODCS contracts.
+
+See docs/fluvv/SPEC.md for the spec this package implements.
+"""
+
+from fluvv.contract import load_extension
+from fluvv.export import ExportResult, export_artifacts
+from fluvv.lint import LintResult, lint
+from fluvv.models import IngestionConfig, FluvvExtension, Source, Transform
+from fluvv.product import Contract
+
+__all__ = [
+    "Contract",
+    "ExportResult",
+    "IngestionConfig",
+    "LintResult",
+    "FluvvExtension",
+    "Source",
+    "Transform",
+    "export_artifacts",
+    "lint",
+    "load_extension",
+]

fluvv/__main__.py

@@ -0,0 +1,4 @@
+from fluvv.cli import app
+
+if __name__ == "__main__":
+    app()

fluvv/cli.py

@@ -0,0 +1,152 @@
+from __future__ import annotations
+
+from importlib.metadata import entry_points, version
+from pathlib import Path
+from typing import Callable
+
+import typer
+
+from fluvv.contract import RepoRootNotFoundError
+from fluvv.export import export_artifacts
+from fluvv.lint import lint
+from fluvv.manifest import DEFAULT_PATH as DEFAULT_MANIFEST_PATH, write_manifest
+from fluvv.semantic import ProjectCheckError
+
+# Installed packages add commands to the `fluvv` CLI by registering a
+# `register(app)` function under this entry-point group. Fluvv Engine adds
+# ingest and the OpenMetadata commands this way, so the open package never
+# imports it.
+PLUGIN_GROUP = "fluvv.commands"
+
+# Fluvv Engine's commands. Listed only so that, without the engine, they
+# say where they come from instead of failing as unknown commands.
+ENGINE_COMMANDS = ("ingest", "push-openmetadata", "prune-openmetadata", "setup-openmetadata")
+
+
+def _yaml_files(path: Path) -> list[Path]:
+    """What linting `path` means. A file is itself. A repository root (it
+    holds contracts/ or governance/) is the YAML in those two folders, so
+    `fluvv lint .` lints the project and nothing else in it. Any other
+    folder is every YAML file below it, skipping hidden folders like .venv.
+    The generated files under contracts/ are .py and .json, so they're
+    never picked up."""
+    if path.is_file():
+        return [path]
+    roots = [path / top for top in ("contracts", "governance") if (path / top).is_dir()] or [path]
+    return sorted(
+        found
+        for root in roots
+        for found in root.rglob("*.yaml")
+        if not any(part.startswith(".") for part in found.relative_to(root).parts[:-1])
+    )
+
+
+def lint_command(
+    paths: list[Path] = typer.Argument(
+        None,
+        exists=True,
+        help="Contracts, governance files (Domain, Glossary, ODPS DataProduct), or folders of them. "
+        "Default: the repository in the current directory.",
+    ),
+) -> None:
+    """Lint contracts and governance files, each on its own. Exits non-zero
+    if any fail. References between files are checked by `compile`."""
+    all_passed = True
+    for path in paths or [Path(".")]:
+        files = _yaml_files(path)
+        if not files:
+            typer.echo(f"{path}: no .yaml files to lint", err=True)
+            all_passed = False
+        for contract_path in files:
+            result = lint(contract_path)
+            typer.echo(result.report())
+            all_passed &= result.passed
+    raise typer.Exit(code=0 if all_passed else 1)
+
+
+def export_command(
+    contract_paths: list[Path] = typer.Argument(..., help="One or more Fluvv contract YAML files."),
+) -> None:
+    """Export the Spark schema and ingestion_config JSON for one or more
+    contracts. They're written under the repository's contracts/, so each
+    contract must be inside one."""
+    for contract_path in contract_paths:
+        try:
+            result = export_artifacts(contract_path)
+        except RepoRootNotFoundError as exc:
+            typer.echo(str(exc), err=True)
+            raise typer.Exit(code=1)
+        typer.echo(f"{contract_path}:")
+        typer.echo(f"  {result.spark_schema_path}")
+        typer.echo(f"  {result.ingestion_config_path}")
+
+
+def compile_command(
+    out: Path = typer.Option(DEFAULT_MANIFEST_PATH, "--out", help="Where to write the manifest."),
+) -> None:
+    """Compile every contract and governance file under the current
+    directory (the repo root) into one manifest JSON, after checking that
+    each file can be read and their references resolve. Exits non-zero,
+    writing nothing, if not. `lint` checks each file in full."""
+    try:
+        typer.echo(write_manifest(Path("."), out))
+    except ProjectCheckError as exc:
+        typer.echo(exc.report())
+        raise typer.Exit(code=1)
+
+
+def _engine_required(name: str) -> Callable[[], None]:
+    def command() -> None:
+        typer.echo(
+            f"`fluvv {name}` is part of Fluvv Engine, which isn't installed. "
+            "See https://fluvv.se/engine.",
+            err=True,
+        )
+        raise typer.Exit(code=1)
+
+    command.__doc__ = "Part of Fluvv Engine (not installed)."
+    return command
+
+
+def _print_version(show: bool) -> None:
+    # An option callback, so it runs before typer asks for a subcommand.
+    if show:
+        typer.echo(f"fluvv {version('fluvv')}")
+        raise typer.Exit()
+
+
+def _plugin_registrars() -> list[Callable[[typer.Typer], None]]:
+    return [ep.load() for ep in entry_points(group=PLUGIN_GROUP)]
+
+
+def build_app() -> typer.Typer:
+    app = typer.Typer(help="Fluvv tooling: lint, export and compile Fluvv contracts and governance files.")
+
+    @app.callback()
+    def _main(
+        show_version: bool = typer.Option(
+            False, "--version", is_eager=True, callback=_print_version, help="Print the installed fluvv version and exit."
+        ),
+    ) -> None:
+        """(Callback present so `lint` stays an explicit subcommand as more get added.)"""
+
+    app.command(name="lint")(lint_command)
+    app.command(name="export")(export_command)
+    app.command(name="compile")(compile_command)
+    for register in _plugin_registrars():
+        register(app)
+
+    registered = {info.name for info in app.registered_commands}
+    for name in ENGINE_COMMANDS:
+        if name not in registered:
+            app.command(
+                name=name,
+                context_settings={"allow_extra_args": True, "ignore_unknown_options": True},
+            )(_engine_required(name))
+    return app
+
+
+app = build_app()
+
+if __name__ == "__main__":
+    app()

fluvv/contract.py

@@ -0,0 +1,122 @@
+"""Loading a Fluvv contract file and extracting its `sources`/`transform`
+customProperties into the typed structure in fluvv.models."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+from fluvv.models import FluvvExtension
+
+FLUVV_CUSTOM_PROPERTIES = ("sources", "ingestion_config")
+OPTIONAL_FLUVV_CUSTOM_PROPERTIES = ("transform", "businessMetrics")
+"""Validated the same way as FLUVV_CUSTOM_PROPERTIES when present, but
+never raise MissingCustomPropertyError when absent -- a contract that
+doesn't set these is still a complete, valid Fluvv contract. (`transform`
+may only be omitted with a single source; lint checks that.)"""
+
+
+class MissingCustomPropertyError(ValueError):
+    def __init__(self, property_name: str, contract_path: Path):
+        self.property_name = property_name
+        self.contract_path = contract_path
+        super().__init__(
+            f"{contract_path}: missing required customProperties entry '{property_name}'"
+        )
+
+
+class UnresolvableSourceError(ValueError):
+    pass
+
+
+class RepoRootNotFoundError(FileNotFoundError):
+    """A file outside any Fluvv repository was given to something that
+    needs the files around it: resolving upstream sources, or writing the
+    generated files under `contracts/`. Lint never raises it."""
+
+
+def load_odcs_document(contract_path: Path) -> dict[str, Any]:
+    return yaml.safe_load(Path(contract_path).read_text())
+
+
+def yaml_problem(exc: yaml.YAMLError) -> str:
+    """A YAML syntax error in one line, for a report rather than a traceback."""
+    mark, problem = getattr(exc, "problem_mark", None), getattr(exc, "problem", None)
+    if mark is not None and problem:
+        return f"invalid YAML at line {mark.line + 1}: {problem}"
+    return f"invalid YAML: {str(exc).splitlines()[0]}"
+
+
+def find_repo_root(start: Path) -> Path:
+    """The repository root for `start` (SPEC.md Section 1), recognised by
+    the layout, not by fluvv's own files: a contracts repo doesn't
+    contain the package.
+
+    A file where the layout puts it names its root by position --
+    `contracts/<layer>/<domain>/<table>.yaml` or `governance/<kind>/<file>`
+    -- so a domain called `contracts` or `governance` can't be mistaken for
+    the top-level folder. Anything else resolves to the nearest directory
+    holding `contracts/` or `governance/`."""
+    path = start.resolve()
+    for top, depth in (("contracts", 3), ("governance", 2)):
+        if len(path.parents) > depth and path.parents[depth - 1].name == top:
+            return path.parents[depth]
+    for candidate in (path, *path.parents):
+        if (candidate / "contracts").is_dir() or (candidate / "governance").is_dir():
+            return candidate
+    raise RepoRootNotFoundError(
+        f"{start} isn't inside a Fluvv repository: no folder above it holds a "
+        "contracts/ or governance/ directory (SPEC.md Section 1)"
+    )
+
+
+def get_custom_property(doc: dict[str, Any], property_name: str) -> Any | None:
+    for entry in doc.get("customProperties") or []:
+        if entry.get("property") == property_name:
+            return entry.get("value")
+    return None
+
+
+def parse_extension(doc: dict[str, Any], contract_path: Path) -> FluvvExtension:
+    """The extension exactly as written: sources that name a `contract`
+    have no `location`/`format` yet. See resolve_extension()."""
+    values: dict[str, Any] = {}
+    for property_name in FLUVV_CUSTOM_PROPERTIES:
+        value = get_custom_property(doc, property_name)
+        if value is None:
+            raise MissingCustomPropertyError(property_name, contract_path)
+        values[property_name] = value
+    for property_name in OPTIONAL_FLUVV_CUSTOM_PROPERTIES:
+        value = get_custom_property(doc, property_name)
+        if value is not None:
+            values[property_name] = value
+    return FluvvExtension.model_validate(values)
+
+
+def resolve_extension(extension: FluvvExtension, repo_root: Path) -> FluvvExtension:
+    """Fills each contract-backed source's `location`/`format` from that
+    contract's own `servers[0]`, so where a table lives is stated once --
+    in the contract that owns it. Pre-contract sources (`contract: null`)
+    keep what they declare."""
+    sources = []
+    for source in extension.sources:
+        if source.contract is None:
+            if not (source.location and source.format):
+                raise UnresolvableSourceError(f"source '{source.id}': `location` and `format` are required when `contract` is null")
+            sources.append(source)
+            continue
+        upstream_path = Path(repo_root) / source.contract
+        if not upstream_path.is_file():
+            raise UnresolvableSourceError(f"source '{source.id}': contract '{source.contract}' does not exist")
+        server = load_odcs_document(upstream_path)["servers"][0]
+        sources.append(source.model_copy(update={"location": server["path"], "format": server["format"]}))
+    return extension.model_copy(update={"sources": sources})
+
+
+def load_extension(contract_path: Path, resolve: bool = True) -> FluvvExtension:
+    contract_path = Path(contract_path)
+    doc = load_odcs_document(contract_path)
+    extension = parse_extension(doc, contract_path)
+    return resolve_extension(extension, find_repo_root(contract_path)) if resolve else extension

fluvv/export.py

@@ -0,0 +1,55 @@
+"""Generate the artifacts derived from a contract:
+- contracts/spark_schemas/<mirrored path>.py  -- datacontract-cli export("spark")
+- contracts/ingestion_config/<mirrored path>.json -- the sources/transform/
+  ingestion_config customProperties, consumed by the ingestion framework
+  at runtime. Fully resolved: a contract-backed source's location/format
+  are filled in from its upstream contract, so the engine never needs to
+  read another contract.
+
+Each output path mirrors the contract's own path relative to contracts/,
+e.g. contracts/bronze/sales/cars.yaml -> contracts/spark_schemas/bronze/sales/cars.py.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+
+from datacontract.data_contract import DataContract
+
+from fluvv.contract import find_repo_root, load_odcs_document, parse_extension, resolve_extension
+
+
+@dataclass
+class ExportResult:
+    contract_path: Path
+    spark_schema_path: Path
+    ingestion_config_path: Path
+
+
+def _relative_stem(contract_path: Path, repo_root: Path) -> Path:
+    return contract_path.resolve().relative_to((repo_root / "contracts").resolve()).with_suffix("")
+
+
+def export_artifacts(contract_path: Path | str, repo_root: Path | str | None = None) -> ExportResult:
+    contract_path = Path(contract_path)
+    repo_root = Path(repo_root) if repo_root is not None else find_repo_root(contract_path)
+    rel_stem = _relative_stem(contract_path, repo_root)
+
+    data_contract = DataContract(data_contract_file=str(contract_path))
+
+    spark_schema_path = (repo_root / "contracts" / "spark_schemas" / rel_stem).with_suffix(".py")
+    spark_schema_path.parent.mkdir(parents=True, exist_ok=True)
+    spark_schema_path.write_text(data_contract.export("spark"))
+
+    doc = load_odcs_document(contract_path)
+    extension = resolve_extension(parse_extension(doc, contract_path), repo_root)
+    ingestion_config_path = (repo_root / "contracts" / "ingestion_config" / rel_stem).with_suffix(".json")
+    ingestion_config_path.parent.mkdir(parents=True, exist_ok=True)
+    ingestion_config_path.write_text(extension.model_dump_json(by_alias=True, indent=2) + "\n")
+
+    return ExportResult(
+        contract_path=contract_path,
+        spark_schema_path=spark_schema_path,
+        ingestion_config_path=ingestion_config_path,
+    )