varda 0.1.0__py3-none-any.whl

varda/__init__.py

@@ -0,0 +1,29 @@
+"""Varda — dimensional modeling for LinkML.
+
+A profile of LinkML that adds the vocabulary of dimensional modeling — facts,
+dimensions, grain, additivity, slowly-changing dimensions — plus the rules
+that check a model against it and the generators that build from it.
+
+Third parties import :mod:`varda.ext` and nothing else. Everything reachable
+from there is public and versioned; everything else is internal and moves
+without notice.
+"""
+
+from __future__ import annotations
+
+__version__ = "0.1.0"
+
+from .ext import Context, Extension, ExtensionError, Generator
+from .model import DimensionalModel
+from .rules import Finding, RuleSet
+
+__all__ = [
+    "Context",
+    "DimensionalModel",
+    "Extension",
+    "ExtensionError",
+    "Finding",
+    "Generator",
+    "RuleSet",
+    "__version__",
+]

varda/__main__.py

@@ -0,0 +1,8 @@
+"""Allow ``python -m varda``."""
+
+from __future__ import annotations
+
+from .cli import main
+
+if __name__ == "__main__":
+    raise SystemExit(main())

varda/anns.py

@@ -0,0 +1,103 @@
+"""Annotation access.
+
+LinkML stores annotations as ``JsonObj`` in some code paths and plain dicts in
+others, and the two do not share an interface. Every read of an annotation in
+this package goes through :func:`anns`, so that inconsistency is handled in
+exactly one place.
+
+Reads are namespaced. A :class:`Reader` is bound to one prefix and sees only
+that prefix's annotations, which is what lets Varda and a third-party
+extension annotate the same class without either guessing at the other's
+vocabulary. Varda's own reader is :data:`varda`.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any
+
+from jsonasobj2 import items as _ja_items
+
+
+def anns(obj: Any) -> dict[str, Any]:
+    """Return an object's annotations as a plain ``{tag: value}`` dict."""
+    declared = getattr(obj, "annotations", None)
+    if not declared:
+        return {}
+    out: dict[str, Any] = {}
+    for tag, ann in _ja_items(declared):
+        value = (
+            ann.get("value")
+            if isinstance(ann, dict)
+            else getattr(ann, "value", ann)
+        )
+        out[str(tag)] = value
+    return out
+
+
+@dataclass(frozen=True)
+class Reader:
+    """Reads one namespace's annotations off a LinkML object.
+
+    The prefix *is* the namespace, and the match is exact: a reader for
+    ``acme`` never sees ``varda:role``, and Varda's reader never sees
+    ``acme:cost_center``. Reads are deliberately strict about the prefix — an
+    unprefixed ``role:`` annotation is nobody's, not everybody's, and
+    accepting it as Varda's is how one party's vocabulary silently consumes
+    another's.
+    """
+
+    prefix: str
+
+    @property
+    def tag(self) -> str:
+        """The prefix with its colon, as it appears on an annotation."""
+        return f"{self.prefix}:"
+
+    def raw(self, obj: Any, key: str) -> Any:
+        """Read one annotation, with or without the prefix on ``key``.
+
+        Returns the value as LinkML stored it. Only the converters below
+        should need this; everything else wants :meth:`get`.
+        """
+        full = key if key.startswith(self.tag) else self.tag + key
+        return anns(obj).get(full)
+
+    def get(self, obj: Any, key: str) -> str | None:
+        """Read one annotation as a string.
+
+        The coercion is the point. A value arrives as whatever the YAML parser
+        made of it — ``16`` is an int, ``true`` is a bool — and every caller
+        here wants a string. Doing it once means the accessors in
+        :mod:`varda.model` can promise ``str | None`` and be telling the
+        truth, rather than promising it while handing back ``Any``.
+        """
+        value = self.raw(obj, key)
+        return None if value is None else str(value)
+
+    def present(self, obj: Any) -> bool:
+        """Flag whether the object carries any annotation in this namespace."""
+        return any(k.startswith(self.tag) for k in anns(obj))
+
+    def keys(self, obj: Any) -> list[str]:
+        """List this namespace's annotation names, prefix stripped."""
+        n = len(self.tag)
+        return sorted(k[n:] for k in anns(obj) if k.startswith(self.tag))
+
+
+#: Varda's own reader.
+varda = Reader("varda")
+
+get = varda.get
+
+
+def is_model_object(obj: Any) -> bool:
+    """Flag whether an object participates in the dimensional model.
+
+    Deliberately *not* prefix-aware. A class is part of the model because
+    Varda says it is, and a class carrying only ``acme:`` annotations is a
+    class an extension has decorated — not one it has enrolled. Making this
+    prefix-aware would let an extension pull arbitrary classes into the model
+    by annotating them, which is exactly the authority extensions do not have.
+    """
+    return varda.present(obj)

varda/cli.py

@@ -0,0 +1,277 @@
+"""The command line.
+
+Five commands, each doing one thing. Exit codes are part of the contract
+because these run in CI: ``0`` success, ``1`` the model or the run failed,
+``2`` the invocation was wrong. A tool that returns ``0`` for "I could not
+tell" is one that turns a red build green.
+"""
+
+from __future__ import annotations
+
+import argparse
+import pathlib
+import sys
+from typing import TYPE_CHECKING
+
+from . import registry
+from .ext import Context, ExtensionError
+from .model import DimensionalModel
+from .rules import all_rules, check, unknown_codes
+
+if TYPE_CHECKING:
+    from collections.abc import Sequence
+
+    from .ext import Generator
+
+EXIT_OK = 0
+EXIT_FAIL = 1
+EXIT_USAGE = 2
+
+
+def _load(path: str) -> DimensionalModel:
+    """Load a model, resolving symbolic profile imports."""
+    return DimensionalModel.load(path, importmap=registry.importmap())
+
+
+def _extensions_line() -> str:
+    """Name the active extensions, for the header of a run."""
+    names = [
+        f"{active.name} {active.version}" for active in registry.extensions()
+    ]
+    return ", ".join(names)
+
+
+# ---------------------------------------------------------------------------
+# check
+# ---------------------------------------------------------------------------
+
+
+def cmd_check(args: argparse.Namespace) -> int:
+    """Validate a model against every active extension's rules."""
+    model = _load(args.model)
+    exempt = [*registry.exemptions(), *(args.exempt or [])]
+
+    stale = unknown_codes(exempt)
+    if stale:
+        where = "; ".join(stale)
+        print(
+            f"warning: exemption names no registered rule: {where}",
+            file=sys.stderr,
+        )
+        if args.strict:
+            return EXIT_FAIL
+
+    findings = check(model, exemptions=exempt)
+    for finding in findings:
+        print(finding)
+
+    errors = sum(1 for f in findings if f.severity == "error")
+    warnings = sum(1 for f in findings if f.severity == "warning")
+    tables = len(model.tables)
+    print(
+        f"\n{tables} tables checked against {len(all_rules())} rules "
+        f"({_extensions_line()}): {errors} errors, {warnings} warnings"
+    )
+    if errors or (args.strict and warnings):
+        return EXIT_FAIL
+    return EXIT_OK
+
+
+# ---------------------------------------------------------------------------
+# generate
+# ---------------------------------------------------------------------------
+
+
+def _selected(only: Sequence[str] | None) -> list[Generator]:
+    """Pick the generators to run, or raise on a name nobody registers."""
+    available = registry.generators()
+    if not only:
+        return list(available)
+    by_name = {g.name: g for g in available}
+    missing = [n for n in only if n not in by_name]
+    if missing:
+        known = ", ".join(sorted(by_name)) or "none"
+        msg = f"unknown generator(s): {', '.join(missing)}. Known: {known}"
+        raise KeyError(msg)
+    return [by_name[n] for n in only]
+
+
+def cmd_generate(args: argparse.Namespace) -> int:
+    """Run generators and write their output.
+
+    Two phases on purpose. Every generator runs and every result is collected
+    before a single file is written, so a generator that raises half way
+    through leaves no partial output tree behind. A half-generated estate is
+    worse than none: it looks complete, and the parts that are stale are the
+    parts nobody thinks to check.
+    """
+    model = _load(args.model)
+    try:
+        chosen = _selected(args.only)
+    except KeyError as exc:
+        print(str(exc).strip("'\""), file=sys.stderr)
+        return EXIT_USAGE
+    if not chosen:
+        print("no generators registered", file=sys.stderr)
+        return EXIT_USAGE
+
+    ctx = Context(model=model, source=model.source, schema=args.schema)
+    collected: dict[str, str] = {}
+    for gen in chosen:
+        try:
+            produced = gen.run(ctx)
+        except Exception as exc:  # noqa: BLE001
+            # Deliberately broad. A third-party generator may raise anything,
+            # and the CLI's job at this point is to fail closed with a legible
+            # message naming the culprit, rather than to hand the operator a
+            # traceback through somebody else's code. Nothing has been written
+            # yet, which is the whole reason for collecting before writing.
+            print(
+                f"{gen.name} failed: {type(exc).__name__}: {exc}\n"
+                f"nothing was written",
+                file=sys.stderr,
+            )
+            return EXIT_FAIL
+        undeclared = sorted(set(produced) - set(gen.artifacts))
+        if undeclared:
+            print(
+                f"{gen.name} wrote undeclared path(s): {', '.join(undeclared)}",
+                file=sys.stderr,
+            )
+            return EXIT_FAIL
+        collected.update(produced)
+
+    out = pathlib.Path(args.out)
+    for rel, content in sorted(collected.items()):
+        target = out / rel
+        target.parent.mkdir(parents=True, exist_ok=True)
+        target.write_text(content, encoding="utf-8")
+        print(f"wrote {target}")
+    print(f"\n{len(collected)} artifacts from {len(chosen)} generators")
+    return EXIT_OK
+
+
+# ---------------------------------------------------------------------------
+# The rules, ext and importmap commands
+# ---------------------------------------------------------------------------
+
+
+def cmd_rules(args: argparse.Namespace) -> int:
+    """List every registered rule."""
+    overrides = registry.severities()
+    for code, severity, title, fn in all_rules():
+        effective = overrides.get(code, severity)
+        mark = "*" if effective != severity else " "
+        print(f"{code}  {effective:<7}{mark} {title}")
+        if args.verbose and fn.__doc__:
+            for line in fn.__doc__.strip().splitlines():
+                print(f"          {line.strip()}")
+            print()
+    print(f"\n{len(all_rules())} rules ({_extensions_line()})")
+    return EXIT_OK
+
+
+def cmd_ext(args: argparse.Namespace) -> int:
+    """Describe the active extensions."""
+    for active in registry.extensions():
+        if args.name and active.name != args.name:
+            continue
+        print(f"{active.name} {active.version}  [{active.prefix}:]")
+        print(f"    origin      {active.origin or 'unknown'}")
+        print(f"    rule tag    {active.rule_tag}")
+        n_rules = len(active.rules.rules) if active.rules else 0
+        print(f"    rules       {n_rules}")
+        if active.profile:
+            print(f"    profile     {active.profile}")
+            for target in registry.TARGETS:
+                tags = sorted(
+                    t
+                    for t in registry.declared_annotations(target)
+                    if t.startswith(f"{active.prefix}:")
+                )
+                if tags:
+                    print(f"    {target:<11} {', '.join(tags)}")
+        if active.generators:
+            names = ", ".join(g.name for g in active.generators)
+            print(f"    generators  {names}")
+        print()
+    return EXIT_OK
+
+
+def cmd_importmap(_: argparse.Namespace) -> int:
+    """Print the map that resolves symbolic profile imports."""
+    for prefix, path in sorted(registry.importmap().items()):
+        print(f"{prefix}={path}")
+    return EXIT_OK
+
+
+# ---------------------------------------------------------------------------
+# Wiring
+# ---------------------------------------------------------------------------
+
+
+def build_parser() -> argparse.ArgumentParser:
+    """Build the argument parser."""
+    parser = argparse.ArgumentParser(
+        prog="varda",
+        description="Dimensional modeling for LinkML.",
+    )
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    p_check = sub.add_parser("check", help="validate a model")
+    p_check.add_argument("model")
+    p_check.add_argument(
+        "--exempt",
+        action="append",
+        metavar="CODE",
+        help="skip a rule; repeatable",
+    )
+    p_check.add_argument(
+        "--strict",
+        action="store_true",
+        help="fail on warnings and on exemptions that name no rule",
+    )
+    p_check.set_defaults(fn=cmd_check)
+
+    p_gen = sub.add_parser("generate", help="write artifacts from a model")
+    p_gen.add_argument("model")
+    p_gen.add_argument("--out", default="out", help="output directory")
+    p_gen.add_argument("--schema", default="mart", help="SQL schema name")
+    p_gen.add_argument(
+        "--only",
+        action="append",
+        metavar="NAME",
+        help="run only this generator; repeatable",
+    )
+    p_gen.set_defaults(fn=cmd_generate)
+
+    p_rules = sub.add_parser("rules", help="list conformance rules")
+    p_rules.add_argument("-v", "--verbose", action="store_true")
+    p_rules.set_defaults(fn=cmd_rules)
+
+    p_ext = sub.add_parser("ext", help="describe active extensions")
+    p_ext.add_argument("name", nargs="?")
+    p_ext.set_defaults(fn=cmd_ext)
+
+    p_map = sub.add_parser("importmap", help="print the LinkML import map")
+    p_map.set_defaults(fn=cmd_importmap)
+
+    return parser
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    """Entry point."""
+    args = build_parser().parse_args(argv)
+    try:
+        result: int = args.fn(args)
+    except ExtensionError as exc:
+        print(f"extension error: {exc}", file=sys.stderr)
+        return EXIT_FAIL
+    except FileNotFoundError as exc:
+        print(f"no such file: {exc.filename}", file=sys.stderr)
+        return EXIT_USAGE
+    return result
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

varda/ext.py

@@ -0,0 +1,181 @@
+"""The extension interface — the only module a third party imports.
+
+An extension is one party's contribution to Varda: a namespace prefix, a
+LinkML profile declaring the annotations that prefix permits, and optionally
+a set of conformance rules and generators. Varda itself is an instance of
+:class:`Extension` (see :mod:`varda.registry`), which is the test that this
+interface is real rather than a second-class bolt-on: if the core can be
+expressed through it, a third party is not working around anything.
+
+The governing principle is **one party, one namespace; extensions add, they
+never redefine**. An extension may introduce annotations, enumerations and
+rules under its own prefix. It may not add a value to ``TableRole``, change
+what ``semi_additive`` means, or replace a Varda generator — because every
+generator dispatches exhaustively on those closed vocabularies and raises on
+a value it cannot map. Widening one from outside turns that discipline into a
+fallback path. See the note above ``enums:`` in ``profile/varda.yaml``.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+from functools import cached_property
+from typing import TYPE_CHECKING, Any
+
+from linkml_runtime.utils.schemaview import SchemaView
+
+from .anns import Reader
+
+if TYPE_CHECKING:
+    import pathlib
+    from collections.abc import Callable
+
+    from .model import DimensionalModel
+    from .rules import RuleSet
+
+#: A prefix is a namespace, and it becomes a YAML key, a Python identifier and
+#: part of a filename. The intersection of what all three accept is narrow.
+PREFIX_PATTERN = re.compile(r"^[a-z][a-z0-9_]*$")
+
+#: A rule tag is the letters every code in a set begins with: `V` for Varda,
+#: `ACME` for an extension whose prefix is `acme`.
+TAG_PATTERN = re.compile(r"^[A-Z]+$")
+
+Severity = str  # "error" | "warning" | "info"
+
+#: Declared here rather than in `rules`, because the registry validates
+#: severity overrides too, and two copies of a closed three-value vocabulary
+#: is exactly the drift this design exists to prevent.
+SEVERITIES = frozenset({"error", "warning", "info"})
+
+
+class ExtensionError(Exception):
+    """An extension is malformed, or collides with another one.
+
+    Always raised, never warned. The cost of raising is a clear error at
+    startup; the cost of warning is an estate that half-works for two years
+    because nobody reads startup warnings.
+    """
+
+
+@dataclass(frozen=True)
+class Context:
+    """Everything a generator is given, and nothing more.
+
+    Deliberately a closed record rather than the CLI's argument namespace. A
+    generator that can reach the CLI's state ends up depending on flags, and
+    then its output is a function of how it was invoked rather than of the
+    model — which is the property the whole generate-and-compare arrangement
+    depends on.
+    """
+
+    model: DimensionalModel
+    source: pathlib.Path
+    schema: str = "mart"
+
+
+@dataclass(frozen=True, eq=False)
+class Generator:
+    """A named producer of files, declaring what it will write.
+
+    ``run`` returns ``{relative path: content}`` and never touches the disk.
+    That is what makes the fail-closed guarantee in :func:`varda.cli.generate`
+    possible: every generator is run and every result collected before
+    anything is written, so a failure half way through leaves no half-written
+    output tree behind.
+
+    Paths are declared up front, in ``artifacts``, so that two extensions
+    claiming the same output file are caught at startup rather than by one
+    silently overwriting the other's work.
+
+    ``eq=False`` gives identity comparison, because ``run`` is a function and
+    two distinct generators wrapping the same function are still two
+    generators.
+    """
+
+    name: str
+    artifacts: tuple[str, ...]
+    run: Callable[[Context], dict[str, str]]
+
+
+@dataclass(frozen=True, eq=False)
+class Extension:
+    """One party's contribution to Varda.
+
+    The only required fields are ``name`` and ``prefix``. An extension that
+    declares nothing but a profile is legal and useful: it adds vocabulary
+    that ``varda check`` will then accept, which is the smallest thing an
+    organization typically wants.
+
+    ``eq=False`` because ``rules`` holds a mutable :class:`RuleSet`, which
+    makes the dataclass unhashable under the default ``eq``; identity is the
+    right comparison for a registered singleton anyway.
+    """
+
+    name: str
+    prefix: str
+    version: str = "0"
+
+    #: Path to a LinkML schema declaring this extension's annotations. Its
+    #: `default_prefix` must equal `prefix` — the registry checks, because a
+    #: profile that declares annotations under a prefix nobody reads is a
+    #: vocabulary that silently never applies.
+    profile: pathlib.Path | None = None
+
+    #: Conformance rules. Every code must begin with `rule_tag`.
+    rules: RuleSet | None = None
+
+    #: Defaults to `prefix.upper()`. Varda's is `V`, not by special-casing
+    #: but because tag uniqueness is checked across all extensions and Varda
+    #: registers first.
+    rule_tag: str = ""
+
+    #: `{rule code: severity}` — this extension's opinion about the severity
+    #: of a rule, including one it does not own. Two extensions disagreeing
+    #: about one code is refused rather than resolved; see the registry.
+    severity_defaults: dict[str, Severity] = field(default_factory=dict)
+
+    #: The importable package name, used to resolve the profile for the
+    #: LinkML import map so a domain model can write `imports: - acme`.
+    package: str = ""
+
+    generators: tuple[Generator, ...] = ()
+
+    #: Where this extension was found — entry point, config file, or
+    #: injection. Excluded from comparison because it is diagnostic only.
+    origin: str = field(default="", compare=False)
+
+    def __post_init__(self) -> None:
+        """Derive the rule tag from the prefix when it was not given."""
+        if not self.rule_tag:
+            object.__setattr__(self, "rule_tag", self.prefix.upper())
+
+    @staticmethod
+    def reader(prefix: str) -> Reader:
+        """Build a reader for a namespace.
+
+        Exposed here so an extension never imports :mod:`varda.anns`: the
+        whole public surface is this module, and a third party reading its
+        own annotations is the most common thing it will do.
+        """
+        return Reader(prefix)
+
+    @property
+    def anns(self) -> Reader:
+        """A reader bound to this extension's own prefix."""
+        return Reader(self.prefix)
+
+    @cached_property
+    def profile_view(self) -> SchemaView | None:
+        """The parsed profile, or ``None`` if this extension declares none."""
+        return None if self.profile is None else SchemaView(str(self.profile))
+
+    @cached_property
+    def profile_version(self) -> str | None:
+        """The version string the profile declares, if it declares one."""
+        view = self.profile_view
+        if view is None:
+            return None
+        version: Any = view.schema.version
+        return None if version is None else str(version)

varda/gen_docs.py

@@ -0,0 +1,90 @@
+"""Documentation generation.
+
+Emits one Markdown reference for the whole model. The audience is an analyst
+deciding whether a table answers their question, so the ordering is facts
+first — a fact is what someone starts from — and the grain sentence is given
+the most prominent position it can have.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from .ext import Context
+    from .model import DimensionalModel, Table
+
+ADDITIVITY = {
+    "additive": "sums across every dimension",
+    "semi_additive": "does not sum across",
+    "non_additive": "never sum; recompute from components",
+}
+
+
+def _columns(table: Table) -> list[str]:
+    rows = [
+        "| Column | Role | Type | Notes |",
+        "| --- | --- | --- | --- |",
+    ]
+    for column in table.columns:
+        notes: list[str] = []
+        if column.references:
+            notes.append(f"→ `{column.references}`")
+        if column.additivity:
+            phrase = ADDITIVITY.get(column.additivity, column.additivity)
+            if column.additivity == "semi_additive" and (
+                column.semi_additive_over
+            ):
+                phrase = f"{phrase} `{column.semi_additive_over}`"
+            notes.append(phrase)
+        if column.unit:
+            notes.append(f"unit: `{column.unit}`")
+        if column.description:
+            notes.append(column.description)
+        rows.append(
+            f"| `{column.physical}` | {column.role or '—'} | "
+            f"{column.range} | {'; '.join(notes) or '—'} |"
+        )
+    return rows
+
+
+def _table(table: Table) -> str:
+    lines = [f"### {table.name}", ""]
+    if table.description:
+        lines += [table.description, ""]
+    facts = [f"**Physical name:** `{table.physical}`"]
+    if table.grain:
+        facts.append(f"**Grain:** {table.grain}")
+    if table.fact_type:
+        facts.append(f"**Fact type:** {table.fact_type}")
+    if table.scd:
+        facts.append(f"**Slowly-changing:** {table.scd}")
+    lines += [*facts, "", *_columns(table), ""]
+    return "\n".join(lines)
+
+
+def generate(model: DimensionalModel) -> str:
+    """Render the model as a Markdown reference."""
+    out = [
+        "# Model reference",
+        "",
+        "Generated by varda. Do not edit.",
+        "",
+        f"Source: `{model.source.name}`",
+        "",
+    ]
+    for heading, tables in (
+        ("Facts", model.facts),
+        ("Dimensions", model.dimensions),
+        ("Bridges", model.bridges),
+    ):
+        if not tables:
+            continue
+        out += [f"## {heading}", ""]
+        out += [_table(t) for t in tables]
+    return "\n".join(out).rstrip() + "\n"
+
+
+def run(ctx: Context) -> dict[str, str]:
+    """Run this generator and return its artifacts."""
+    return {"docs/model.md": generate(ctx.model)}