ocsf-toolkit 0.1.0a1__py3-none-any.whl

ocsf_toolkit/__init__.py

@@ -0,0 +1,55 @@
+"""Version-pinned OCSF semantics: the schema as data, event building and checking, observables, diff."""
+
+from ocsf_toolkit._version import __version__
+from ocsf_toolkit.codegen import generate
+from ocsf_toolkit.diff import SchemaDiff, diff
+from ocsf_toolkit.engine import (
+    LENIENT,
+    REPAIR,
+    STRICT,
+    ConformanceError,
+    Finding,
+    Kind,
+    Policy,
+    Report,
+    extract,
+)
+from ocsf_toolkit.ir import SchemaIR
+from ocsf_toolkit.models import build, parse, repair
+from ocsf_toolkit.observables import Observable, merge
+from ocsf_toolkit.schema import (
+    ResolvedAttribute,
+    ResolvedItem,
+    ResolvedSchema,
+    load,
+    load_ir,
+    resolve,
+)
+
+__all__ = [
+    "LENIENT",
+    "REPAIR",
+    "STRICT",
+    "ConformanceError",
+    "Finding",
+    "Kind",
+    "Observable",
+    "Policy",
+    "Report",
+    "ResolvedAttribute",
+    "ResolvedItem",
+    "ResolvedSchema",
+    "SchemaDiff",
+    "SchemaIR",
+    "__version__",
+    "build",
+    "diff",
+    "extract",
+    "generate",
+    "load",
+    "load_ir",
+    "merge",
+    "parse",
+    "repair",
+    "resolve",
+]

ocsf_toolkit/_version.py

@@ -0,0 +1,24 @@
+# file generated by vcs-versioning
+# don't change, don't track in version control
+from __future__ import annotations
+
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
+
+version: str
+__version__: str
+__version_tuple__: tuple[int | str, ...]
+version_tuple: tuple[int | str, ...]
+commit_id: str | None
+__commit_id__: str | None
+
+__version__ = version = '0.1.0a1'
+__version_tuple__ = version_tuple = (0, 1, 0, 'a1')
+
+__commit_id__ = commit_id = None

ocsf_toolkit/cli/__init__.py

@@ -0,0 +1,113 @@
+"""The ``ocsf-toolkit`` console script: an argparse root dispatching to one subcommand module."""
+
+import argparse
+import os
+import subprocess
+import sys
+from collections.abc import Sequence
+
+from ocsf_toolkit import __version__
+from ocsf_toolkit.cli import check as check_command
+from ocsf_toolkit.cli import compile as compile_command
+from ocsf_toolkit.cli import diff as diff_command
+from ocsf_toolkit.cli import generate as generate_command
+from ocsf_toolkit.cli import ir as ir_command
+from ocsf_toolkit.cli import lock as lock_command
+from ocsf_toolkit.cli import models as models_command
+
+
+def build_parser() -> argparse.ArgumentParser:
+    """Build the argument parser: the root options and one subparser per command."""
+    # Shared options, each given only to the commands that read it.
+    lockfile = argparse.ArgumentParser(add_help=False)
+    lockfile.add_argument("--lock", default="ocsf.lock", help="lockfile path (default: %(default)s)")
+    build = argparse.ArgumentParser(add_help=False)
+    build.add_argument("--build-dir", default="build", help="checkout cache and output root (default: %(default)s)")
+    compiler = argparse.ArgumentParser(add_help=False)
+    compiler.add_argument(
+        "--compiler-cmd",
+        help="command that compiles a schema checkout to stdout, replacing uvx "
+        "(air-gapped builds); the checkout path is appended",
+    )
+    compiling = [lockfile, build, compiler]
+
+    parser = argparse.ArgumentParser(prog="ocsf-toolkit", description="Version-pinned OCSF schema tooling.")
+    parser.add_argument("--version", action="version", version=f"%(prog)s {__version__}")
+    sub = parser.add_subparsers(dest="command", required=True, metavar="COMMAND")
+    compile_command.register(
+        sub.add_parser(
+            "compile",
+            parents=compiling,
+            help="compile lockfile schemas and check their hashes",
+            description="Clone each pinned commit, compile it with ocsf-schema-compiler under "
+            "Python 3.14, and assert compiled_sha256 and the compiled version.",
+        )
+    )
+    lock_command.register(
+        sub.add_parser(
+            "lock",
+            parents=compiling,
+            help="add an OCSF version to ocsf.lock",
+            description="Resolve an upstream ref, pin its commit, compile it, and record compiled_sha256.",
+        )
+    )
+    ir_command.register(
+        sub.add_parser(
+            "ir",
+            help="export or check canonical IR JSON",
+            description="Build the IR from build/compiled and write or check data/ir-<ver>.json.",
+        ),
+        [lockfile, build],
+    )
+    check_command.register(
+        sub.add_parser(
+            "check",
+            help="report how events deviate from an OCSF version",
+            description="Check each event in JSON Lines files against an OCSF version and print "
+            "its findings; exit 1 if any event has an error.",
+        )
+    )
+    diff_command.register(
+        sub.add_parser(
+            "diff",
+            help="show what changed between two OCSF versions",
+            description="Compare two IRs field by field, matching classes by uid so a renamed "
+            "class shows as a rename; exit 0 if equal, 1 if they differ, 2 on error.",
+        )
+    )
+    generate_command.register(
+        sub.add_parser(
+            "generate",
+            parents=compiling,
+            help="generate a typed msgspec model package",
+            description="Project an OCSF version or IR into a typed package of msgspec Structs: "
+            "superset classes by default, exact classes for a fixed --profiles set.",
+        )
+    )
+    models_command.register(
+        sub.add_parser(
+            "models",
+            help="check or record the generated model trees (regeneration gate)",
+            description="Render each model-window version twice from its shipped IR. check asserts the "
+            "tree is stable and matches models_tree_sha256 in the lockfile; record writes that hash.",
+        ),
+        [lockfile],
+    )
+    return parser
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    """Run one command and return its exit status; an expected failure prints one line to stderr."""
+    args = build_parser().parse_args(argv)
+    try:
+        status: int = args.run(args)
+        sys.stdout.flush()  # a closed pipe shows up here for output that fit in the buffer
+    except BrokenPipeError:
+        # The reader went away (`| head`): not an error of ours. Point stdout at devnull so the
+        # interpreter's own flush at exit doesn't raise again, and stop quietly.
+        os.dup2(os.open(os.devnull, os.O_WRONLY), sys.stdout.fileno())
+        return 1
+    except (ValueError, OSError, subprocess.CalledProcessError) as exc:
+        print(f"ocsf-toolkit: error: {exc}", file=sys.stderr)
+        return 1
+    return status

ocsf_toolkit/cli/_common.py

@@ -0,0 +1,19 @@
+"""Helpers more than one subcommand uses."""
+
+from pathlib import Path
+
+from ocsf_toolkit.ir import SchemaIR
+from ocsf_toolkit.schema import load, load_ir
+
+
+def printable(text: str) -> str:
+    """Escape control characters: paths and messages carry keys taken from the input."""
+    return "".join(c if c.isprintable() else repr(c)[1:-1] for c in text)
+
+
+def source_ir(source: str) -> SchemaIR:
+    """Load a shipped OCSF version by number, or an IR JSON file by path."""
+    path = Path(source)
+    if source.endswith(".json") or path.is_file():
+        return load_ir(path)
+    return load(source).ir

ocsf_toolkit/cli/_upstream.py

@@ -0,0 +1,147 @@
+"""Upstream ocsf-schema: check out a pinned commit and compile it into a verified schema (on 3.14)."""
+
+import hashlib
+import json
+import shlex
+import shutil
+import subprocess
+import sys
+import tempfile
+from collections.abc import Sequence
+from dataclasses import dataclass
+from pathlib import Path
+from typing import cast, final
+
+
+@final
+@dataclass(frozen=True, slots=True)
+class Compiler:
+    """How to run ocsf-schema-compiler: the pinned version under its Python, or a replacement command."""
+
+    version: str  # ocsf-schema-compiler release, from the lockfile's [toolchain]
+    python: str  # the interpreter uvx runs it under
+    cmd: str | None = None  # replaces the uvx invocation (air-gapped builds); the checkout is appended
+
+
+def compiler_argv(src: Path, compiler: Compiler, extensions: Sequence[Path] = ()) -> list[str]:
+    """Return the command line that compiles the schema checkout at ``src`` to JSON on stdout.
+
+    Each of ``extensions`` is a directory of custom extensions, passed as ``-e PATH``.
+    """
+    ext_args = [arg for path in extensions for arg in ("-e", str(path))]
+    if compiler.cmd is not None:
+        return [*shlex.split(compiler.cmd), str(src), *ext_args]
+    return [
+        "uvx",
+        "--quiet",
+        "--python",
+        compiler.python,
+        "--from",
+        f"ocsf-schema-compiler=={compiler.version}",
+        "python",
+        "-m",
+        "ocsf_schema_compiler",
+        str(src),
+        *ext_args,
+    ]
+
+
+def checkout(repo: str, commit: str, cache_dir: Path) -> Path:
+    """Return a checkout of ``repo`` at ``commit``, reusing ``cache_dir/<commit>`` when present."""
+    dest = cache_dir / commit
+    if dest.is_dir():
+        head = _git("rev-parse", "HEAD", cwd=dest)
+        if head != commit:
+            raise ValueError(f"{dest} is a checkout of {head}, not {commit}; delete it and retry")
+        return dest
+    cache_dir.mkdir(parents=True, exist_ok=True)
+    tmp = Path(tempfile.mkdtemp(dir=cache_dir, prefix=f".{commit[:12]}-"))
+    try:
+        _git("init", "--quiet", cwd=tmp)
+        _git("fetch", "--quiet", "--depth", "1", "--", repo, commit, cwd=tmp)
+        _git("-c", "advice.detachedHead=false", "checkout", "--quiet", "FETCH_HEAD", cwd=tmp)
+        tmp.rename(dest)
+    finally:
+        if tmp.exists():
+            shutil.rmtree(tmp)
+    return dest
+
+
+def verify(output: bytes, *, version: str, sha256: str | None) -> tuple[bytes, str]:
+    """Check compiler output against the lockfile; return its canonical bytes and their sha256.
+
+    The compiler emits keys in filesystem directory order (unsorted ``os.walk``), so its raw bytes
+    differ between machines; the hash is over canonical JSON (sorted keys, compact separators,
+    UTF-8), the same rule as the IR export. The compiled ``version`` must equal the declared one:
+    an upstream tag can point at the wrong commit (tag ``1.5.0`` carries ``1.6.0-dev``).
+    ``sha256=None`` skips the hash check, for an entry whose hash is being recorded.
+    """
+    try:
+        compiled = json.loads(output)
+    except ValueError as exc:
+        raise ValueError(f"compiler output is not JSON: {exc}") from exc
+    actual: object = None
+    if isinstance(compiled, dict):
+        actual = cast(dict[str, object], compiled).get("version")
+    if actual != version:
+        raise ValueError(
+            f"compiled schema reports version {actual!r} but the lockfile declares {version!r}; "
+            "the pinned commit is not that release (upstream tag 1.5.0 is a known case)"
+        )
+    canonical = json.dumps(compiled, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8")
+    digest = hashlib.sha256(canonical).hexdigest()
+    if sha256 is not None and digest != sha256:
+        raise ValueError(
+            f"compiled schema {version} has sha256 {digest}, lockfile has {sha256}; compiler output changed"
+        )
+    return canonical, digest
+
+
+def compile_commit(
+    *, repo: str, commit: str, version: str, sha256: str | None, compiler: Compiler, build_dir: Path
+) -> tuple[Path, str]:
+    """Check out, compile and verify one pinned schema; write it to ``build/compiled/``.
+
+    Returns the output path and its sha256. Nothing is written unless verification passes.
+    """
+    data, digest = compiled_output(
+        repo=repo, commit=commit, version=version, sha256=sha256, compiler=compiler, build_dir=build_dir
+    )
+    out = build_dir / "compiled" / f"schema-{version}.json"
+    atomic_write(out, data)
+    return out, digest
+
+
+def compiled_output(
+    *,
+    repo: str,
+    commit: str,
+    version: str,
+    sha256: str | None,
+    compiler: Compiler,
+    build_dir: Path,
+    extensions: Sequence[Path] = (),
+) -> tuple[bytes, str]:
+    """Check out and compile one pinned schema, with any custom extensions.
+
+    Return the verified canonical bytes and their sha256 without writing them anywhere.
+    """
+    print(f"{version}: checkout {repo} @ {commit}", file=sys.stderr)
+    src = checkout(repo, commit, build_dir / "ocsf-schema")
+    argv = compiler_argv(src, compiler, extensions)
+    print(f"{version}: {shlex.join(argv)}", file=sys.stderr)
+    output = subprocess.run(argv, check=True, stdout=subprocess.PIPE).stdout
+    return verify(output, version=version, sha256=sha256)
+
+
+def atomic_write(path: Path, data: bytes) -> None:
+    """Write data to path through a sibling ``.partial`` file, so a reader never sees half of it."""
+    path.parent.mkdir(parents=True, exist_ok=True)
+    partial = path.with_name(path.name + ".partial")
+    partial.write_bytes(data)
+    partial.replace(path)
+
+
+def _git(*args: str, cwd: Path) -> str:
+    result = subprocess.run(["git", *args], cwd=cwd, check=True, stdout=subprocess.PIPE, text=True)
+    return result.stdout.strip()

ocsf_toolkit/cli/check.py

@@ -0,0 +1,77 @@
+"""The ``check`` subcommand: a conformance report for each event in JSON Lines files."""
+
+import argparse
+import contextlib
+import json
+import sys
+from collections.abc import Iterator
+from pathlib import Path
+from typing import Any, TextIO, cast
+
+from ocsf_toolkit.cli._common import printable
+from ocsf_toolkit.models import parse
+from ocsf_toolkit.schema import load
+from ocsf_toolkit.walk import dotted
+
+
+def register(parser: argparse.ArgumentParser) -> None:
+    """Add the ``check`` arguments to its subparser."""
+    parser.add_argument("--version", required=True, help="OCSF version to check against")
+    parser.add_argument(
+        "--profiles",
+        help="comma-separated active profiles (default: each event's metadata.profiles)",
+    )
+    parser.add_argument(
+        "--class",
+        dest="class_key",
+        metavar="KEY",
+        help="class key (default: inferred from class_uid, then type_uid)",
+    )
+    parser.add_argument("--json", action="store_true", help="print one JSON report per event instead of text")
+    parser.add_argument("files", nargs="+", metavar="FILE", help="JSON Lines file of events; - reads stdin")
+    parser.set_defaults(run=run)
+
+
+def run(args: argparse.Namespace) -> int:
+    """Check each event and print its findings; return 1 if any event has an error."""
+    profiles = None if args.profiles is None else [p for p in args.profiles.split(",") if p]
+    try:  # fail once on a bad selection rather than once per event
+        schema = load(args.version, profiles=profiles or ())
+    except KeyError as exc:
+        raise ValueError(exc.args[0]) from None
+    if args.class_key is not None and args.class_key not in schema.classes:
+        raise ValueError(f"no class {args.class_key!r} in OCSF {args.version}")
+    failed = sum(not _check_event(f"{name}:{line}", text, args, profiles) for name, line, text in _lines(args.files))
+    return 1 if failed else 0
+
+
+def _check_event(where: str, text: str, args: argparse.Namespace, profiles: list[str] | None) -> bool:
+    """Print one event's findings; return whether it has no errors."""
+    try:
+        event: object = json.loads(text)
+    except ValueError as exc:
+        print(f"{where}: error: not JSON: {exc}")
+        return False
+    if not isinstance(event, dict):
+        print(f"{where}: error: not a JSON object")
+        return False
+    _, report = parse(cast(dict[str, Any], event), version=args.version, profiles=profiles, class_key=args.class_key)
+    if args.json:
+        print(json.dumps({"source": where, **report.to_dict()}))  # ASCII: no raw controls
+        return report.ok()
+    for f in report.findings:
+        if f.severity != "info":
+            path = dotted(f.path) or "."
+            print(printable(f"{where}: {f.severity} {f.kind.value} {path}: {f.message}"))
+    return report.ok()
+
+
+def _lines(files: list[str]) -> Iterator[tuple[str, int, str]]:
+    for name in files:
+        stream: contextlib.AbstractContextManager[TextIO] = (
+            contextlib.nullcontext(sys.stdin) if name == "-" else Path(name).open(encoding="utf-8")  # noqa: SIM115 -- entered on the next line
+        )
+        with stream as lines:
+            for n, text in enumerate(lines, 1):
+                if text.strip():
+                    yield name, n, text

ocsf_toolkit/cli/compile.py

@@ -0,0 +1,39 @@
+"""The ``compile`` subcommand: rebuild ``build/compiled/schema-<ver>.json`` for lockfile entries."""
+
+import argparse
+from pathlib import Path
+
+from ocsf_toolkit.cli import _upstream, lock
+
+
+def register(parser: argparse.ArgumentParser) -> None:
+    """Add the ``compile`` arguments to its subparser."""
+    parser.add_argument(
+        "versions",
+        nargs="*",
+        metavar="VERSION",
+        help="OCSF versions to compile (default: every lockfile entry)",
+    )
+    parser.set_defaults(run=run)
+
+
+def run(args: argparse.Namespace) -> int:
+    """Compile each requested lockfile schema and check its hash and version."""
+    lockfile = lock.read(Path(args.lock))
+    versions: list[str] = args.versions or [entry.version for entry in lockfile.schemas]
+    for version in versions:
+        entry = lock.find(lockfile, version)
+        out, digest = _upstream.compile_commit(
+            repo=entry.repo,
+            commit=entry.commit,
+            version=entry.version,
+            sha256=entry.compiled_sha256,
+            compiler=lock.compiler(lockfile, args.compiler_cmd),
+            build_dir=Path(args.build_dir),
+        )
+        print(
+            f"{entry.version}: ok  ref {entry.ref}  commit {entry.commit}\n"
+            f"{entry.version}: compiled version matches the declared one\n"
+            f"{entry.version}: sha256 {digest} == lockfile -> {out}"
+        )
+    return 0

ocsf_toolkit/cli/diff.py

@@ -0,0 +1,65 @@
+"""The ``diff`` subcommand: what changed between two OCSF versions or IR files."""
+
+import argparse
+import json
+import sys
+
+from ocsf_toolkit.cli._common import printable, source_ir
+from ocsf_toolkit.diff import Change, SchemaDiff, diff, plain
+
+
+def register(parser: argparse.ArgumentParser) -> None:
+    """Add the ``diff`` arguments to its subparser."""
+    for name in ("old", "new"):
+        parser.add_argument(name, metavar=name.upper(), help="a shipped OCSF version or an IR file")
+    output = parser.add_mutually_exclusive_group()
+    output.add_argument("--json", action="store_true", help="print the diff as one JSON object")
+    output.add_argument(
+        "--semver",
+        action="store_true",
+        help="print only the release category for re-pinning OLD as NEW: patch, minor or major",
+    )
+    parser.set_defaults(run=run)
+
+
+def run(args: argparse.Namespace) -> int:
+    """Print the diff of two IRs; return 0 when they are equal, 1 when they differ, 2 when one can't be read."""
+    try:
+        old, new = source_ir(args.old), source_ir(args.new)
+    except (ValueError, OSError) as exc:
+        print(f"ocsf-toolkit: error: {exc}", file=sys.stderr)
+        return 2
+    result = diff(old, new)
+    if args.semver:
+        category, counted = semver(result)
+        print(f"{category} ({counted} change(s) outside provenance)")
+    elif args.json:
+        print(json.dumps(result.to_dict()))
+    else:
+        for change in result.changes:
+            print(printable(_line(change)))
+    return 1 if result.changes else 0
+
+
+def semver(result: SchemaDiff) -> tuple[str, int]:
+    """Return the library release a changed IR of an already-shipped OCSF version calls for.
+
+    Repo plan §4.1 gray case: nothing -> patch, only additions -> minor, anything else -> major.
+
+    Provenance is left out: a compiler bump rewrites it without changing what the IR says.
+    """
+    changes = [c for c in result.changes if c.path[:1] != ("provenance",)]
+    if not changes:
+        return "patch", 0
+    if all(c.kind == "added" for c in changes):
+        return "minor", len(changes)
+    return "major", len(changes)
+
+
+def _line(c: Change) -> str:
+    where = ".".join(str(p) for p in c.path)
+    if c.kind == "renamed":
+        return f"renamed {where} -> {c.after}"
+    if c.kind != "changed":
+        return f"{c.kind} {where}"
+    return f"changed {where}: {json.dumps(plain(c.before))} -> {json.dumps(plain(c.after))}"

ocsf_toolkit/cli/generate.py

@@ -0,0 +1,90 @@
+"""The ``generate`` subcommand: a typed model package from an OCSF version or IR (D#5)."""
+
+import argparse
+from pathlib import Path
+
+from ocsf_toolkit.cli import _upstream, lock
+from ocsf_toolkit.cli._common import source_ir
+from ocsf_toolkit.cli.trees import files_sha256, read_tree
+from ocsf_toolkit.codegen import generate
+from ocsf_toolkit.ir import Provenance, SchemaIR
+from ocsf_toolkit.loader import load_compiled
+
+
+def register(parser: argparse.ArgumentParser) -> None:
+    """Add the ``generate`` arguments to its subparser."""
+    parser.add_argument(
+        "source",
+        metavar="SOURCE",
+        help="a shipped OCSF version or an IR file; with --ext, a lockfile version to compile",
+    )
+    parser.add_argument("--package", required=True, help="import name of the generated package")
+    parser.add_argument("--out", required=True, help="directory to write <package>/ into")
+    parser.add_argument(
+        "--profiles",
+        help="comma-separated profiles: emit exact classes for this set (default: superset)",
+    )
+    parser.add_argument(
+        "--extensions",
+        help="comma-separated extensions to include (default: all; empty for none)",
+    )
+    parser.add_argument(
+        "--ext",
+        action="append",
+        default=[],
+        metavar="PATH",
+        help="a directory of custom extensions to compile SOURCE with (repeatable; compiled through uvx "
+        "under the lockfile's python-compile)",
+    )
+    parser.set_defaults(run=run)
+
+
+def run(args: argparse.Namespace) -> int:
+    """Generate the model package and print where it was written."""
+    ir = _extended(args) if args.ext else source_ir(args.source)
+    try:
+        path = generate(
+            ir,
+            package=args.package,
+            out=Path(args.out),
+            profiles=_names(args.profiles),
+            extensions=_names(args.extensions),
+        )
+    except KeyError as exc:  # an unknown profile or extension name
+        raise ValueError(exc.args[0]) from exc
+    modules = sum(1 for _ in path.rglob("*.py"))
+    mode = f"exact for profiles {args.profiles}" if args.profiles is not None else "superset"
+    print(f"{path}: OCSF {ir.version}, {mode}, {modules} modules")
+    return 0
+
+
+def _names(value: str | None) -> list[str] | None:
+    return None if value is None else [n.strip() for n in value.split(",") if n.strip()]
+
+
+def _extended(args: argparse.Namespace) -> SchemaIR:
+    """Compile a lockfile version with custom extension directories into an IR."""
+    lockfile = lock.read(Path(args.lock))
+    entry = lock.find(lockfile, args.source)
+    paths = [Path(p) for p in args.ext]
+    for path in paths:
+        if not path.is_dir():
+            raise ValueError(f"--ext {path}: not a directory")
+    data, digest = _upstream.compiled_output(
+        repo=entry.repo,
+        commit=entry.commit,
+        version=entry.version,
+        sha256=None,  # extensions change the output; the declared version is still checked
+        compiler=lock.compiler(lockfile, args.compiler_cmd),
+        build_dir=Path(args.build_dir),
+        extensions=paths,
+    )
+    provenance = Provenance(
+        schema_repo=entry.repo,
+        ref=entry.ref,
+        commit=entry.commit,
+        compiler_version=lockfile.toolchain.ocsf_schema_compiler,
+        compiler_output_sha256=digest,
+        extensions=tuple((p.resolve().name, "local", files_sha256(read_tree(p))) for p in paths),
+    )
+    return load_compiled(data, provenance)