lairs 0.1.0__py3-none-any.whl

lairs/__init__.py

@@ -0,0 +1,142 @@
+"""lairs: a read/write dataset client for the Layers format.
+
+lairs reads and writes ``pub.layers.*`` records over ATProto, validates them
+against models generated from the Layers lexicons, and exposes them through a
+``datasets``-like API with first-class tooling for audio, video, and neural
+modalities.
+"""
+
+from __future__ import annotations
+
+from importlib.metadata import PackageNotFoundError
+from importlib.metadata import version as _distribution_version
+from typing import TYPE_CHECKING
+
+from lairs.atproto.auth import Session, authed_client, login
+from lairs.data import Corpus, load_corpus
+from lairs.discovery import (
+    DatasetFilter,
+    DatasetSummary,
+    RepoTableOfContents,
+    discover_datasets,
+    list_datasets,
+    table_of_contents,
+)
+from lairs.integrations.registry import (
+    get_codec,
+    get_exporter,
+    get_knowledge_base,
+)
+from lairs.records.blobref import BlobRef
+
+if TYPE_CHECKING:
+    from lairs.integrations.ports import Codec, Exporter, KnowledgeBase
+
+__all__ = [
+    "BlobRef",
+    "Corpus",
+    "DatasetFilter",
+    "DatasetSummary",
+    "RepoTableOfContents",
+    "Session",
+    "__version__",
+    "authed_client",
+    "codec",
+    "discover_datasets",
+    "exporter",
+    "knowledge_base",
+    "list_datasets",
+    "load_corpus",
+    "login",
+    "table_of_contents",
+]
+
+_FALLBACK_VERSION = "0.1.0"
+"""The version reported when no installed distribution metadata is available.
+
+This literal is the single source of truth for source and editable trees where
+``importlib.metadata`` cannot find an installed ``lairs`` distribution. It must
+be kept in step with the ``version`` field in ``pyproject.toml``.
+"""
+
+
+def _resolve_version() -> str:
+    """Return the installed distribution version, falling back to a literal.
+
+    Returns
+    -------
+    str
+        The version string from the installed ``lairs`` distribution metadata,
+        or ``_FALLBACK_VERSION`` when the package is not installed (for example
+        when running from a source checkout).
+    """
+    try:
+        return _distribution_version("lairs")
+    except PackageNotFoundError:
+        return _FALLBACK_VERSION
+
+
+__version__ = _resolve_version()
+
+
+def codec(name: str) -> type[Codec]:
+    """Look up a registered codec adapter class by name.
+
+    Parameters
+    ----------
+    name : str
+        The codec name (for example ``"conllu"`` or ``"brat"``).
+
+    Returns
+    -------
+    type
+        The registered codec class.
+
+    Raises
+    ------
+    lairs.integrations.registry.UnknownAdapterError
+        If no codec is registered under ``name``.
+    """
+    return get_codec(name)
+
+
+def exporter(name: str) -> type[Exporter]:
+    """Look up a registered exporter adapter class by name.
+
+    Parameters
+    ----------
+    name : str
+        The exporter name (for example ``"hf"`` or ``"torch"``).
+
+    Returns
+    -------
+    type
+        The registered exporter class.
+
+    Raises
+    ------
+    lairs.integrations.registry.UnknownAdapterError
+        If no exporter is registered under ``name``.
+    """
+    return get_exporter(name)
+
+
+def knowledge_base(name: str) -> type[KnowledgeBase]:
+    """Look up a registered knowledge-base adapter class by name.
+
+    Parameters
+    ----------
+    name : str
+        The knowledge-base name (for example ``"wikidata"``).
+
+    Returns
+    -------
+    type
+        The registered knowledge-base class.
+
+    Raises
+    ------
+    lairs.integrations.registry.UnknownAdapterError
+        If no knowledge base is registered under ``name``.
+    """
+    return get_knowledge_base(name)

lairs/_aturi.py

@@ -0,0 +1,59 @@
+"""AT-URI parsing helpers shared across lairs.
+
+Small, dependency-free helpers for pulling the authority and collection segments
+out of an ``at://`` URI. Centralised here so the discovery, CLI, and data layers
+parse AT-URIs the same way.
+
+These helpers are positional string splitters, not validators. They assume a
+well-formed ``at://authority/collection/rkey`` URI and return an empty string
+for a missing segment; they do not check the ``at://`` scheme or the authority
+shape, so malformed input yields a best-effort segment rather than an error.
+Callers that need validation must do it before calling.
+"""
+
+from __future__ import annotations
+
+__all__ = ["authority_of", "nsid_of"]
+
+_AT_URI_PREFIX = "at://"
+"""The scheme prefix every AT-URI carries."""
+
+_MIN_PARTS_WITH_COLLECTION = 2
+"""The number of path segments an AT-URI needs to carry a collection NSID."""
+
+
+def authority_of(uri: str) -> str:
+    """Return the authority (DID or handle) segment of an AT-URI.
+
+    Parameters
+    ----------
+    uri : str
+        The AT-URI to parse.
+
+    Returns
+    -------
+    str
+        The authority segment, or an empty string when ``uri`` is empty.
+    """
+    body = uri.removeprefix(_AT_URI_PREFIX)
+    return body.split("/", 1)[0] if body else ""
+
+
+def nsid_of(uri: str) -> str:
+    """Return the collection NSID segment of an AT-URI.
+
+    Parameters
+    ----------
+    uri : str
+        The AT-URI to parse.
+
+    Returns
+    -------
+    str
+        The collection NSID, or an empty string when the URI has no collection.
+    """
+    body = uri.removeprefix(_AT_URI_PREFIX)
+    parts = body.split("/")
+    if len(parts) >= _MIN_PARTS_WITH_COLLECTION:
+        return parts[1]
+    return ""

lairs/_codegen/__init__.py

@@ -0,0 +1,30 @@
+"""Codegen pipeline that turns vendored lexicons into generated models.
+
+The pipeline parses each lexicon into a panproto ``Schema``, walks it into
+didactic spec dicts, builds models, and emits Python module text.
+"""
+
+from __future__ import annotations
+
+from lairs._codegen.emit import emit_module
+from lairs._codegen.manifest import Manifest, load_manifest
+from lairs._codegen.pipeline import check, generate, namespace_specs
+from lairs._codegen.schema_to_spec import (
+    FieldSpec,
+    ModelSpec,
+    VariantSpec,
+    schema_to_specs,
+)
+
+__all__ = [
+    "FieldSpec",
+    "Manifest",
+    "ModelSpec",
+    "VariantSpec",
+    "check",
+    "emit_module",
+    "generate",
+    "load_manifest",
+    "namespace_specs",
+    "schema_to_specs",
+]

lairs/_codegen/emit.py

@@ -0,0 +1,450 @@
+"""Emit Python module text for generated models.
+
+Renders :class:`~lairs._codegen.schema_to_spec.ModelSpec` value models into
+committed module source text with a generated-by header and the source manifest
+hash. Emission is deterministic (stable class ordering and stable field
+ordering) so the ``lairs gen --check`` drift gate is meaningful. The emitted
+modules are the import surface of :mod:`lairs.records`; they are rich didactic
+models carrying descriptions, optionality, refined value types, integer ranges,
+knownValues, and union discriminators, which the lossy spec-synthesis path could
+not reconstruct.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from collections.abc import Sequence
+
+    from lairs._codegen.schema_to_spec import FieldSpec, ModelSpec, VariantSpec
+
+__all__ = ["emit_module"]
+
+_HEADER_LINE = "# generated by lairs gen; do not edit"
+
+
+def emit_module(
+    specs: Sequence[ModelSpec],
+    *,
+    manifest_hash: str,
+) -> str:
+    """Render record and union specs to Python module source text.
+
+    Parameters
+    ----------
+    specs : collections.abc.Sequence of lairs._codegen.schema_to_spec.ModelSpec
+        The specs for one namespace, already ordered so embed targets precede
+        the models that embed them.
+    manifest_hash : str
+        The content hash of the source lexicon tree, recorded in the header so
+        the committed module records the lexicon revision it was generated from.
+
+    Returns
+    -------
+    str
+        The module source text, with a generated-by header, the manifest hash,
+        a module docstring, imports, the emitted classes, and an ``__all__``.
+    """
+    ordered = _order_specs(specs)
+    uses_datetime = any(_spec_uses_datetime(spec) for spec in ordered)
+    uses_literal = any(spec.is_union for spec in ordered)
+    uses_blobref = any(_spec_uses_blobref(spec) for spec in ordered)
+    uses_mixed_case = any(_spec_uses_mixed_case(spec) for spec in ordered)
+
+    # codes inherent to atproto-faithful generated didactic models. N815: the
+    # python attribute keeps the camelCase wire key. TC001/TC003: didactic
+    # resolves annotations eagerly, so the annotated imports must stay at
+    # runtime rather than move under TYPE_CHECKING.
+    codes: set[str] = set()
+    if uses_mixed_case:
+        codes.add("N815")
+    if uses_datetime:
+        codes.add("TC003")
+    if uses_blobref:
+        codes.add("TC001")
+
+    blocks: list[str] = []
+    blocks.append(_header(manifest_hash, codes))
+    blocks.append(_module_docstring(ordered))
+    blocks.append(
+        _imports(
+            uses_datetime=uses_datetime,
+            uses_literal=uses_literal,
+            uses_blobref=uses_blobref,
+        )
+    )
+    blocks.append(_dunder_all(ordered))
+    blocks.extend(_class_text(spec) for spec in ordered)
+    body = "\n\n".join(blocks)
+    return _suppress_long_lines(body.rstrip()) + "\n"
+
+
+_LINE_LIMIT = 88
+
+
+def _suppress_long_lines(text: str) -> str:
+    """Append an ``E501`` suppression to any over-length line.
+
+    Lexicon descriptions are copied verbatim and can exceed the line limit;
+    ``ruff format`` does not wrap string-literal arguments, so an over-length
+    description keeps a targeted suppression rather than being truncated.
+    """
+    return "\n".join(_suppress_line(line) for line in text.split("\n"))
+
+
+def _suppress_line(line: str) -> str:
+    """Return a line with an ``E501`` suppression when it is over-length."""
+    if len(line) > _LINE_LIMIT and "# noqa" not in line:
+        return f"{line}  # noqa: E501"
+    return line
+
+
+def _order_specs(specs: Sequence[ModelSpec]) -> list[ModelSpec]:
+    """Return specs in a stable, dependency-respecting order.
+
+    Specs are sorted by class name for determinism, then a stable topological
+    pass moves each embed or union-variant target ahead of its referrer so the
+    emitted forward references resolve without quoting. Targets defined in other
+    namespaces are imported, not reordered.
+    """
+    by_name = {spec.name: spec for spec in specs}
+    alphabetical = sorted(specs, key=lambda spec: spec.name)
+    ordered: list[ModelSpec] = []
+    placed: set[str] = set()
+
+    def visit(spec: ModelSpec, stack: frozenset[str]) -> None:
+        if spec.name in placed or spec.name in stack:
+            return
+        for dep in _local_dependencies(spec, by_name):
+            visit(by_name[dep], stack | {spec.name})
+        placed.add(spec.name)
+        ordered.append(spec)
+
+    for spec in alphabetical:
+        visit(spec, frozenset())
+    return ordered
+
+
+def _local_dependencies(
+    spec: ModelSpec,
+    by_name: dict[str, ModelSpec],
+) -> list[str]:
+    """Return the names of same-namespace specs ``spec`` depends on."""
+    deps: set[str] = {
+        variant.target for variant in spec.variants if variant.target in by_name
+    }
+    field_targets = (_field_local_target(field) for field in spec.fields)
+    deps.update(
+        target for target in field_targets if target is not None and target in by_name
+    )
+    return sorted(deps)
+
+
+def _field_local_target(field: FieldSpec) -> str | None:
+    """Return the same-namespace model a field embeds or unions, if any."""
+    if field.type_kind in {"embed", "union"}:
+        return field.target
+    if field.type_kind == "array" and field.item is not None:
+        return _field_local_target(field.item)
+    return None
+
+
+def _header(manifest_hash: str, noqa_codes: set[str]) -> str:
+    """Return the generated-by header recording the lexicon tree hash.
+
+    The hash is written without a trailing colon after a bare token so it does
+    not trip ruff's commented-out-code heuristic. A file-level ``ruff: noqa``
+    directive carries exactly the codes the module triggers so the suppression
+    itself is never flagged unused.
+    """
+    lines = [_HEADER_LINE, f"# lexicon tree hash {manifest_hash}"]
+    directive = _noqa_directive(noqa_codes)
+    if directive is not None:
+        lines.append(directive)
+    return "\n".join(lines)
+
+
+def _noqa_directive(noqa_codes: set[str]) -> str | None:
+    """Return a file-level ruff suppression line, or ``None`` when empty."""
+    if not noqa_codes:
+        return None
+    listed = ", ".join(sorted(noqa_codes))
+    return f"# ruff: noqa: {listed}"
+
+
+def _module_docstring(specs: Sequence[ModelSpec]) -> str:
+    """Return the module docstring naming the source namespace."""
+    namespace = _shared_namespace(specs)
+    return (
+        f'"""Generated models for the {namespace} lexicon namespace.\n\n'
+        "This module is emitted by ``lairs gen`` from the vendored lexicons and\n"
+        "must not be edited by hand. Each class mirrors a lexicon record, object,\n"
+        'or union definition.\n"""'
+    )
+
+
+def _shared_namespace(specs: Sequence[ModelSpec]) -> str:
+    """Return the namespace shared by a sequence of specs.
+
+    Records and objects in one emitted module all share their first three
+    dotted nsid components (for example ``pub.layers.annotation``); the module
+    docstring names that common prefix rather than each file's nsid.
+    """
+    nsids = sorted({spec.nsid for spec in specs})
+    if not nsids:
+        return "pub.layers"
+    prefix_parts = nsids[0].split(".")[:3]
+    return ".".join(prefix_parts)
+
+
+def _imports(*, uses_datetime: bool, uses_literal: bool, uses_blobref: bool) -> str:
+    """Return the import block for the emitted module.
+
+    The generated modules deliberately omit ``from __future__ import
+    annotations``: didactic resolves field annotations eagerly at class
+    creation, so the annotated imports must be live runtime names, and keeping
+    them eager also lets ruff see them as used rather than typing-only.
+    """
+    stdlib: list[str] = []
+    if uses_datetime:
+        stdlib.append("from datetime import datetime")
+    if uses_literal:
+        stdlib.append("from typing import Literal")
+    lines: list[str] = []
+    if stdlib:
+        lines.extend(stdlib)
+        lines.append("")
+    lines.append("import didactic.api as dx")
+    if uses_blobref:
+        lines.append("")
+        lines.append("from lairs.records.blobref import BlobRef")
+    return "\n".join(lines)
+
+
+def _dunder_all(specs: Sequence[ModelSpec]) -> str:
+    """Return the module ``__all__`` listing every emitted class."""
+    names = sorted(_emitted_names(specs))
+    quoted = ",\n    ".join(f'"{name}"' for name in names)
+    if not names:
+        return "__all__: list[str] = []"
+    return f"__all__ = [\n    {quoted},\n]"
+
+
+def _emitted_names(specs: Sequence[ModelSpec]) -> list[str]:
+    """Return every class name a spec sequence emits, variants included."""
+    names: list[str] = []
+    for spec in specs:
+        names.append(spec.name)
+        names.extend(variant.class_name for variant in spec.variants)
+    return names
+
+
+def _class_text(spec: ModelSpec) -> str:
+    """Render a single spec to a class definition (or union family)."""
+    if spec.is_union:
+        return _union_text(spec)
+    return _model_text(spec)
+
+
+def _model_text(spec: ModelSpec) -> str:
+    """Render a record or object spec to a ``dx.Model`` subclass."""
+    docstring = _class_docstring(spec.description or f"The {spec.def_name} definition.")
+    lines = [f"class {spec.name}(dx.Model):", docstring, ""]
+    for field in spec.fields:
+        lines.extend(_field_lines(field))
+    return "\n".join(lines)
+
+
+def _union_text(spec: ModelSpec) -> str:
+    """Render a formal union spec to a ``dx.TaggedUnion`` family."""
+    discriminator = spec.discriminator or "kind"
+    root = "\n".join(
+        [
+            f'class {spec.name}(dx.TaggedUnion, discriminator="{discriminator}"):',
+            _class_docstring(spec.description or f"The {spec.def_name} union."),
+            "",
+        ]
+    )
+    blocks = [root]
+    blocks.extend(
+        _variant_text(spec, variant, discriminator) for variant in spec.variants
+    )
+    return "\n\n\n".join(blocks)
+
+
+def _variant_text(
+    spec: ModelSpec,
+    variant: VariantSpec,
+    discriminator: str,
+) -> str:
+    """Render a single union variant subclass."""
+    lines = [f"class {variant.class_name}({spec.name}):"]
+    lines.append(
+        _class_docstring(f"The {variant.discriminator_value!r} member of {spec.name}.")
+    )
+    lines.append("")
+    lines.append(
+        f"    {discriminator}: Literal[{variant.discriminator_value!r}] = dx.field("
+    )
+    lines.append(f"        default={variant.discriminator_value!r},")
+    lines.append(
+        f'        description="discriminator pinning this member to '
+        f'{variant.discriminator_value}",'
+    )
+    lines.append("    )")
+    lines.append(f"    value: {variant.target} | None = dx.field(")
+    lines.append("        default=None,")
+    lines.append('        description="the wrapped member model",')
+    lines.append("    )")
+    return "\n".join(lines)
+
+
+def _field_lines(field: FieldSpec) -> list[str]:
+    """Render a single field declaration to source lines.
+
+    Lexicon property names are camelCase wire keys that must round-trip
+    verbatim through ATProto JSON, so the python attribute keeps the camelCase
+    name; the file-level ``N815`` suppression in the header covers them.
+    """
+    annotation = _field_annotation(field)
+    args = _field_args(field)
+    lines = [f"    {field.name}: {annotation} = dx.field("]
+    lines.extend(f"        {arg}" for arg in args)
+    lines.append("    )")
+    return lines
+
+
+def _is_mixed_case(name: str) -> bool:
+    """Return whether a field name is mixedCase (carries an inner capital).
+
+    A purely lowercase or snake_case name does not trip ``N815``; only names
+    with an interior upper-case letter do.
+    """
+    return name != name.lower() and "_" not in name
+
+
+def _field_annotation(field: FieldSpec) -> str:
+    """Return the python annotation source for a field."""
+    base = _base_annotation(field)
+    if field.required:
+        return base
+    return f"{base} | None"
+
+
+# scalar field kinds whose python annotation is a fixed builtin or alias.
+_SCALAR_ANNOTATIONS: dict[str, str] = {
+    "str": "str",
+    "int": "int",
+    "bool": "bool",
+    "datetime": "datetime",
+    "bytes": "bytes",
+    "blob": "BlobRef",
+}
+
+
+def _base_annotation(field: FieldSpec) -> str:
+    """Return the unwrapped python annotation source for a field."""
+    kind = field.type_kind
+    if kind == "embed":
+        return f"dx.Embed[{field.target}]"
+    if kind == "union":
+        return f"{field.target}"
+    if kind == "array":
+        item = field.item
+        element = _base_annotation(item) if item is not None else "str"
+        return f"tuple[{element}, ...]"
+    return _SCALAR_ANNOTATIONS.get(kind, "str")
+
+
+def _field_args(field: FieldSpec) -> list[str]:
+    """Return the ``dx.field`` keyword argument source lines for a field."""
+    args: list[str] = []
+    if not field.required:
+        if field.type_kind == "array":
+            args.append("default_factory=tuple,")
+        else:
+            args.append("default=None,")
+    if field.description is not None:
+        args.append(f"description={_py_str(field.description)},")
+    extras = _field_extras(field)
+    if extras:
+        args.append(f"extras={{{extras}}},")
+    if field.type_kind == "bytes":
+        args.append("opaque=True,")
+    if not args:
+        # an unconditional placeholder keeps the call well-formed; required
+        # scalar fields with no description fall here
+        args.append('description="generated field",')
+    return args
+
+
+def _field_extras(field: FieldSpec) -> str:
+    """Return the ``extras`` mapping literal contents for a field, if any."""
+    entries: list[str] = []
+    if field.string_format is not None:
+        entries.append(f'"format": {_py_str(field.string_format)}')
+    if field.known_values:
+        values = ", ".join(_py_str(value) for value in field.known_values)
+        entries.append(f'"knownValues": ({values},)')
+    if field.minimum is not None:
+        entries.append(f'"minimum": {field.minimum}')
+    if field.maximum is not None:
+        entries.append(f'"maximum": {field.maximum}')
+    if field.min_length is not None:
+        entries.append(f'"minLength": {field.min_length}')
+    if field.max_length is not None:
+        entries.append(f'"maxLength": {field.max_length}')
+    return ", ".join(entries)
+
+
+def _class_docstring(summary: str) -> str:
+    """Return an indented one-line numpy-style class docstring."""
+    text = " ".join(summary.split())
+    if not text.endswith("."):
+        text = f"{text}."
+    return f'    """{_escape_docstring(text)}"""'
+
+
+def _escape_docstring(text: str) -> str:
+    """Escape a docstring body so it is a valid triple-quoted literal."""
+    return text.replace("\\", "\\\\").replace('"""', '\\"\\"\\"')
+
+
+def _py_str(value: str) -> str:
+    """Return a python source string literal for ``value``."""
+    escaped = value.replace("\\", "\\\\").replace('"', '\\"')
+    return f'"{escaped}"'
+
+
+def _spec_uses_datetime(spec: ModelSpec) -> bool:
+    """Return whether any field of a spec is datetime-typed."""
+    return any(_field_uses_datetime(field) for field in spec.fields)
+
+
+def _field_uses_datetime(field: FieldSpec) -> bool:
+    """Return whether a field (or its array element) is datetime-typed."""
+    if field.type_kind == "datetime":
+        return True
+    if field.type_kind == "array" and field.item is not None:
+        return _field_uses_datetime(field.item)
+    return False
+
+
+def _spec_uses_mixed_case(spec: ModelSpec) -> bool:
+    """Return whether any field of a spec carries a camelCase wire name."""
+    return any(_is_mixed_case(field.name) for field in spec.fields)
+
+
+def _spec_uses_blobref(spec: ModelSpec) -> bool:
+    """Return whether any field of a spec is a blob reference."""
+    return any(_field_uses_blobref(field) for field in spec.fields)
+
+
+def _field_uses_blobref(field: FieldSpec) -> bool:
+    """Return whether a field (or its array element) is a blob reference."""
+    if field.type_kind == "blob":
+        return True
+    if field.type_kind == "array" and field.item is not None:
+        return _field_uses_blobref(field.item)
+    return False