lairs 0.1.0__tar.gz

lairs-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,109 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+  workflow_dispatch:
+
+# cancel superseded runs on the same ref (a new push to a PR supersedes the old).
+concurrency:
+  group: ci-${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+env:
+  # pin the Python the whole workflow resolves against.
+  UV_PYTHON: "3.14"
+
+jobs:
+  # static gates: formatting, lint, types, the Any/object ban, codegen drift,
+  # and a strict docs build. no docker, so this finishes fast and fails early.
+  quality:
+    name: Quality gates
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          enable-cache: true
+
+      - name: Install Python ${{ env.UV_PYTHON }}
+        run: uv python install ${{ env.UV_PYTHON }}
+
+      - name: Sync dependencies (dev + docs)
+        run: uv sync --frozen --group docs
+
+      - name: Ruff format
+        run: uv run ruff format --check lairs tests
+
+      - name: Ruff lint
+        run: uv run ruff check lairs tests
+
+      - name: Install TensorFlow nightly
+        # TensorFlow ships no stable cp314 wheel yet (the tf extra markers it out
+        # of the locked env), so the nightly provides one. Installing it lets ty
+        # resolve `import tensorflow` in the tfdata exporter. decord and
+        # label-studio-sdk have no cp314 wheel at all and remain unexercised.
+        run: uv pip install tf-nightly
+
+      - name: Type check (ty)
+        run: uv run ty check --error-on-warning
+
+      - name: Forbid Any in annotations
+        # `Any` is the unsound escape hatch; ruff ANN401 (part of the ALL
+        # ruleset above) flags it in annotation positions precisely, with no
+        # docstring false positives. sound `object` is allowed: it must be
+        # narrowed before use, and the ALL ruleset itself requires annotating
+        # ignored *args/**kwargs, for which `object` is the only sound choice.
+        run: uv run ruff check lairs tests --select ANN401
+
+      - name: Generated-model drift gate
+        run: uv run lairs gen --check
+
+      - name: Build documentation (strict)
+        run: uv run --group docs mkdocs build --strict
+
+  # the full suite, including every integration test. the pds_server fixture
+  # starts a real bluesky PDS through docker compose (docker and compose v2 are
+  # preinstalled on the runner), so --run-integration exercises the genuine
+  # read/write path rather than skipping it.
+  test:
+    name: Test suite (all tests, with Docker PDS)
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          enable-cache: true
+
+      - name: Install Python ${{ env.UV_PYTHON }}
+        run: uv python install ${{ env.UV_PYTHON }}
+
+      - name: Sync dependencies (dev group with every installable extra)
+        run: uv sync --frozen
+
+      - name: Install TensorFlow nightly
+        # no stable cp314 wheel yet; the nightly runs the tfdata exporter tests
+        # rather than skipping them. decord and label-studio-sdk have no cp314
+        # wheel at all, so a small number of their tests skip cleanly.
+        run: uv pip install tf-nightly
+
+      - name: Confirm Docker is available
+        run: |
+          docker version
+          docker compose version
+
+      - name: Pre-pull the PDS image
+        # warm the integration container so the health check does not race a
+        # cold image pull. the fixture manages compose lifecycle per session.
+        run: docker compose -f tests/pds/docker-compose.yml pull
+
+      - name: Run the full test suite
+        run: uv run pytest --run-integration -v

lairs-0.1.0/.github/workflows/docs.yml

@@ -0,0 +1,70 @@
+name: Docs
+
+# build the documentation and publish it to GitHub Pages. the site is served at
+# https://layers.pub/lairs/: the layers-pub organization's Pages custom apex
+# domain (layers.pub) serves each project repository's Pages at /<repo>/, so the
+# lairs project site lands at /lairs/. no per-repo CNAME is needed; the project
+# inherits the organization's domain.
+on:
+  push:
+    branches: [main]
+    paths:
+      - "docs/**"
+      - "mkdocs.yml"
+      - "lairs/**" # docstrings feed the API reference via mkdocstrings
+      - "pyproject.toml"
+      - ".github/workflows/docs.yml"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+# one Pages deployment at a time; let an in-flight deploy finish rather than
+# cancelling it.
+concurrency:
+  group: pages
+  cancel-in-progress: false
+
+env:
+  UV_PYTHON: "3.14"
+
+jobs:
+  build:
+    name: Build site
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          enable-cache: true
+
+      - name: Install Python ${{ env.UV_PYTHON }}
+        run: uv python install ${{ env.UV_PYTHON }}
+
+      - name: Sync docs dependencies
+        run: uv sync --frozen --group docs
+
+      - name: Build the site (strict)
+        run: uv run --group docs mkdocs build --strict
+
+      - name: Upload the Pages artifact
+        uses: actions/upload-pages-artifact@v3
+        with:
+          path: site
+
+  deploy:
+    name: Deploy to GitHub Pages
+    needs: build
+    runs-on: ubuntu-latest
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    permissions:
+      pages: write
+      id-token: write
+    steps:
+      - name: Deploy
+        id: deployment
+        uses: actions/deploy-pages@v4

lairs-0.1.0/.github/workflows/release.yml

@@ -0,0 +1,169 @@
+name: Release
+
+# tags shaped like v0.1.0 cut a release: verify, build, publish to PyPI via
+# Trusted Publishing, and create the GitHub release.
+on:
+  push:
+    tags: ["v*"]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+env:
+  UV_PYTHON: "3.14"
+
+jobs:
+  # gate the release on the same checks CI runs, so a tag can never publish a
+  # red build. the full suite (including the Docker PDS integration tests) runs
+  # here too.
+  verify:
+    name: Verify before release
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          enable-cache: true
+
+      - name: Install Python ${{ env.UV_PYTHON }}
+        run: uv python install ${{ env.UV_PYTHON }}
+
+      - name: Sync dependencies (dev + docs)
+        run: uv sync --frozen --group docs
+
+      - name: Ruff format
+        run: uv run ruff format --check lairs tests
+
+      - name: Ruff lint
+        run: uv run ruff check lairs tests
+
+      - name: Install TensorFlow nightly
+        # no stable cp314 wheel yet; the nightly lets ty resolve tensorflow and
+        # runs the tfdata exporter tests rather than skipping them.
+        run: uv pip install tf-nightly
+
+      - name: Type check (ty)
+        run: uv run ty check --error-on-warning
+
+      - name: Generated-model drift gate
+        run: uv run lairs gen --check
+
+      - name: Build documentation (strict)
+        run: uv run --group docs mkdocs build --strict
+
+      - name: Pre-pull the PDS image
+        run: docker compose -f tests/pds/docker-compose.yml pull
+
+      - name: Run the full test suite
+        run: uv run pytest --run-integration
+
+  # build the sdist and wheel once and hand them to every downstream job.
+  build:
+    name: Build distributions
+    needs: verify
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          enable-cache: true
+
+      - name: Install Python ${{ env.UV_PYTHON }}
+        run: uv python install ${{ env.UV_PYTHON }}
+
+      - name: Build sdist and wheel
+        run: uv build
+
+      - name: Check distribution metadata
+        run: uvx twine check dist/*
+
+      - name: Confirm the built version matches the tag
+        if: startsWith(github.ref, 'refs/tags/v')
+        run: |
+          set -euo pipefail
+          tag_version="${GITHUB_REF_NAME#v}"
+          built=$(ls dist/lairs-*.tar.gz | sed -E 's#.*/lairs-(.*)\.tar\.gz#\1#')
+          if [ "$tag_version" != "$built" ]; then
+            echo "::error::tag $GITHUB_REF_NAME does not match built version $built"
+            exit 1
+          fi
+          echo "tag and built version agree: $built"
+
+      - name: Upload distributions
+        uses: actions/upload-artifact@v4
+        with:
+          name: dist
+          path: dist/
+          if-no-files-found: error
+
+  # publish to PyPI with Trusted Publishing (OIDC): no token is stored. the
+  # `pypi` environment scopes the trusted publisher and gates the step on any
+  # protection rules you add to it.
+  pypi:
+    name: Publish to PyPI
+    needs: build
+    if: startsWith(github.ref, 'refs/tags/v')
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/project/lairs/
+    permissions:
+      id-token: write
+    steps:
+      - name: Download distributions
+        uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+      - name: Publish
+        uses: pypa/gh-action-pypi-publish@release/v1
+
+  # create the GitHub release from the tag, with the changelog section as notes
+  # and the built artifacts attached.
+  github-release:
+    name: GitHub release
+    needs: [build, pypi]
+    if: startsWith(github.ref, 'refs/tags/v')
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Download distributions
+        uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+      - name: Extract the changelog section for this tag
+        run: |
+          set -euo pipefail
+          version="${GITHUB_REF_NAME#v}"
+          awk -v v="$version" '
+            $0 ~ "^## \\[" v "\\]" { flag = 1; next }
+            flag && /^## \[/ { exit }
+            flag { print }
+          ' CHANGELOG.md > release-notes.md
+          if [ ! -s release-notes.md ]; then
+            echo "See [CHANGELOG.md](CHANGELOG.md)." > release-notes.md
+          fi
+
+      - name: Create the release
+        uses: softprops/action-gh-release@v2
+        with:
+          body_path: release-notes.md
+          files: dist/*
+          fail_on_unmatched_files: true
+
+  # documentation is built and deployed to GitHub Pages by the dedicated Docs
+  # workflow (.github/workflows/docs.yml) on every push to main. the release
+  # commit (version bump and changelog) lands on main before the tag, so the
+  # published site at https://layers.pub/lairs/ is already current by release
+  # time. nothing to deploy here.

lairs-0.1.0/.gitignore

@@ -0,0 +1,39 @@
+# virtual environments
+.venv/
+venv/
+
+# python bytecode and caches
+__pycache__/
+*.pyc
+*.pyo
+
+# tool caches
+.ruff_cache/
+.ty_cache/
+.pytest_cache/
+.mypy_cache/
+.coverage
+.coverage.*
+htmlcov/
+
+# build artifacts
+build/
+dist/
+*.egg-info/
+
+# data and cache directories
+/data/
+.lairs/
+*.parquet
+blobs/
+
+# dev notes and local-only design doc
+notes/
+PLAN.md
+CLAUDE.md
+
+# OS cruft
+.DS_Store
+
+# mkdocs build output
+/site/

lairs-0.1.0/CHANGELOG.md

@@ -0,0 +1,65 @@
+# Changelog
+
+All notable changes to `lairs` are documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+## [0.1.0] - 2026-06-25
+
+The first public release. `lairs` is a read/write dataset client for the
+[Layers](https://github.com/layers-pub) format on the AT Protocol: the mental
+model is `datasets` and `git` for decentralised linguistic annotation. It is
+built on [didactic](https://github.com/panproto/didactic) and
+[panproto](https://github.com/panproto/phrom); every structured value is a
+didactic model.
+
+### Added
+
+- **Generated record models.** Typed models for the 26 `pub.layers.*` record
+  types, generated from the vendored Layers lexicons, with a content-addressed
+  `BlobRef` value type and a drift gate (`lairs gen --check`).
+- **ATProto access layer.** Read records and whole repositories from a Personal
+  Data Server over XRPC, decode CAR/DAG-CBOR commits, resolve handles and DIDs,
+  and follow the firehose with cursor-based reconnect.
+- **Schema-aware local store.** A panproto-backed, git-like repository where a
+  corpus snapshot is a commit and a dataset version is a tag, with collision-free
+  record files, a deletion/tombstone path, and revision-to-revision diffs. An
+  Arrow/Parquet materialiser flattens records and polymorphic anchors into typed
+  columns.
+- **Dataset and corpus API.** A HuggingFace-`datasets`-like surface with lazy and
+  streaming `Dataset`, feature derivation, and a `Corpus` scoped to its
+  membership records with train/dev/test split accessors.
+- **Authoring and publishing.** Validated-by-construction builders, blob upload,
+  and dependency-ordered bulk publishing to the authenticated user's own
+  repository, with an idempotent re-publish that is a no-op for unchanged
+  records (including blob-bearing media, expression, and persona records).
+- **Media layer.** On-demand resolution of audio, video, and time-series signals
+  behind injected fetcher and content-addressed cache ports, with anchor
+  resolution over the full anchor union (text, token, temporal, spatio-temporal,
+  page, and external targets).
+- **Dataset discovery.** Crawl the Layers network for corpora, maintain a local
+  searchable index with a DuckDB query accelerator, tail the firehose to keep the
+  index fresh (including deletions), and diff index revisions.
+- **Format codecs.** brat stand-off and CoNLL-U import/export, discoverable
+  through entry points.
+- **Framework exporters.** HuggingFace `datasets` and Hub push/pull, PyTorch
+  (map-style and worker-sharded iterable), `tf.data`, and WebDataset exporters,
+  each behind an optional extra.
+- **Knowledge-base connectors.** Wikidata, OpenRefine reconciliation, and Glazing
+  connectors behind a common port, with experiment-tracking provenance for
+  MLflow and Weights & Biases.
+- **Terminal explorer.** `lairs tui`, a colourful three-tab TUI to Explore the
+  discovery index, Browse every record type in a repository with model-driven,
+  view-switched visualisations (CoNLL-U grids, dependency trees, span overlays,
+  judgment distributions, alignments, and more), and Query materialised data with
+  SQL, a KWIC concordance, and a CQL token-pattern language.
+- **Command-line interface.** The `lairs` command for vendoring lexicons,
+  regenerating models, pulling and materialising corpora, publishing, inspecting
+  repositories, discovering datasets, building and searching the index, managing
+  sessions, and launching the explorer.
+
+[Unreleased]: https://github.com/layers-pub/lairs/compare/v0.1.0...HEAD
+[0.1.0]: https://github.com/layers-pub/lairs/releases/tag/v0.1.0

lairs-0.1.0/CODE_OF_CONDUCT.md

@@ -0,0 +1,133 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+We as members, contributors, and leaders pledge to make participation in our
+community a harassment-free experience for everyone, regardless of age, body
+size, visible or invisible disability, ethnicity, sex characteristics, gender
+identity and expression, level of experience, education, socio-economic status,
+nationality, personal appearance, race, caste, color, religion, or sexual
+identity and orientation.
+
+We pledge to act and interact in ways that contribute to an open, welcoming,
+diverse, inclusive, and healthy community.
+
+## Our Standards
+
+Examples of behavior that contributes to a positive environment for our
+community include:
+
+- Demonstrating empathy and kindness toward other people
+- Being respectful of differing opinions, viewpoints, and experiences
+- Giving and gracefully accepting constructive feedback
+- Accepting responsibility and apologizing to those affected by our mistakes,
+  and learning from the experience
+- Focusing on what is best not just for us as individuals, but for the overall
+  community
+
+Examples of unacceptable behavior include:
+
+- The use of sexualized language or imagery, and sexual attention or advances of
+  any kind
+- Trolling, insulting or derogatory comments, and personal or political attacks
+- Public or private harassment
+- Publishing others' private information, such as a physical or email address,
+  without their explicit permission
+- Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Enforcement Responsibilities
+
+Community leaders are responsible for clarifying and enforcing our standards of
+acceptable behavior and will take appropriate and fair corrective action in
+response to any behavior that they deem inappropriate, threatening, offensive,
+or harmful.
+
+Community leaders have the right and responsibility to remove, edit, or reject
+comments, commits, code, wiki edits, issues, and other contributions that are
+not aligned to this Code of Conduct, and will communicate reasons for moderation
+decisions when appropriate.
+
+## Scope
+
+This Code of Conduct applies within all community spaces, and also applies when
+an individual is officially representing the community in public spaces.
+Examples of representing our community include using an official email address,
+posting via an official social media account, or acting as an appointed
+representative at an online or offline event.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported to the community leaders responsible for enforcement at
+aaronstevenwhite@gmail.com. All complaints will be reviewed and investigated
+promptly and fairly.
+
+All community leaders are obligated to respect the privacy and security of the
+reporter of any incident.
+
+## Enforcement Guidelines
+
+Community leaders will follow these Community Impact Guidelines in determining
+the consequences for any action they deem in violation of this Code of Conduct:
+
+### 1. Correction
+
+**Community Impact**: Use of inappropriate language or other behavior deemed
+unprofessional or unwelcome in the community.
+
+**Consequence**: A private, written warning from community leaders, providing
+clarity around the nature of the violation and an explanation of why the
+behavior was inappropriate. A public apology may be requested.
+
+### 2. Warning
+
+**Community Impact**: A violation through a single incident or series of
+actions.
+
+**Consequence**: A warning with consequences for continued behavior. No
+interaction with the people involved, including unsolicited interaction with
+those enforcing the Code of Conduct, for a specified period of time. This
+includes avoiding interactions in community spaces as well as external channels
+like social media. Violating these terms may lead to a temporary or permanent
+ban.
+
+### 3. Temporary Ban
+
+**Community Impact**: A serious violation of community standards, including
+sustained inappropriate behavior.
+
+**Consequence**: A temporary ban from any sort of interaction or public
+communication with the community for a specified period of time. No public or
+private interaction with the people involved, including unsolicited interaction
+with those enforcing the Code of Conduct, is allowed during this period.
+Violating these terms may lead to a permanent ban.
+
+### 4. Permanent Ban
+
+**Community Impact**: Demonstrating a pattern of violation of community
+standards, including sustained inappropriate behavior, harassment of an
+individual, or aggression toward or disparagement of classes of individuals.
+
+**Consequence**: A permanent ban from any sort of public interaction within the
+community.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage],
+version 2.1, available at
+[https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1].
+
+Community Impact Guidelines were inspired by
+[Mozilla's code of conduct enforcement ladder][mozilla].
+
+[homepage]: https://www.contributor-covenant.org
+[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html
+[mozilla]: https://github.com/mozilla/diversity
+
+For answers to common questions about this code of conduct, see the FAQ at
+[https://www.contributor-covenant.org/faq][faq]. Translations are available at
+[https://www.contributor-covenant.org/translations][translations].
+
+[faq]: https://www.contributor-covenant.org/faq
+[translations]: https://www.contributor-covenant.org/translations