matview-cache 0.1.0__tar.gz

matview_cache-0.1.0/.github/dependabot.yml

@@ -0,0 +1,12 @@
+# Action versions only. The Python dependencies are deliberately declared as ranges with
+# floors the codec-floor job actually checks, so a bot raising the floors would be
+# undoing that on purpose rather than by accident.
+version: 2
+updates:
+  - package-ecosystem: github-actions
+    directory: /
+    schedule:
+      interval: monthly
+    groups:
+      actions:
+        patterns: ["*"]

matview_cache-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,47 @@
+name: CI
+
+on:
+  push:
+    branches: ["**"]
+  pull_request:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  quality:
+    name: Lint, format & type check
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+      - uses: astral-sh/setup-uv@v7
+      - run: uv python install 3.11
+      - run: make quality
+
+  test:
+    name: Tests (Python ${{ matrix.python-version }})
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.11", "3.12", "3.13", "3.14"]
+    env:
+      UV_PYTHON: ${{ matrix.python-version }}
+    steps:
+      - uses: actions/checkout@v7
+      - uses: astral-sh/setup-uv@v7
+      - run: uv python install ${{ matrix.python-version }}
+      - name: Core only, with no codec extras installed
+        run: make test-core
+      - name: With every codec extra installed
+        run: make test-all
+
+  codec-determinism-floor:
+    name: Golden codec bytes under the oldest supported libraries
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+      - uses: astral-sh/setup-uv@v7
+      - run: uv python install 3.11
+      - run: make codec-floor

matview_cache-0.1.0/.github/workflows/docs.yml

@@ -0,0 +1,43 @@
+name: Docs
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: docs-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  build:
+    name: Build site
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+      - uses: astral-sh/setup-uv@v7
+      - run: uv python install 3.11
+      # --strict, so a broken cross-reference or a missing page fails here rather than
+      # shipping as a dead link.
+      - run: make docs
+      - uses: actions/upload-pages-artifact@v5
+        with:
+          path: site
+
+  deploy:
+    name: Deploy to Pages
+    needs: build
+    if: github.ref == 'refs/heads/main' && github.event_name == 'push'
+    runs-on: ubuntu-latest
+    permissions:
+      pages: write
+      id-token: write
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    steps:
+      - id: deployment
+        uses: actions/deploy-pages@v5

matview_cache-0.1.0/.github/workflows/release.yml

@@ -0,0 +1,110 @@
+# Publishing is tag-driven and credential-free: PyPI trusted publishing exchanges this
+# workflow's OIDC token for an upload token, so there is no long-lived secret in the
+# repository. That binds releases to *this file name* and the environment below, which is
+# why renaming either means re-registering the publisher on PyPI.
+name: Release
+
+on:
+  push:
+    tags: ["v*"]
+  # A dry run publishes to TestPyPI instead, so the whole path can be exercised without
+  # burning a version number on the real index.
+  workflow_dispatch:
+    inputs:
+      index:
+        description: Where to publish
+        default: testpypi
+        type: choice
+        options: [testpypi, pypi]
+
+permissions:
+  contents: read
+
+jobs:
+  verify:
+    name: Checks, and the tag against __version__
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v5
+      - run: uv python install 3.11
+      - run: make check
+      # The version has one home, `__version__` in src/matview/__init__.py, and hatchling
+      # reads the distribution's version from it. A tag that disagrees would publish a
+      # file whose name and metadata contradict each other, so it fails here instead.
+      - name: The tag must match the packaged version
+        if: startsWith(github.ref, 'refs/tags/v')
+        run: |
+          set -euo pipefail
+          packaged=$(uv run --no-project --with . python -c 'import matview; print(matview.__version__)')
+          tag=${GITHUB_REF_NAME#v}
+          if [ "$packaged" != "$tag" ]; then
+            echo "tag v$tag disagrees with __version__ $packaged" >&2
+            exit 1
+          fi
+
+  build:
+    name: Build and check the distributions
+    needs: verify
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v5
+      - run: uv python install 3.11
+      # Builds the sdist and wheel, validates the metadata, and installs the wheel in
+      # isolation to check that the package it produces actually imports, ships py.typed
+      # and runs its console script.
+      - run: make release-check
+      - uses: actions/upload-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+  publish:
+    name: Publish to ${{ (github.event_name == 'push' || inputs.index == 'pypi') && 'PyPI' || 'TestPyPI' }}
+    needs: build
+    runs-on: ubuntu-latest
+    environment:
+      name: ${{ (github.event_name == 'push' || inputs.index == 'pypi') && 'pypi' || 'testpypi' }}
+    permissions:
+      id-token: write
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+      - uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          repository-url: ${{ (github.event_name == 'push' || inputs.index == 'pypi') && 'https://upload.pypi.org/legacy/' || 'https://test.pypi.org/legacy/' }}
+
+  github-release:
+    name: GitHub release
+    needs: publish
+    if: startsWith(github.ref, 'refs/tags/v')
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+      # The release notes are the CHANGELOG section for this version, not a generated
+      # commit list: the changelog is written for readers and the commit list is not.
+      - name: Extract this version's changelog section
+        run: |
+          set -euo pipefail
+          version=${GITHUB_REF_NAME#v}
+          awk -v v="$version" '
+            $0 ~ "^## \\[" v "\\]" {inside = 1; next}
+            inside && /^## / {exit}
+            inside {print}
+          ' CHANGELOG.md > notes.md
+          if [ ! -s notes.md ]; then
+            echo "CHANGELOG.md has no section for $version" >&2
+            exit 1
+          fi
+      - env:
+          GH_TOKEN: ${{ github.token }}
+        run: gh release create "$GITHUB_REF_NAME" --title "$GITHUB_REF_NAME" --notes-file notes.md dist/*

matview_cache-0.1.0/.gitignore

@@ -0,0 +1,17 @@
+__pycache__/
+*.py[cod]
+*.egg-info/
+build/
+dist/
+.venv/
+venv/
+.pytest_cache/
+.mypy_cache/
+.ruff_cache/
+.coverage
+htmlcov/
+*.db
+*.db-wal
+*.db-shm
+uv.lock
+site/

matview_cache-0.1.0/.pre-commit-config.yaml

@@ -0,0 +1,42 @@
+# Run on every commit, so a formatting slip is caught before the push rather than by
+# a red CI run afterwards.
+#
+#     uv run --extra dev pre-commit install
+#
+# ruff and mypy are pinned to exactly the versions in the `dev` extra. Unpinned, the
+# hook and the local venv drift apart and the same code passes in one and fails in the
+# other, which is confusing in proportion to how rarely it happens.
+# SPEC.md and docs/design-spec.md are documents reproduced as given, not code. SPEC.md
+# in particular states in its own header that it is verbatim, and it is the contract a
+# second-language implementation is validated against. A whitespace hook quietly
+# editing either would make that claim false and add noise to any diff against the
+# source document, so they are out of scope for the fixers.
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v6.0.0
+    hooks:
+      - id: trailing-whitespace
+        exclude: ^(SPEC\.md|docs/design-spec\.md)$
+      - id: end-of-file-fixer
+        exclude: ^(SPEC\.md|docs/design-spec\.md)$
+      - id: check-yaml
+      - id: check-toml
+      - id: check-merge-conflict
+      - id: check-added-large-files
+        args: ["--maxkb=512"]
+      - id: mixed-line-ending
+        args: ["--fix=lf"]
+
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.16.8
+    hooks:
+      - id: ruff-check
+        args: ["--fix"]
+      - id: ruff-format
+
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v2.3.1
+    hooks:
+      - id: mypy
+        files: ^src/matview/
+        additional_dependencies: ["rfc8785>=0.1.4"]

matview_cache-0.1.0/AGENTS.md

@@ -0,0 +1,56 @@
+# Working on matview
+
+Read `docs/design-spec.md` before changing behaviour, `ROADMAP.md` for what is and is
+not planned, and `DECISIONS.md` for what has already been settled where the spec was
+silent. Add an entry to `DECISIONS.md` for every new judgement call; the spec requires it.
+
+Each of the ten rules below has a test named after it in `tests/test_hard_rules.py`, so a
+violation names itself in CI output.
+
+## Rules that are not style preferences
+
+These are from the spec's "Rules the implementation must not violate". Each one is
+enough on its own to make the library silently wrong rather than loudly broken, so a
+plausible-looking simplification that touches one of them is a bug, not a cleanup.
+
+1. Never delete an artifact on invalidation. Clear `valid`, keep the bytes.
+2. Never cache a failed compute, in any form.
+3. Artifact, dep edges and cascade land in one transaction or not at all.
+4. Never rewrite a row whose digest and fingerprint already match.
+5. `force` is local; only `force_deps` is transitive.
+6. `on_miss` resolves to the strictest of caller and recipe, never the caller's alone.
+7. Digests come from codec output; codecs must be byte-deterministic.
+8. A dependency's recipe version never enters a dependent's fingerprint.
+9. Empty or `None` results are values, not misses.
+10. Never use pickle.
+
+Rule 8 in particular reads like a bug to anyone encountering it fresh. It is
+deliberate: it is what buys early cutoff. The comment in the code says so; leave it there.
+
+## Other standing constraints
+
+- `SPEC.md` is the cross-language interoperability contract. Changing it invalidates
+  every existing cache, so it changes only by bumping `fingerprint_spec_version`. If it
+  looks wrong, say so rather than editing it.
+- Anything under the spec's "Not in v1" must not exist, including as a hidden fast path
+  or an unused code path.
+- The recipe registry is global and import-time. `Cache` must not be. SQLite connections
+  are per-thread with WAL on.
+- Stale is not absent. Three states, kept distinct everywhere: absent, stale, valid.
+
+## Conventions
+
+- Python 3.11+, `uv` for environments, `ruff` for lint and format, `mypy --strict`.
+- The package lives under `src/`, so the tests import the *installed* distribution rather
+  than the working directory. That is what makes a packaging mistake fail locally.
+- **Run `make check` before pushing.** It is what CI runs: `quality`, `test-core`,
+  `test-all`, `codec-floor`. `make help` lists the rest.
+- The commands live in the `Makefile` and CI calls those targets, so changing how
+  something is checked is an edit there — not an edit under `.github/workflows/`, which
+  needs a token scope this account does not have.
+- `make format-check` is not implied by `make lint`. Running `ruff check --fix` and
+  committing on the strength of it has broken CI here before; `make quality` runs both.
+- The test suite must also pass with **no** codec extras installed (`make test-core`);
+  the core has to work without them.
+- Docstrings are the API documentation: they are rendered by mkdocstrings into the
+  published reference, so they are read by users, not only by readers of the source.

matview_cache-0.1.0/CHANGELOG.md

@@ -0,0 +1,159 @@
+# Changelog
+
+All notable changes to this project are documented here.
+
+The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this
+project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+The one thing that is **not** versioned by this file is the fingerprint specification.
+It lives in [`SPEC.md`](SPEC.md) with its own `fingerprint_spec_version`, recorded in
+every database's `meta` table. Changing it invalidates every existing cache, so it
+changes only by bumping that number — never as part of an ordinary release.
+
+## [Unreleased]
+
+## [0.1.0] — 2026-10-13
+
+First release.
+
+### Added
+
+- Graph queries over the DAG the cache records as it runs, in a new `matview.graph`
+  module and three read-only commands. `lineage` walks upward to the raw inputs that
+  produced a value — the question the library recorded everything for and could not
+  answer. `impact` answers what a changed artifact, a recipe version bump or a `drop`
+  would reach. `cost` reports embodied `compute_ms` over an input closure, a per-recipe
+  rollup, and the critical path through a closure. Four renderings each: indented tree,
+  JSON, Graphviz DOT and Mermaid.
+
+  None of them needs `--recipes`: they read the recorded graph, so they work against a
+  database whose recipes the process has never imported. None of them writes, `impact
+  --drop` included.
+
+  `impact` prints per-recipe counts by default and enumerates only under `--all`. One
+  shared root in the 250k benchmark profile reaches 255,000 dependents.
+
+  `impact --bump` exists mainly to state plainly what rule 8 means: bumping a recipe's
+  version marks **nothing** stale, because a dependency's recipe version is not part of a
+  dependent's fingerprint. The artifacts become unreachable, each dependent re-resolves
+  at the new version on its next read, and the cascade runs only if the recomputed bytes
+  differ. That is early cutoff, and it is the part of the fingerprint spec that confuses
+  every new reader.
+
+- `SqliteStore.inputs_of()`, the transitive upward walk, mirroring `dependents_of()`.
+  Both now take a `depth`. Both terminate on a cycle a corrupted database could contain
+  rather than hanging. Also `deps_of_many()`, `refs_by_ids()`, `ids_of_recipe()`,
+  `counts_by_recipe()` and `recipe_cost_rows()`.
+
+- Generic recipe `max_age` freshness, backed by a distinct `refreshed_at` clock so a
+  successful refresh with unchanged bytes becomes fresh without rewriting content
+  history. Scheduling and domain calendars remain caller concerns. The new artifact
+  column advances the unreleased SQLite schema to version 2.
+
+- Content-addressed compute cache over a dependency graph discovered by execution: a
+  suspending scheduler with constructive traces. Every `ctx.get` is recorded, and the
+  artifact, its dep edges and the invalidation cascade land in one transaction.
+- Three artifact states kept distinct — absent, stale, valid. A stale artifact is
+  verified by recomputing its fingerprint from its stored edges, and returned without
+  recomputing when it still matches. Invalidation never deletes.
+- `SqliteStore`, WAL, connection per thread, writers serialized on `BEGIN IMMEDIATE`,
+  with a configurable `max_artifact_bytes` (64 MiB) that raises rather than writing.
+- Policy: an `on_miss` ceiling resolved against each recipe's own declaration, local
+  `force` and transitive `force_deps`, a propagated wall-clock `deadline`, and batched
+  reporting of every protected miss the walk could reach.
+- `plan()`, which walks without computing and reports absent and stale separately.
+- Codecs `json`, `arrow`, `arrow-series` and `npy`, each byte-deterministic, with
+  partitioned output supported through `decode_parts`. The arrow codecs encode through
+  pyarrow with pinned IPC options rather than through polars' writer, whose output is
+  not stable across polars releases.
+- Foreign recipes: a TOML manifest plus an executable, with inputs declared upfront so
+  the graph can be cycle-checked and planned before anything runs. One JSON job file as
+  the only argument; results read from a file, never from stdout.
+- `SPEC.md` as the cross-language interoperability contract, with golden vectors for
+  hashes and for codec bytes checked into the test suite.
+- Two examples, one financial and one not, both exercised by the test suite.
+
+- `arrow-year` and `arrow-year-pandas`, slicing a frame into one artifact per calendar
+  year on a column named `date`, with `ArrowYearCodec(name=..., column=...)` for any other
+  column. Byte-identical to each other, sorted so that a reordered frame encodes the same,
+  reading timezone-aware timestamps in UTC, and refusing null dates. Appending a day
+  rewrites one part and leaves the closed years untouched, which the tests assert end to
+  end rather than assuming.
+- An `arrow-pandas` codec, byte-identical to `arrow`, decoding to a pandas DataFrame.
+  Either codec reads what the other wrote, so a cache is not partitioned by dataframe
+  library, and a pandas consumer does not need polars installed.
+- A `matview` command-line tool for inspecting a cache: `info`, `ls`, `show`, `stats`,
+  `verify`, `cat` and `plan`. Read-only, `--json` everywhere, stdlib-only so it is present
+  on a plain install. Metadata inspection imports nothing; `--recipes` is required only
+  for a partial key and for `plan`.
+- `SqliteStore(read_only=True)`, which requires the file to exist, creates no schema and
+  leaves the journal mode alone.
+- `inspect_database()`, which describes a database whatever version wrote it.
+- `get_many` and `plan_many`: many roots in one walk, sharing a memo. A single root
+  stops at its first blocked `ctx.get`; independent roots do not stop each other, which
+  is what makes the spec's claim about cold runs — one report of everything that needs
+  ingesting — actually true.
+- `cache.stats.expired`, counting nodes whose `max_age` had elapsed. A reason rather
+  than an outcome, so it is excluded from `lookups`: without it, a rebuild driven by a
+  clock is indistinguishable from one driven by a changed input.
+- `cache.stats`, and a `matview.scheduler` logger emitting one DEBUG event per node
+  resolved. The counters separate a plain hit from a revalidation-without-recompute
+  from a genuine rebuild, which a two-way hit/miss count hides.
+- `MissingNode.path`, the resolution chain from the root, so a protected miss says which
+  node wanted it rather than only that it is missing.
+- `SqliteStore` refuses a database whose `schema_version` it does not implement, as it
+  already did for `fingerprint_spec_version`.
+- `matview --version`, on the top-level parser only. Several subcommands already take a
+  `--version` that filters by recipe version, and the two meanings cannot share a
+  command line.
+- Python 3.14 is supported and classified. The full suite with every codec extra passes on
+  it, and the golden codec fixtures match, so stored artifacts are portable across it.
+
+### Fixed
+
+- `SqliteStore.summary()` overcounted `compute_ms`. It summed the column over artifact
+  rows, and every part of a partitioned value records the cost of the single compute that
+  produced them all — so a three-part value was counted three times. It now groups by
+  `(recipe, version, key_hash)` and reports a `values` count beside `artifacts`, so the
+  deduplication is visible rather than implicit. Every cost aggregate added in this
+  release does the same.
+
+- Maintenance operations over large artifact sets no longer exceed SQLite's bound-variable
+  limit, and cache verification now uses keyed rather than `OFFSET` pagination. SQLite
+  JSON1 (included by default in SQLite 3.38+) is required and checked when opening a store.
+
+- The codec-floor job silently verified a stale build of matview: `uv run --no-project
+  --with .` reinstalls a cached wheel. It refreshes the build now, and a test asserts the
+  installed codec set matches the checked-in fixture, because the symptom was invisible —
+  fewer parametrized cases, reported as success.
+- Arrow offset widths are now normalized, so the stored bytes no longer depend on which
+  dataframe library produced the table. polars emits `large_string`, pandas 2.2 emits
+  `string` and pandas 3.0 emits `large_string`, which made identical data hash differently
+  by producer and moved with a pandas *minor* version. Nothing already stored changed:
+  polars was already producing the wide variants, so the normalization is a no-op on that
+  path and every existing digest holds.
+- Enabling WAL could fail with "database is locked" when two processes opened the same
+  new file at the same moment. SQLite does not invoke the busy handler for a
+  journal-mode change, so `busy_timeout` did not cover it — the exact deployment the
+  design targets. Found by the first cross-process test.
+- `matview.__all__` named `CacheStats` and `BatchResult` without importing them, so
+  `from matview import CacheStats` raised. A test now checks every name in every
+  module's `__all__` actually resolves.
+
+### Changed
+
+- `BaseCodec.partitions` removed: it was read nowhere, and `decode_parts` refusing is the
+  actual mechanism.
+- The package now lives under `src/`, so the test suite exercises the installed
+  distribution rather than the working directory and a packaging mistake fails locally
+  rather than only for a user.
+
+### Notes
+
+- Every choice made where the design document was silent is recorded, with its
+  reasoning, in [`DECISIONS.md`](DECISIONS.md).
+- `plan_many()` remains roughly two orders of magnitude slower per root than
+  `get_many()`, and the batching that was expected to close the gap does not: the loop it
+  would batch is over the parts of one node, not over nodes. The real N+1 is across the
+  depth-first descent and removing it is a scheduler change. Measured both ways; see
+  DECISIONS.md D52 and [`docs/performance.md`](docs/performance.md).

matview_cache-0.1.0/CITATION.cff

@@ -0,0 +1,28 @@
+cff-version: 1.2.0
+message: If you use matview in published work, please cite it.
+title: "matview: materialized views for derived values"
+abstract: >-
+  A content-addressed compute cache over a dependency graph. Recipes declare how a value
+  is computed from parameters and from other recipes' outputs; the graph is recorded as
+  it runs rather than declared, staleness is decided by content digests rather than
+  timestamps, and results are memoized in SQLite for reuse across processes and machines.
+type: software
+authors:
+  - family-names: Tresoldi
+    given-names: Tiago
+    email: tresoldi@gmail.com
+    # orcid: https://orcid.org/0000-0000-0000-0000
+license: MIT
+repository-code: https://github.com/tresoldi/matview
+url: https://tresoldi.github.io/matview/
+keywords:
+  - cache
+  - build system
+  - dependency graph
+  - memoization
+  - reproducibility
+  - sqlite
+version: 0.1.0
+date-released: 2026-10-13
+# A Zenodo doi is added here once the archive assigns one at the first tag; see
+# RELEASING.md.