tallyman-data 0.0.1__tar.gz

tallyman_data-0.0.1/.github/workflows/tests.yml

@@ -0,0 +1,74 @@
+name: tests
+
+on:
+  push:
+    branches: [main, "spike/**"]
+  pull_request:
+    branches: [main]
+
+# CI runs three sequential jobs: lint → fast tests → integration tests.
+#
+# - The React SPA (packages/app/) IS built in the fast job: the SPA-serving
+#   tests assert the real built index.html (#root div), so dist/ must exist
+#   or create_app's catch-all returns 503. The embed bundle (packages/embed/)
+#   is still NOT built — nothing serves it under test.
+# - Integration tests spawn the real buckaroo Tornado subprocess via
+#   tallyman's BuckarooManager. buckaroo==0.14.6 is pinned in
+#   pyproject.toml as a normal PyPI dep, so plain `uv sync` is enough.
+
+jobs:
+  lint:
+    name: ruff
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v3
+        with:
+          enable-cache: true
+      - name: ruff check
+        # uvx runs the latest ruff without polluting the project venv;
+        # matches the local pre-push hook in ~/.claude/settings.json.
+        run: uvx ruff check
+
+  fast:
+    name: fast suite (excludes integration)
+    runs-on: ubuntu-latest
+    needs: lint
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v3
+        with:
+          enable-cache: true
+      # The SPA-serving tests hit create_app's catch-all, which reads
+      # packages/app/dist/index.html. Build it here (pnpm 9 matches the
+      # lockfileVersion 9.0) so those routes return the page, not a 503.
+      - uses: pnpm/action-setup@v4
+        with:
+          version: 9
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 20
+          cache: pnpm
+          cache-dependency-path: packages/app/pnpm-lock.yaml
+      - name: build React SPA
+        run: pnpm -C packages/app install --frozen-lockfile && pnpm -C packages/app build
+      - name: vitest (React unit)
+        run: pnpm -C packages/app test
+      - name: sync deps
+        run: uv sync
+      - name: pytest (fast)
+        # pyproject.toml's addopts already excludes `-m integration`;
+        # being explicit here so the job name matches the behaviour.
+        run: uv run pytest -m "not integration"
+
+  integration:
+    name: integration suite
+    runs-on: ubuntu-latest
+    needs: fast
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v3
+        with:
+          enable-cache: true
+      - run: uv sync
+      - run: uv run pytest -m integration

tallyman_data-0.0.1/.gitignore

@@ -0,0 +1,18 @@
+.venv/
+__pycache__/
+*.pyc
+*.egg-info/
+dist/
+build/
+.ruff_cache/
+.pytest_cache/
+.DS_Store
+node_modules/
+src/tallyman_companion/static/*.map
+# vite build output — rebuild via `cd packages/embed && pnpm build`
+src/tallyman_companion/static/buckaroo-embed.js
+src/tallyman_companion/static/buckaroo-embed.css
+# xorq-backed catalog git repo (managed externally via `xorq catalog`)
+catalog/
+# Stray ./projects/ from anything run with TALLYMAN_HOME unset to repo root.
+projects/

tallyman_data-0.0.1/.mcp.json

@@ -0,0 +1,13 @@
+{
+  "mcpServers": {
+    "tallyman": {
+      "command": "uv",
+      "args": ["run", "tallyman", "mcp"],
+      "cwd": "/Users/paddy/tallyman",
+      "env": {
+        "TALLYMAN_PROJECT": "spike",
+        "TALLYMAN_COMPANION_URL": "http://127.0.0.1:7860"
+      }
+    }
+  }
+}

tallyman_data-0.0.1/PKG-INFO

@@ -0,0 +1,163 @@
+Metadata-Version: 2.4
+Name: tallyman-data
+Version: 0.0.1
+Summary: tallyman — deconstructed notebook platform
+Requires-Python: <3.14,>=3.13
+Requires-Dist: buckaroo==0.14.15
+Requires-Dist: click>=8.1
+Requires-Dist: fastapi>=0.115
+Requires-Dist: fastmcp>=2.0
+Requires-Dist: geopandas>=1.1.3
+Requires-Dist: h3>=4.4.2
+Requires-Dist: httpx>=0.27
+Requires-Dist: ibis-framework>=12.0.0
+Requires-Dist: markdown>=3.10.2
+Requires-Dist: pandas>=2.2
+Requires-Dist: pillow>=12.2.0
+Requires-Dist: polars
+Requires-Dist: pyarrow>=16
+Requires-Dist: pydantic>=2.7
+Requires-Dist: pygments>=2.20.0
+Requires-Dist: scikit-learn<2.0,>=1.4
+Requires-Dist: scipy>=1.17.1
+Requires-Dist: seaborn>=0.13.2
+Requires-Dist: sse-starlette>=2.1
+Requires-Dist: tomli-w>=1.0
+Requires-Dist: tornado>=6.0
+Requires-Dist: uvicorn[standard]>=0.30
+Requires-Dist: vl-convert-python>=1.9.0.post1
+Requires-Dist: xorq==0.3.26
+Description-Content-Type: text/markdown
+
+# tallyman-notebooks
+
+Spike for the Tallyman London 2026 talk *"The Future of Notebooks in a Claude Code World"*.
+The plan and proposal live in `plan.md` / `proposal.md`. This README covers the V0 spike only.
+
+## V0 scope
+
+End-to-end: a Claude Code MCP tool that compiles a xorq expression, materializes
+a parquet to a content-hashed catalog entry on disk, and pushes a live update
+to a browser companion via SSE.
+
+What's working:
+
+- **MCP tools:**
+  - Catalog: `catalog_run`, `catalog_load_parquet`, `catalog_create`,
+    `catalog_revise`, `catalog_alias`, `catalog_rename`, `catalog_unalias`,
+    `catalog_list`, `catalog_diff`.
+  - Notebook: `notebook_reorder`, `notebook_remove`, `notebook_edit_markdown`.
+- **Companion** (FastAPI on `:7860`):
+  - `/catalog`, `/catalog/<hash>` or `/catalog/<alias>` — entry list and detail
+    with V_n chips, forensic history, and a link to internal lineage.
+  - `/notebook` — curated narrative: cells anchored on aliases, vertical layout,
+    inline markdown editor, ↑/↓ reorder, × remove.
+  - `/lineage` and `/lineage/<hash>` — catalog DAG (cross-entry parents derived
+    from `from_catalog`) and per-entry internal expression DAG. Pure SVG, no
+    Cytoscape dep.
+  - `/diff/<alias>[/<va>/<vb>]` — version diff with code diff, schema diff,
+    per-column stats, key-joined side-by-side, and head() side-by-side.
+  - `/errors/<id>` — build-failure detail.
+  - `/api/{entries,aliases,errors,notebook,lineage,catalog_dag}` — JSON.
+  - `/api/sse` — live updates (`new_entry`, `build_failed`, `alias_changed`,
+    `notebook_changed`).
+- **Buckaroo subprocess** — `tallyman run` spawns `python -m buckaroo.server`
+  on `:8700` (falls back to a random port if busy), watches for the
+  `BUCKAROO_PORT=...` handshake, and lazily creates per-entry sessions on
+  first view by POSTing the entry's `xorq_build/` dir to Buckaroo's
+  `/load_expr` endpoint (PR 776) — sort/search push down to the xorq
+  backend rather than paging over a materialised parquet. The build dir
+  is expanded into a tmp copy first so `${TALLYMAN_PROJECT_ROOT}`
+  placeholders are resolved before xorq's loader sees them. Sessions are
+  persisted under `catalog/buckaroo_sessions.json` and invalidated by
+  start-time when Buckaroo restarts; tmp dirs are cleaned on
+  `BuckarooManager.stop()`. Tear-down rides along with the companion.
+  Disable with `--no-buckaroo`.
+- **Build artifacts are portable.** xorq's absolute filesystem paths are
+  rewritten to `${TALLYMAN_PROJECT_ROOT}` on write and expanded back on load.
+- **`tallyman serve <project_dir>`** — read-only companion against a project
+  directory that may live anywhere on disk. Mutation routes return 403.
+
+What's NOT yet implemented (see `TICKETS.md` for the full punchlist):
+
+1. SortableJS drag-reorder for the notebook (current ↑/↓ buttons are the
+   accessibility fallback; drag is the headline UX).
+2. Column-level lineage (xorq has the data; current view is op-level only).
+3. `tallyman pack` / `tallyman replay` (today's hand-off is `cp -r` / `tar`).
+4. ML training pipeline (storyboard beats 7-8).
+
+## Running the spike
+
+```sh
+uv sync
+uv run tallyman init spike            # creates ~/.tallyman/projects/spike/ + fixture
+uv run tallyman run --project spike   # edit-mode companion on http://127.0.0.1:7860
+```
+
+The companion's dataframe embed is a Vite React library in `packages/embed/`
+that builds into `src/tallyman_companion/static/buckaroo-embed.{js,css}`. The
+build artifact is **not** committed — Node + pnpm are install-time
+prerequisites:
+
+```sh
+cd packages/embed && pnpm install && pnpm build      # one-time per checkout
+# or, for active embed development:
+cd packages/embed && pnpm dev                        # vite build --watch
+```
+
+Bump `buckaroo-js-core` in `packages/embed/package.json` to match the
+Python `buckaroo` pin in `pyproject.toml` and rebuild whenever either
+side moves.
+
+In another terminal, launch Claude Code from this directory; it picks up
+`.mcp.json` and exposes the `tallyman` MCP server.
+
+Recommended prompts:
+
+> Use catalog_load_parquet to load `orders.parquet`.
+>
+> Now use catalog_create to make a named entry `shoe_sales` that groups orders
+> by region and totals the price.
+>
+> Now revise shoe_sales to filter to category == "boots" only.
+
+Watch the browser update live as each tool call lands. Named entries float to
+the top of the catalog with a V_n chip; the previous version sticks around in
+forensic history.
+
+### Serving a project as an artifact
+
+Once you've authored a project, hand it off:
+
+```sh
+tar czf my-project.tgz -C ~/.tallyman/projects spike
+# colleague extracts somewhere
+tar xzf my-project.tgz -C ~/projects/
+uv run tallyman serve ~/projects/spike
+```
+
+The companion runs read-only: same catalog, same forensic history, no edit
+affordances. Mutation routes return 403.
+
+## Conventions worth knowing
+
+- xorq 0.3.x reads use `xo.deferred_read_parquet` (NOT `xo.read_parquet` — that
+  resolves through ibis's backend loader and fails). Use
+  `import xorq.api as xo` and `import xorq.vendor.ibis as ibis`. Do NOT
+  `import ibis` directly.
+- Prefer `from tallyman_xorq.io import from_project; t = from_project("name.parquet")`
+  over absolute paths — the catalog records project-relative intent and the
+  build is portable across machines/users.
+- Content hash is xorq's build hash — same code + same inputs → same hash → same
+  entry dir (idempotent).
+- All catalog state lives on disk. The MCP server holds no in-memory state; the
+  companion only holds the SSE subscriber list.
+- `TALLYMAN_PROJECT_PATH` overrides project_dir() resolution for the active project.
+  Used by `tallyman serve` to point at a project directory anywhere on disk.
+
+## Tests
+
+```sh
+uv run pytest               # 76 tests, ~5s
+uv run pytest tests/test_portable.py    # the portability proof
+```

tallyman_data-0.0.1/README.md

@@ -0,0 +1,132 @@
+# tallyman-notebooks
+
+Spike for the Tallyman London 2026 talk *"The Future of Notebooks in a Claude Code World"*.
+The plan and proposal live in `plan.md` / `proposal.md`. This README covers the V0 spike only.
+
+## V0 scope
+
+End-to-end: a Claude Code MCP tool that compiles a xorq expression, materializes
+a parquet to a content-hashed catalog entry on disk, and pushes a live update
+to a browser companion via SSE.
+
+What's working:
+
+- **MCP tools:**
+  - Catalog: `catalog_run`, `catalog_load_parquet`, `catalog_create`,
+    `catalog_revise`, `catalog_alias`, `catalog_rename`, `catalog_unalias`,
+    `catalog_list`, `catalog_diff`.
+  - Notebook: `notebook_reorder`, `notebook_remove`, `notebook_edit_markdown`.
+- **Companion** (FastAPI on `:7860`):
+  - `/catalog`, `/catalog/<hash>` or `/catalog/<alias>` — entry list and detail
+    with V_n chips, forensic history, and a link to internal lineage.
+  - `/notebook` — curated narrative: cells anchored on aliases, vertical layout,
+    inline markdown editor, ↑/↓ reorder, × remove.
+  - `/lineage` and `/lineage/<hash>` — catalog DAG (cross-entry parents derived
+    from `from_catalog`) and per-entry internal expression DAG. Pure SVG, no
+    Cytoscape dep.
+  - `/diff/<alias>[/<va>/<vb>]` — version diff with code diff, schema diff,
+    per-column stats, key-joined side-by-side, and head() side-by-side.
+  - `/errors/<id>` — build-failure detail.
+  - `/api/{entries,aliases,errors,notebook,lineage,catalog_dag}` — JSON.
+  - `/api/sse` — live updates (`new_entry`, `build_failed`, `alias_changed`,
+    `notebook_changed`).
+- **Buckaroo subprocess** — `tallyman run` spawns `python -m buckaroo.server`
+  on `:8700` (falls back to a random port if busy), watches for the
+  `BUCKAROO_PORT=...` handshake, and lazily creates per-entry sessions on
+  first view by POSTing the entry's `xorq_build/` dir to Buckaroo's
+  `/load_expr` endpoint (PR 776) — sort/search push down to the xorq
+  backend rather than paging over a materialised parquet. The build dir
+  is expanded into a tmp copy first so `${TALLYMAN_PROJECT_ROOT}`
+  placeholders are resolved before xorq's loader sees them. Sessions are
+  persisted under `catalog/buckaroo_sessions.json` and invalidated by
+  start-time when Buckaroo restarts; tmp dirs are cleaned on
+  `BuckarooManager.stop()`. Tear-down rides along with the companion.
+  Disable with `--no-buckaroo`.
+- **Build artifacts are portable.** xorq's absolute filesystem paths are
+  rewritten to `${TALLYMAN_PROJECT_ROOT}` on write and expanded back on load.
+- **`tallyman serve <project_dir>`** — read-only companion against a project
+  directory that may live anywhere on disk. Mutation routes return 403.
+
+What's NOT yet implemented (see `TICKETS.md` for the full punchlist):
+
+1. SortableJS drag-reorder for the notebook (current ↑/↓ buttons are the
+   accessibility fallback; drag is the headline UX).
+2. Column-level lineage (xorq has the data; current view is op-level only).
+3. `tallyman pack` / `tallyman replay` (today's hand-off is `cp -r` / `tar`).
+4. ML training pipeline (storyboard beats 7-8).
+
+## Running the spike
+
+```sh
+uv sync
+uv run tallyman init spike            # creates ~/.tallyman/projects/spike/ + fixture
+uv run tallyman run --project spike   # edit-mode companion on http://127.0.0.1:7860
+```
+
+The companion's dataframe embed is a Vite React library in `packages/embed/`
+that builds into `src/tallyman_companion/static/buckaroo-embed.{js,css}`. The
+build artifact is **not** committed — Node + pnpm are install-time
+prerequisites:
+
+```sh
+cd packages/embed && pnpm install && pnpm build      # one-time per checkout
+# or, for active embed development:
+cd packages/embed && pnpm dev                        # vite build --watch
+```
+
+Bump `buckaroo-js-core` in `packages/embed/package.json` to match the
+Python `buckaroo` pin in `pyproject.toml` and rebuild whenever either
+side moves.
+
+In another terminal, launch Claude Code from this directory; it picks up
+`.mcp.json` and exposes the `tallyman` MCP server.
+
+Recommended prompts:
+
+> Use catalog_load_parquet to load `orders.parquet`.
+>
+> Now use catalog_create to make a named entry `shoe_sales` that groups orders
+> by region and totals the price.
+>
+> Now revise shoe_sales to filter to category == "boots" only.
+
+Watch the browser update live as each tool call lands. Named entries float to
+the top of the catalog with a V_n chip; the previous version sticks around in
+forensic history.
+
+### Serving a project as an artifact
+
+Once you've authored a project, hand it off:
+
+```sh
+tar czf my-project.tgz -C ~/.tallyman/projects spike
+# colleague extracts somewhere
+tar xzf my-project.tgz -C ~/projects/
+uv run tallyman serve ~/projects/spike
+```
+
+The companion runs read-only: same catalog, same forensic history, no edit
+affordances. Mutation routes return 403.
+
+## Conventions worth knowing
+
+- xorq 0.3.x reads use `xo.deferred_read_parquet` (NOT `xo.read_parquet` — that
+  resolves through ibis's backend loader and fails). Use
+  `import xorq.api as xo` and `import xorq.vendor.ibis as ibis`. Do NOT
+  `import ibis` directly.
+- Prefer `from tallyman_xorq.io import from_project; t = from_project("name.parquet")`
+  over absolute paths — the catalog records project-relative intent and the
+  build is portable across machines/users.
+- Content hash is xorq's build hash — same code + same inputs → same hash → same
+  entry dir (idempotent).
+- All catalog state lives on disk. The MCP server holds no in-memory state; the
+  companion only holds the SSE subscriber list.
+- `TALLYMAN_PROJECT_PATH` overrides project_dir() resolution for the active project.
+  Used by `tallyman serve` to point at a project directory anywhere on disk.
+
+## Tests
+
+```sh
+uv run pytest               # 76 tests, ~5s
+uv run pytest tests/test_portable.py    # the portability proof
+```