smartpipe-cli 1.3.1__tar.gz → 1.4.0rc1__tar.gz

{smartpipe_cli-1.3.1 → smartpipe_cli-1.4.0rc1}/.gitignore

@@ -23,3 +23,7 @@ context/
 /plan/
 /TODO.md
 /idea.md
+
+# Local-only scratch folder — never tracked, never in the sdist
+/local/
+.claude/

{smartpipe_cli-1.3.1 → smartpipe_cli-1.4.0rc1}/CHANGELOG.md

@@ -5,6 +5,86 @@ Format: [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) · Versioning:
 
 ## [Unreleased]
 
+## [1.4.0] — 2026-07-08
+
+The identity release.
+
+### The item — one law for everything in a pipe
+- **The `__` metadata spine.** Tool metadata lives in a reserved
+  double-underscore namespace that rides every record: `__source` (path plus
+  HOW the item was cut - file, lines, jsonl, pages, minutes, segment index),
+  `__media` (one transport object for bytes crossing a pipe), `__score`
+  (join), `__invalid`/`__error`/`__raw` (kept failures). Known fields
+  round-trip through saved files; unknown `__` fields warn once and carry.
+  User data owns everything up to one leading underscore.
+- **Records in, records out.** A plain-prompt `map` over records now returns
+  a record (`{"result": …}` plus the spine) instead of bare prose - structure
+  and provenance survive transformation. Text lines still leave as text.
+- **The `--as` ingestion dial** (`file` | `lines` | `jsonl`) on every input
+  verb. Auto defaults: `.jsonl` means rows (loud per-line errors on
+  non-JSON), everything else means whole-document; stdin keeps per-line
+  sniffing; `--as file` slurps stdin whole (new). Media files refuse text
+  granularities with signposts to `split`.
+- **The binary is the reader.** `smartpipe report.pdf | …` - a first
+  argument that exists on disk streams items; verbs always win names,
+  `./name` disambiguates. Every input verb also takes positional FILE
+  arguments; `--in` survives as a hidden alias.
+- **`write` — the egress mirror.** Items route to files by template
+  (`{name}`, `{stem}`, `{index}`, any record field for content fan-out);
+  egress mirrors ingress via the spine - whole-file items become files,
+  line/row items reassemble into their sources in original order. Emits
+  written paths so pipes continue. Text-only records leave as plain text.
+- **`readable` — the human door.** The same YAML-ish renderer as the TTY
+  preview, as an explicit pipe stage: nesting indented, lists bulleted,
+  multi-line strings as block scalars, spine dimmed, long values truncated
+  with counts (`--full` to disable), media summarized (`image/png (48 KB)`)
+  - never base64. Piped machine output is untouched JSONL; `--bare` strips
+  the spine from it.
+- **Mixed streams are visible.** A kind census notes mixed inputs;
+  `where`/`summarize` report rows that had no fields; `--strict-rows` turns
+  the notes into errors. Multi-line plain output into a pipe warns that
+  line framing is ambiguous (use `--output json`).
+
+### Judgment you can gate, retry, and keep
+- **`--keep-invalid`** (map/extend): after the repair retry fails, keep the
+  failure as one JSON row (`__invalid`/`__error`/`__raw`) instead of a skip
+  - dim one-liner at a TTY, full row when piped.
+- **`smartpipe schema`** - the free rungs: compile braces/DSL to JSON Schema,
+  `--check FILE` validates rows (exit 0/1), `--example` prints a validating
+  instance, bare stdin mode is a REPL. **`--dry-run`** (map/extend) prints
+  the composed first request and exits before any model resolution.
+- **Nullability is declared.** `?` on any type (`string?`, `number[]?`, bare
+  `field?`) compiles to a null union; bare fields mean scalar-or-scalar-list
+  and never admit null (D48, shipped earlier in this cycle).
+- **Circuit breaker + `fallback-model`.** Five consecutive transport
+  failures stop the run with a provider-down screen (`SMARTPIPE_BREAKER`
+  tunes; 0 disables) - or, with a configured fallback, the run switches
+  chat models wholesale, re-runs the failed window on the successor, and
+  the receipt splits counts per model. Embedding fallback is refused
+  (one vector space per run). One fallback, no chains.
+- **Deterministic rungs.** `join --on 'left.K == right.K'` alone is a free
+  key-equality join (all kinds, `--unmatched`); with a prompt it becomes
+  blocking - equality narrows pairs, the judge reasons within blocks, the
+  receipt reports pairs avoided. `distinct --exact` folds byte/value
+  duplicates only (records canonicalized, media hashed by bytes) - zero
+  model calls. Verb tables mark the free rungs.
+
+### The terminal behaves
+- Result writes pause/erase/redraw the status line under an arbiter - the
+  spinner can never interleave with output again. Only the final pipe stage
+  (stdout a TTY) animates; mid-pipe stages stay line-atomic.
+- Records render at the TTY as YAML-ish blocks. The welcome screen links
+  docs, cookbook, and issues. The wizard offers to install shell
+  completions (consented, idempotent).
+
+### Docs: a shape, not a pile
+- New Learn track (`docs/learn/1…6`), `concepts/the-item.md` (the five
+  laws), `concepts/feeding-smartpipe.md` (the ingestion chapter), nav
+  regrouped Learn/Verbs/Concepts/Cookbook/Reference, index as mode-router,
+  every example reconciled to the new syntax with outputs from tests, qa/
+  flows extended (a live walk of which caught and fixed a real write
+  round-trip bug).
+
 ## [1.3.1] — 2026-07-08
 
 ### Fixed

{smartpipe_cli-1.3.1 → smartpipe_cli-1.4.0rc1}/CITATION.cff

@@ -7,5 +7,5 @@ authors:
     given-names: Prabal
 repository-code: "https://github.com/prabal-rje/smartpipe"
 license: Apache-2.0
-version: 1.3.1
+version: 1.4.0rc1
 date-released: '2026-07-08'

smartpipe_cli-1.4.0rc1/PKG-INFO

@@ -0,0 +1,264 @@
+Metadata-Version: 2.4
+Name: smartpipe-cli
+Version: 1.4.0rc1
+Summary: Semantic pipes and queries for your terminal - PDFs, images, audio, video, and text through Unix verbs that understand.
+Project-URL: Homepage, https://github.com/prabal-rje/smartpipe
+Project-URL: Changelog, https://github.com/prabal-rje/smartpipe/blob/main/CHANGELOG.md
+Project-URL: Issues, https://github.com/prabal-rje/smartpipe/issues
+Author-email: Prabal Gupta <prabal@rjeinc.ca>
+License-Expression: Apache-2.0
+License-File: LICENSE
+License-File: NOTICE
+Keywords: audio,cli,embeddings,grep,llm,multimodal,ollama,pdf,pipes,semantic,unix,video
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Text Processing :: Filters
+Classifier: Topic :: Utilities
+Requires-Python: >=3.11
+Requires-Dist: anthropic>=0.40
+Requires-Dist: click>=8.1
+Requires-Dist: fastembed>=0.4; python_version < '3.14'
+Requires-Dist: faster-whisper>=1.0; python_version < '3.14'
+Requires-Dist: httpx>=0.27
+Requires-Dist: imageio-ffmpeg>=0.5
+Requires-Dist: jsonschema>=4.23
+Requires-Dist: markitdown[docx,pdf,pptx,xlsx]>=0.1.6; python_version < '3.14'
+Requires-Dist: pdfminer-six>=20231228
+Requires-Dist: pypdf>=5.0
+Requires-Dist: svgwrite>=1.4
+Requires-Dist: tomli-w>=1.0
+Description-Content-Type: text/markdown
+
+# smartpipe
+
+[![CI](https://github.com/prabal-rje/smartpipe/actions/workflows/ci.yml/badge.svg)](https://github.com/prabal-rje/smartpipe/actions/workflows/ci.yml)
+[![PyPI](https://img.shields.io/pypi/v/smartpipe-cli)](https://pypi.org/project/smartpipe-cli/)
+[![Python](https://img.shields.io/badge/python-3.11%E2%80%933.13-blue)](pyproject.toml)
+[![Docs](https://img.shields.io/badge/docs-prabal--rje.github.io%2Fsmartpipe-blue)](https://prabal-rje.github.io/smartpipe/)
+[![License: Apache 2.0](https://img.shields.io/badge/License-Apache_2.0-blue)](LICENSE)
+
+**Semantic pipes and queries for your terminal.**
+
+Run PDFs, images, audio, video, and text through Unix verbs that understand
+their input.
+
+[![smartpipe demo - click to play with sound and controls](https://github.com/prabal-rje/smartpipe/releases/download/demo-assets/smartpipe-teaser-v2.gif)](https://prabal-rje.github.io/smartpipe/demo/)
+
+*[▶ Watch the full 70-second demo](https://prabal-rje.github.io/smartpipe/demo/) - sound, controls, no download.*
+
+## Install
+
+```bash
+# zero-install trial (or: pip install smartpipe-cli)
+uvx --from smartpipe-cli smartpipe
+```
+
+## Point it at a model
+
+```bash
+# log in with a ChatGPT account - covers most people
+smartpipe auth login
+```
+
+No ChatGPT plan? Use local [Ollama][ollama] or a cloud API key - see
+[Models & providers][models].
+
+## Examples
+
+```bash
+# summarize each filing, figures included
+smartpipe map "summarize the key risk" 'filings/*.pdf'
+
+# keep only the calls that sound frustrated - audio, understood
+smartpipe filter "the caller sounds frustrated" 'calls/*.mp3'
+
+# text on stdin works the same way
+echo "hello world" \
+| smartpipe map "translate to Spanish"
+# → hola mundo
+```
+
+PDFs pass through with their figures. Scanned pages route to a vision model.
+Audio is sent natively where the model hears it, transcribed locally otherwise.
+Video is sent whole where the wire supports it, or split into frames plus a
+transcript. Each conversion is noted per row.
+
+It composes with `grep`, `jq`, `sort`, and `tail -f`: `stdin` to `stdout`,
+one item at a time.
+
+## Verbs
+
+A **verb** is one operation on your data - `map`, `filter`, `cluster`. Each reads
+`stdin` (or named FILES) and writes `stdout`, so verbs pipe into each other and
+into ordinary Unix tools. Every verb is documented at
+[prabal-rje.github.io/smartpipe][docs].
+
+**Semantic verbs** call a model:
+
+| Verb | What it does | Feels like |
+|---|---|---|
+| [`map`][map] | transform each item - text or media - with a prompt | `sed`, but it understands |
+| [`extend`][extend] | add extracted fields; keep everything else | your record, plus columns |
+| [`filter`][filter] | keep items matching a plain-English condition | `grep`, but semantic |
+| [`embed`][embed] / [`top_k`][top_k] | vectors; rank by similarity | `sort \| head`, by meaning |
+| [`reduce`][reduce] | synthesize many items into one | `awk` END, but literate |
+| [`join`][join] | match two inputs (`--kind inner\|leftouter\|anti`); `--on` alone is free | SQL join, but semantic |
+| [`cluster`][cluster] | group by meaning, label each group | themes with sizes and quotes |
+| [`distinct`][distinct] | fold near-duplicates; `--exact` is free | `sort -u`, by meaning |
+| [`diff`][diff] | what distinguishes two sets | the post-incident answer |
+| [`outliers`][outliers] | the items least like the rest | novelty, surfaced |
+
+**Free verbs** never call a model. Run them first to cut the corpus before any
+paid stage:
+
+| Verb | What it does | Feels like |
+|---|---|---|
+| [`where`][where] | filter on exact field predicates | SQL `WHERE` |
+| [`summarize`][summarize] | count, average, percentiles, time buckets | SQL `GROUP BY` |
+| [`sort`][sort] | order items by a field | `sort` |
+| [`sample`][sample] | take a seeded random subset | `shuf` |
+| [`getschema`][getschema] | list fields, types, and coverage | `head`, for structure |
+| [`split`][split] | break items into pieces (pages, minutes) | `split` |
+| [`chart`][cli] | terminal bars, SVG, facets, time series | quick plots |
+
+Some semantic verbs have a **conditionally free mode**: `join --on` (key
+equality, no prompt), `distinct --exact` (hash-only folding), `map`/`extend`
+`--dry-run` (compose without sending), and `smartpipe schema` with a
+braces/DSL expression. Each stays at zero model calls by construction.
+
+## A one-minute tour
+
+```bash
+# 1. point smartpipe at a model (ChatGPT login, a cloud key, or local Ollama)
+smartpipe config
+
+# 2. ask one question across a folder of mixed documents
+smartpipe map "What does this say about pricing?" 'docs/*.pdf'
+
+# 3. typed extraction - braces carry names, types, and guidance
+cat tickets.jsonl \
+| smartpipe extend "Add {label enum(bug, feature, praise), urgency number: 0 to 1}"
+
+# 4. group feedback by meaning, label each theme, chart it
+cat feedback.txt \
+| smartpipe cluster --top 8 \
+| smartpipe chart cluster --save themes.svg
+
+# 5. cut for free with `where`, then let the model judge only what is left
+cat app.log \
+| smartpipe where 'text has "ERROR"' \
+| smartpipe filter "an actual outage"
+
+# 6. save a whole pipeline as a file; it runs as a command
+smartpipe run triage.sem --dry-run   # prints the stage graph and cost, makes zero calls
+
+# 7. month-end close: the vision model IS the OCR; the anti-join is the worklist
+smartpipe map "Extract {vendor string, invoice_number string, total number}" 'invoices/2026-06/*.pdf' \
+| tee june-invoices.ndjson \
+| smartpipe join "the same payment" --right ledger.jsonl --kind anti > missing-from-ledger.jsonl
+
+# 8. video RAG, no vector database: index a folder of recordings once, ask any day
+smartpipe embed 'sessions/**/*.mp4' > sessions.embeddings
+smartpipe top_k 3 --near "user gives up after the coupon fails" < sessions.embeddings
+```
+
+Numbers 7 and 8 are full recipes -
+[invoice reconciliation](docs/cookbook/invoice-reconciliation.md) and
+[video RAG](docs/cookbook/video-qa.md) - two of a dozen in the
+[cookbook](docs/cookbook/README.md).
+
+New to this? The [Learn track][quickstart] starts at zero and assumes nothing, including
+what a "model" is.
+
+## For AI agents
+
+Driving smartpipe from an agent? [SKILL.md](SKILL.md) is the operating
+manual - invocation patterns, the cost model, machine-output contracts,
+and quoting gotchas, written for programmatic use.
+
+## Where your data goes
+
+Some steps run locally no matter which chat model you pick: embeddings
+(`fastembed`) and transcription (`whisper`) are built in.
+
+For chat, [Ollama][ollama] runs models on your machine. Any cloud model sends
+that run's data to its provider - `gpt-5.4-mini`, `claude-opus-4-8`,
+`gemini-3.1-flash-lite`, `mistral-large-latest`, `openrouter/…`.
+
+API keys come from environment variables and are never stored. ChatGPT
+subscribers can run `smartpipe auth login` instead. `smartpipe usage` keeps
+local run and token totals; see [Privacy & security][privacy] for the details.
+
+## Unix behavior
+
+- **`stdout` is data, `stderr` is diagnostics.** Progress and receipts never
+  touch your pipe.
+- **Adapts to where it runs.** Readable tables at a terminal; `JSONL` when piped
+  into another command.
+- **Order-preserving.** Output order matches input order, even with parallel calls.
+- **Failure-tolerant.** One bad item is a warning, not a crash.
+
+## Learn more
+
+Full docs: **[prabal-rje.github.io/smartpipe][docs]**.
+
+- [Learn track][quickstart] - zero to first result, six short chapters
+- [Install][install] - packages and platforms
+- [Working with files & media][files] - PDFs, scans, images, audio, video
+- [CLI reference][cli] - every flag, format, and exit code
+- [Models & providers][models] - local Ollama, cloud keys, ChatGPT login
+- [Privacy & security][privacy]
+
+## How to cite
+
+If smartpipe is useful in your research, cite it (or run `smartpipe cite`):
+
+```bibtex
+@software{gupta_smartpipe_2026,
+  author = {Gupta, Prabal},
+  title = {smartpipe: semantic pipes for your terminal},
+  year = {2026},
+  version = {1.4.0rc1},
+  license = {Apache-2.0},
+  url = {https://github.com/prabal-rje/smartpipe}
+}
+```
+
+GitHub's "Cite this repository" button (from [CITATION.cff](CITATION.cff)) gives APA too.
+
+## Development
+
+Built in the open, under **Apache-2.0**. Contributor setup and the quality
+gates are in [CONTRIBUTING.md](CONTRIBUTING.md); the manual release pass
+lives in [`qa/`](qa/README.md). The CLI surface is a SemVer contract.
+
+[docs]: https://prabal-rje.github.io/smartpipe/
+[quickstart]: https://prabal-rje.github.io/smartpipe/learn/1-first-pipeline/
+[install]: https://prabal-rje.github.io/smartpipe/install/
+[files]: https://prabal-rje.github.io/smartpipe/inputs/files/
+[cli]: https://prabal-rje.github.io/smartpipe/reference/cli/
+[models]: https://prabal-rje.github.io/smartpipe/concepts/models-and-providers/
+[privacy]: https://prabal-rje.github.io/smartpipe/privacy/
+[ollama]: https://ollama.com
+[map]: https://prabal-rje.github.io/smartpipe/verbs/map/
+[extend]: https://prabal-rje.github.io/smartpipe/verbs/extend/
+[filter]: https://prabal-rje.github.io/smartpipe/verbs/filter/
+[embed]: https://prabal-rje.github.io/smartpipe/verbs/embed/
+[top_k]: https://prabal-rje.github.io/smartpipe/verbs/top-k/
+[reduce]: https://prabal-rje.github.io/smartpipe/verbs/reduce/
+[join]: https://prabal-rje.github.io/smartpipe/verbs/join/
+[cluster]: https://prabal-rje.github.io/smartpipe/verbs/cluster/
+[distinct]: https://prabal-rje.github.io/smartpipe/verbs/distinct/
+[diff]: https://prabal-rje.github.io/smartpipe/verbs/diff/
+[outliers]: https://prabal-rje.github.io/smartpipe/verbs/outliers/
+[where]: https://prabal-rje.github.io/smartpipe/verbs/where/
+[summarize]: https://prabal-rje.github.io/smartpipe/verbs/summarize/
+[sort]: https://prabal-rje.github.io/smartpipe/verbs/sort/
+[sample]: https://prabal-rje.github.io/smartpipe/verbs/sample/
+[getschema]: https://prabal-rje.github.io/smartpipe/verbs/getschema/
+[split]: https://prabal-rje.github.io/smartpipe/verbs/split/

smartpipe_cli-1.4.0rc1/README.md

@@ -0,0 +1,228 @@
+# smartpipe
+
+[![CI](https://github.com/prabal-rje/smartpipe/actions/workflows/ci.yml/badge.svg)](https://github.com/prabal-rje/smartpipe/actions/workflows/ci.yml)
+[![PyPI](https://img.shields.io/pypi/v/smartpipe-cli)](https://pypi.org/project/smartpipe-cli/)
+[![Python](https://img.shields.io/badge/python-3.11%E2%80%933.13-blue)](pyproject.toml)
+[![Docs](https://img.shields.io/badge/docs-prabal--rje.github.io%2Fsmartpipe-blue)](https://prabal-rje.github.io/smartpipe/)
+[![License: Apache 2.0](https://img.shields.io/badge/License-Apache_2.0-blue)](LICENSE)
+
+**Semantic pipes and queries for your terminal.**
+
+Run PDFs, images, audio, video, and text through Unix verbs that understand
+their input.
+
+[![smartpipe demo - click to play with sound and controls](https://github.com/prabal-rje/smartpipe/releases/download/demo-assets/smartpipe-teaser-v2.gif)](https://prabal-rje.github.io/smartpipe/demo/)
+
+*[▶ Watch the full 70-second demo](https://prabal-rje.github.io/smartpipe/demo/) - sound, controls, no download.*
+
+## Install
+
+```bash
+# zero-install trial (or: pip install smartpipe-cli)
+uvx --from smartpipe-cli smartpipe
+```
+
+## Point it at a model
+
+```bash
+# log in with a ChatGPT account - covers most people
+smartpipe auth login
+```
+
+No ChatGPT plan? Use local [Ollama][ollama] or a cloud API key - see
+[Models & providers][models].
+
+## Examples
+
+```bash
+# summarize each filing, figures included
+smartpipe map "summarize the key risk" 'filings/*.pdf'
+
+# keep only the calls that sound frustrated - audio, understood
+smartpipe filter "the caller sounds frustrated" 'calls/*.mp3'
+
+# text on stdin works the same way
+echo "hello world" \
+| smartpipe map "translate to Spanish"
+# → hola mundo
+```
+
+PDFs pass through with their figures. Scanned pages route to a vision model.
+Audio is sent natively where the model hears it, transcribed locally otherwise.
+Video is sent whole where the wire supports it, or split into frames plus a
+transcript. Each conversion is noted per row.
+
+It composes with `grep`, `jq`, `sort`, and `tail -f`: `stdin` to `stdout`,
+one item at a time.
+
+## Verbs
+
+A **verb** is one operation on your data - `map`, `filter`, `cluster`. Each reads
+`stdin` (or named FILES) and writes `stdout`, so verbs pipe into each other and
+into ordinary Unix tools. Every verb is documented at
+[prabal-rje.github.io/smartpipe][docs].
+
+**Semantic verbs** call a model:
+
+| Verb | What it does | Feels like |
+|---|---|---|
+| [`map`][map] | transform each item - text or media - with a prompt | `sed`, but it understands |
+| [`extend`][extend] | add extracted fields; keep everything else | your record, plus columns |
+| [`filter`][filter] | keep items matching a plain-English condition | `grep`, but semantic |
+| [`embed`][embed] / [`top_k`][top_k] | vectors; rank by similarity | `sort \| head`, by meaning |
+| [`reduce`][reduce] | synthesize many items into one | `awk` END, but literate |
+| [`join`][join] | match two inputs (`--kind inner\|leftouter\|anti`); `--on` alone is free | SQL join, but semantic |
+| [`cluster`][cluster] | group by meaning, label each group | themes with sizes and quotes |
+| [`distinct`][distinct] | fold near-duplicates; `--exact` is free | `sort -u`, by meaning |
+| [`diff`][diff] | what distinguishes two sets | the post-incident answer |
+| [`outliers`][outliers] | the items least like the rest | novelty, surfaced |
+
+**Free verbs** never call a model. Run them first to cut the corpus before any
+paid stage:
+
+| Verb | What it does | Feels like |
+|---|---|---|
+| [`where`][where] | filter on exact field predicates | SQL `WHERE` |
+| [`summarize`][summarize] | count, average, percentiles, time buckets | SQL `GROUP BY` |
+| [`sort`][sort] | order items by a field | `sort` |
+| [`sample`][sample] | take a seeded random subset | `shuf` |
+| [`getschema`][getschema] | list fields, types, and coverage | `head`, for structure |
+| [`split`][split] | break items into pieces (pages, minutes) | `split` |
+| [`chart`][cli] | terminal bars, SVG, facets, time series | quick plots |
+
+Some semantic verbs have a **conditionally free mode**: `join --on` (key
+equality, no prompt), `distinct --exact` (hash-only folding), `map`/`extend`
+`--dry-run` (compose without sending), and `smartpipe schema` with a
+braces/DSL expression. Each stays at zero model calls by construction.
+
+## A one-minute tour
+
+```bash
+# 1. point smartpipe at a model (ChatGPT login, a cloud key, or local Ollama)
+smartpipe config
+
+# 2. ask one question across a folder of mixed documents
+smartpipe map "What does this say about pricing?" 'docs/*.pdf'
+
+# 3. typed extraction - braces carry names, types, and guidance
+cat tickets.jsonl \
+| smartpipe extend "Add {label enum(bug, feature, praise), urgency number: 0 to 1}"
+
+# 4. group feedback by meaning, label each theme, chart it
+cat feedback.txt \
+| smartpipe cluster --top 8 \
+| smartpipe chart cluster --save themes.svg
+
+# 5. cut for free with `where`, then let the model judge only what is left
+cat app.log \
+| smartpipe where 'text has "ERROR"' \
+| smartpipe filter "an actual outage"
+
+# 6. save a whole pipeline as a file; it runs as a command
+smartpipe run triage.sem --dry-run   # prints the stage graph and cost, makes zero calls
+
+# 7. month-end close: the vision model IS the OCR; the anti-join is the worklist
+smartpipe map "Extract {vendor string, invoice_number string, total number}" 'invoices/2026-06/*.pdf' \
+| tee june-invoices.ndjson \
+| smartpipe join "the same payment" --right ledger.jsonl --kind anti > missing-from-ledger.jsonl
+
+# 8. video RAG, no vector database: index a folder of recordings once, ask any day
+smartpipe embed 'sessions/**/*.mp4' > sessions.embeddings
+smartpipe top_k 3 --near "user gives up after the coupon fails" < sessions.embeddings
+```
+
+Numbers 7 and 8 are full recipes -
+[invoice reconciliation](docs/cookbook/invoice-reconciliation.md) and
+[video RAG](docs/cookbook/video-qa.md) - two of a dozen in the
+[cookbook](docs/cookbook/README.md).
+
+New to this? The [Learn track][quickstart] starts at zero and assumes nothing, including
+what a "model" is.
+
+## For AI agents
+
+Driving smartpipe from an agent? [SKILL.md](SKILL.md) is the operating
+manual - invocation patterns, the cost model, machine-output contracts,
+and quoting gotchas, written for programmatic use.
+
+## Where your data goes
+
+Some steps run locally no matter which chat model you pick: embeddings
+(`fastembed`) and transcription (`whisper`) are built in.
+
+For chat, [Ollama][ollama] runs models on your machine. Any cloud model sends
+that run's data to its provider - `gpt-5.4-mini`, `claude-opus-4-8`,
+`gemini-3.1-flash-lite`, `mistral-large-latest`, `openrouter/…`.
+
+API keys come from environment variables and are never stored. ChatGPT
+subscribers can run `smartpipe auth login` instead. `smartpipe usage` keeps
+local run and token totals; see [Privacy & security][privacy] for the details.
+
+## Unix behavior
+
+- **`stdout` is data, `stderr` is diagnostics.** Progress and receipts never
+  touch your pipe.
+- **Adapts to where it runs.** Readable tables at a terminal; `JSONL` when piped
+  into another command.
+- **Order-preserving.** Output order matches input order, even with parallel calls.
+- **Failure-tolerant.** One bad item is a warning, not a crash.
+
+## Learn more
+
+Full docs: **[prabal-rje.github.io/smartpipe][docs]**.
+
+- [Learn track][quickstart] - zero to first result, six short chapters
+- [Install][install] - packages and platforms
+- [Working with files & media][files] - PDFs, scans, images, audio, video
+- [CLI reference][cli] - every flag, format, and exit code
+- [Models & providers][models] - local Ollama, cloud keys, ChatGPT login
+- [Privacy & security][privacy]
+
+## How to cite
+
+If smartpipe is useful in your research, cite it (or run `smartpipe cite`):
+
+```bibtex
+@software{gupta_smartpipe_2026,
+  author = {Gupta, Prabal},
+  title = {smartpipe: semantic pipes for your terminal},
+  year = {2026},
+  version = {1.4.0rc1},
+  license = {Apache-2.0},
+  url = {https://github.com/prabal-rje/smartpipe}
+}
+```
+
+GitHub's "Cite this repository" button (from [CITATION.cff](CITATION.cff)) gives APA too.
+
+## Development
+
+Built in the open, under **Apache-2.0**. Contributor setup and the quality
+gates are in [CONTRIBUTING.md](CONTRIBUTING.md); the manual release pass
+lives in [`qa/`](qa/README.md). The CLI surface is a SemVer contract.
+
+[docs]: https://prabal-rje.github.io/smartpipe/
+[quickstart]: https://prabal-rje.github.io/smartpipe/learn/1-first-pipeline/
+[install]: https://prabal-rje.github.io/smartpipe/install/
+[files]: https://prabal-rje.github.io/smartpipe/inputs/files/
+[cli]: https://prabal-rje.github.io/smartpipe/reference/cli/
+[models]: https://prabal-rje.github.io/smartpipe/concepts/models-and-providers/
+[privacy]: https://prabal-rje.github.io/smartpipe/privacy/
+[ollama]: https://ollama.com
+[map]: https://prabal-rje.github.io/smartpipe/verbs/map/
+[extend]: https://prabal-rje.github.io/smartpipe/verbs/extend/
+[filter]: https://prabal-rje.github.io/smartpipe/verbs/filter/
+[embed]: https://prabal-rje.github.io/smartpipe/verbs/embed/
+[top_k]: https://prabal-rje.github.io/smartpipe/verbs/top-k/
+[reduce]: https://prabal-rje.github.io/smartpipe/verbs/reduce/
+[join]: https://prabal-rje.github.io/smartpipe/verbs/join/
+[cluster]: https://prabal-rje.github.io/smartpipe/verbs/cluster/
+[distinct]: https://prabal-rje.github.io/smartpipe/verbs/distinct/
+[diff]: https://prabal-rje.github.io/smartpipe/verbs/diff/
+[outliers]: https://prabal-rje.github.io/smartpipe/verbs/outliers/
+[where]: https://prabal-rje.github.io/smartpipe/verbs/where/
+[summarize]: https://prabal-rje.github.io/smartpipe/verbs/summarize/
+[sort]: https://prabal-rje.github.io/smartpipe/verbs/sort/
+[sample]: https://prabal-rje.github.io/smartpipe/verbs/sample/
+[getschema]: https://prabal-rje.github.io/smartpipe/verbs/getschema/
+[split]: https://prabal-rje.github.io/smartpipe/verbs/split/

smartpipe_cli-1.4.0rc1/demo/video/README.md

@@ -0,0 +1,17 @@
+# smartpipe demo video
+
+A self-contained [Remotion](https://remotion.dev) project that renders the
+~70-second smartpipe demo (1920x1080, 30 fps): six scenes - cold open, the
+hook, multimodal, cost honesty, scale, close - all driven by one reusable
+`<Terminal>` component and the frame-math constants in `src/config.ts`
+(tune scene lengths there). Run `npm install` once, then `npm run studio`
+to preview in the browser or `npm run render` to produce
+`out/smartpipe-demo.mp4`; each scene is also registered as its own
+composition (`ColdOpen`, `Hook`, `Multimodal`, `CostHonesty`, `Scale`,
+`Close`), so `npx remotion render Hook out/hook.mp4` re-renders one scene
+independently. The music bed is `public/silicon-prism-waltz.m4a` (60 s,
+looped over the 70 s cut with a fade-out); if you ever need a
+no-strings-attached replacement, `node scripts/make-music.mjs` (ffmpeg
+required) synthesizes an original ambient track from scratch into
+`public/music.m4a` - royalty-free by construction - and you can point the
+`<Audio>` element in `src/Main.tsx` at it.

smartpipe_cli-1.4.0rc1/demo/video/src/Main.tsx

@@ -0,0 +1,55 @@
+import React from "react";
+import { AbsoluteFill, Audio, interpolate, Series, staticFile } from "remotion";
+import { SCENE, TOTAL_FRAMES } from "./config";
+import { Background } from "./components/Background";
+import { Close } from "./scenes/Close";
+import { ColdOpen } from "./scenes/ColdOpen";
+import { CostHonesty } from "./scenes/CostHonesty";
+import { Hook } from "./scenes/Hook";
+import { Multimodal } from "./scenes/Multimodal";
+import { Scale } from "./scenes/Scale";
+
+/** The full ~70s cut: one persistent background, six scenes in series. */
+export const Main: React.FC = () => {
+  return (
+    <AbsoluteFill>
+      {/* Music bed: Silicon Prism Waltz (60s, looped to cover the 70s cut).
+          Volume envelope: quick fade-in, steady bed, fade-out over the close.
+          A fully synthesized fallback lives at scripts/make-music.mjs. */}
+      <Audio
+        loop
+        src={staticFile("silicon-prism-waltz.m4a")}
+        volume={(f) =>
+          0.55 *
+          interpolate(
+            f,
+            [0, 36, TOTAL_FRAMES - 96, TOTAL_FRAMES - 8],
+            [0, 1, 1, 0],
+            { extrapolateLeft: "clamp", extrapolateRight: "clamp" },
+          )
+        }
+      />
+      <Background />
+      <Series>
+        <Series.Sequence durationInFrames={SCENE.coldOpen}>
+          <ColdOpen />
+        </Series.Sequence>
+        <Series.Sequence durationInFrames={SCENE.hook}>
+          <Hook />
+        </Series.Sequence>
+        <Series.Sequence durationInFrames={SCENE.multimodal}>
+          <Multimodal />
+        </Series.Sequence>
+        <Series.Sequence durationInFrames={SCENE.cost}>
+          <CostHonesty />
+        </Series.Sequence>
+        <Series.Sequence durationInFrames={SCENE.scale}>
+          <Scale />
+        </Series.Sequence>
+        <Series.Sequence durationInFrames={SCENE.close}>
+          <Close />
+        </Series.Sequence>
+      </Series>
+    </AbsoluteFill>
+  );
+};