lacing 0.0.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. lacing-0.0.2/.claude/settings.json +12 -0
  2. lacing-0.0.2/.claude/skills/lacing-adapter-authoring/SKILL.md +210 -0
  3. lacing-0.0.2/.claude/skills/lacing-architecture/SKILL.md +97 -0
  4. lacing-0.0.2/.claude/skills/lacing-schema-codegen/SKILL.md +183 -0
  5. lacing-0.0.2/.claude/skills/lacing-time-and-intervals/SKILL.md +139 -0
  6. lacing-0.0.2/.gitattributes +1 -0
  7. lacing-0.0.2/.github/workflows/ci.yml +209 -0
  8. lacing-0.0.2/.gitignore +117 -0
  9. lacing-0.0.2/CLAUDE.md +35 -0
  10. lacing-0.0.2/LICENSE +21 -0
  11. lacing-0.0.2/PKG-INFO +240 -0
  12. lacing-0.0.2/README.md +207 -0
  13. lacing-0.0.2/lacing/__init__.py +72 -0
  14. lacing-0.0.2/lacing/adapters/__init__.py +133 -0
  15. lacing-0.0.2/lacing/adapters/annot.py +175 -0
  16. lacing-0.0.2/lacing/adapters/eaf.py +428 -0
  17. lacing-0.0.2/lacing/adapters/textgrid.py +336 -0
  18. lacing-0.0.2/lacing/adapters/web_annotation.py +429 -0
  19. lacing-0.0.2/lacing/adapters/webvtt.py +312 -0
  20. lacing-0.0.2/lacing/allen.py +220 -0
  21. lacing-0.0.2/lacing/cli.py +253 -0
  22. lacing-0.0.2/lacing/model.py +131 -0
  23. lacing-0.0.2/lacing/quality.py +239 -0
  24. lacing-0.0.2/lacing/store/__init__.py +38 -0
  25. lacing-0.0.2/lacing/store/base.py +131 -0
  26. lacing-0.0.2/lacing/store/memory.py +221 -0
  27. lacing-0.0.2/lacing/store/postgres.py +754 -0
  28. lacing-0.0.2/lacing/store/sqlite.py +710 -0
  29. lacing-0.0.2/lacing/tier.py +227 -0
  30. lacing-0.0.2/lacing/time.py +294 -0
  31. lacing-0.0.2/misc/docs/Annotation systems - formats, algorithms, architectures, and tooling.md +386 -0
  32. lacing-0.0.2/misc/docs/Backend Architecture for Time-Interval Annotation Systems.md +633 -0
  33. lacing-0.0.2/misc/docs/Frontend UI for Multitrack Time-Interval Annotation Editors.md +557 -0
  34. lacing-0.0.2/misc/docs/Lacing Development Roadmap.md +278 -0
  35. lacing-0.0.2/misc/docs/Open-Source Codebase Deep-Dive for Timeline : Multitrack Annotation Editors- What to Build On and What to Steal From.md +730 -0
  36. lacing-0.0.2/pyproject.toml +174 -0
  37. lacing-0.0.2/tests/__init__.py +0 -0
  38. lacing-0.0.2/tests/conftest.py +45 -0
  39. lacing-0.0.2/tests/test_adapter_annot.py +186 -0
  40. lacing-0.0.2/tests/test_adapter_eaf.py +306 -0
  41. lacing-0.0.2/tests/test_adapter_textgrid.py +255 -0
  42. lacing-0.0.2/tests/test_adapter_web_annotation.py +337 -0
  43. lacing-0.0.2/tests/test_adapter_webvtt.py +239 -0
  44. lacing-0.0.2/tests/test_allen.py +166 -0
  45. lacing-0.0.2/tests/test_cli.py +229 -0
  46. lacing-0.0.2/tests/test_model.py +175 -0
  47. lacing-0.0.2/tests/test_quality.py +203 -0
  48. lacing-0.0.2/tests/test_store_memory.py +231 -0
  49. lacing-0.0.2/tests/test_store_postgres.py +502 -0
  50. lacing-0.0.2/tests/test_store_sqlite.py +420 -0
  51. lacing-0.0.2/tests/test_time.py +179 -0
@@ -0,0 +1,12 @@
1
+ {
2
+ "permissions": {
3
+ "allow": [
4
+ "Edit(/.claude/skills/lacing-architecture/**)",
5
+ "Edit(/.claude/skills/lacing-time-and-intervals/**)",
6
+ "Edit(/.claude/skills/lacing-adapter-authoring/**)"
7
+ ],
8
+ "additionalDirectories": [
9
+ "/Users/thorwhalen/Dropbox/py/proj/t/lacing/.claude/skills"
10
+ ]
11
+ }
12
+ }
@@ -0,0 +1,210 @@
1
+ ---
2
+ name: lacing-adapter-authoring
3
+ description: Use when adding, modifying, or reviewing an I/O format adapter for lacing — Praat TextGrid, ELAN EAF, WebVTT, JAMS, Label Studio JSON, W3C Web Annotation, OpenTimelineIO, CoNLL, brat standoff, SubRip, TTML, CSV, or any new format. Triggers on "add a … adapter", "import/export … format", "support … in lacing", "round-trip … format", or any work under lacing/adapters/. Encodes the plugin contract, the round-trip test pattern, license-checking the underlying parser, schema URI conventions, and the common pitfalls (offset invalidation, rate mismatch, lossy round-trips).
4
+ ---
5
+
6
+ # Lacing — Adapter Authoring
7
+
8
+ Format support in lacing is **plugin-only**. The core never imports a format
9
+ module. Each adapter is a small, independent file in `lacing/adapters/` that
10
+ registers itself.
11
+
12
+ ## The plugin contract
13
+
14
+ Every adapter exposes two functions and one registration call. The exact
15
+ signature is fixed by [lacing/adapters/__init__.py](../../../lacing/adapters/__init__.py):
16
+
17
+ ```python
18
+ # lacing/adapters/<format>.py
19
+ import os
20
+ from lacing.adapters import register_adapter
21
+ from lacing.store import IntervalAnnotationStore, MemoryStore
22
+
23
+ ADAPTER_NAME = "textgrid"
24
+ BODY_SCHEMA_URI = "annot://schema/textgrid-label/v1"
25
+
26
+ def load(
27
+ source: str | bytes | os.PathLike,
28
+ *,
29
+ rate: int = DEFAULT_RATE,
30
+ asset_id: str = "...",
31
+ attribution: str = "anonymous",
32
+ **kwargs,
33
+ ) -> IntervalAnnotationStore:
34
+ """Parse `source` into an in-memory store."""
35
+ ...
36
+
37
+ def dump(
38
+ store: IntervalAnnotationStore,
39
+ target: str | os.PathLike | None = None,
40
+ **kwargs,
41
+ ) -> bytes | None:
42
+ """Serialize `store`. If `target` is None, return bytes; else write to target."""
43
+ ...
44
+
45
+ register_adapter(
46
+ name=ADAPTER_NAME,
47
+ load=load,
48
+ dump=dump,
49
+ extensions=(".TextGrid",), # tuple, lowercased on register
50
+ media_types=("text/x-praat-textgrid",), # tuple
51
+ body_schema_uris=(BODY_SCHEMA_URI,), # tuple — adapters may emit multiple
52
+ description="...",
53
+ )
54
+ ```
55
+
56
+ Notes on the surface:
57
+ - `register_adapter` is keyword-only and returns the `AdapterSpec`.
58
+ - `body_schema_uris` is **plural / tuple** — adapters may produce multiple
59
+ body shapes (e.g. cue + chapter schemas).
60
+ - `extensions` may be passed with or without leading dots; they're
61
+ normalized to lowercase with leading dot in the registry.
62
+ - Use `register_adapter` (the registry, not direct imports) so users can
63
+ swap or add formats without touching core.
64
+
65
+ ## Round-trip is the acceptance test
66
+
67
+ Every adapter ships with a round-trip test:
68
+
69
+ ```python
70
+ # tests/adapters/test_<format>.py
71
+ def test_roundtrip_<format>(sample_file):
72
+ store1 = load(sample_file)
73
+ blob = dump(store1)
74
+ store2 = load(blob)
75
+ assert_stores_equivalent(store1, store2)
76
+ ```
77
+
78
+ `assert_stores_equivalent` compares **annotations modulo provenance
79
+ timestamps and IDs**, not byte-equality. Some formats are lossy by design
80
+ (WebVTT loses tier metadata). Document each lossy edge in the adapter
81
+ docstring with what gets dropped.
82
+
83
+ ## Body schema URI
84
+
85
+ Every adapter declares one or more `body_schema_uri` values it produces.
86
+
87
+ - Format: `annot://schema/<name>/v<major>`.
88
+ - Bumps are **additive-only** by default. Breaking change → new major + a
89
+ registered migration in `lacing/schema.py`.
90
+ - The URI travels with each annotation; the validator picks the right Zod /
91
+ Pydantic schema by URI.
92
+
93
+ ## License-check the parser before you depend on it
94
+
95
+ Before adding a parser library, check ANN-DOC §E or look up the package on
96
+ PyPI. **Never bring in:**
97
+
98
+ | Banned | Why | Use instead |
99
+ |--------|-----|-------------|
100
+ | `praat-parselmouth` | GPLv3+ | `praatio` (MIT) |
101
+ | `aeneas` | AGPL v3 | Montreal Forced Aligner (MIT) or write a thin parser |
102
+ | `portion` | LGPLv3 | `intervaltree` (Apache-2.0) |
103
+ | Peaks.js | LGPL-3.0 | wavesurfer.js v7 (BSD-3) |
104
+ | anything LGPL/GPL/AGPL/BSL | viral / commercial trap | find MIT/BSD/Apache equivalent or write a small parser |
105
+
106
+ If the only library is non-MIT/BSD/Apache, **write the parser** — most
107
+ annotation formats are XML/JSON/CSV with simple grammars (TextGrid is ~100
108
+ lines, EAF is XML).
109
+
110
+ ## Time discipline at the parsing boundary
111
+
112
+ Parsers receive floats from external formats. Convert immediately:
113
+
114
+ ```python
115
+ def _to_rational(seconds: float | str, rate: int = 24000) -> RationalTime:
116
+ f = Fraction(seconds).limit_denominator(rate)
117
+ val = int(f * rate)
118
+ return RationalTime(value=val, rate=rate)
119
+ ```
120
+
121
+ - Convert **once at the parse boundary**; everything internal is `RationalTime`.
122
+ - If the source format guarantees integer ticks (MIDI PPQN, ELAN ms), use them directly — don't go through float.
123
+ - Document the rate assumption in the adapter docstring.
124
+
125
+ ## Tier mapping
126
+
127
+ Every format maps to lacing's ELAN tier stereotypes (Rule 6 in
128
+ `lacing-architecture`). When the source format has no equivalent
129
+ (WebVTT has flat tracks, no parent), pick `NONE` and document it.
130
+
131
+ | Format | Stereotype map |
132
+ |--------|----------------|
133
+ | Praat TextGrid IntervalTier | `NONE` |
134
+ | Praat TextGrid PointTier | `NONE` (point annotations) |
135
+ | ELAN EAF parent tier | `NONE` |
136
+ | ELAN EAF child with `Time_Subdivision` | `TIME_SUBDIVISION` |
137
+ | ELAN EAF `Symbolic_Subdivision` | `SYMBOLIC_SUBDIVISION` |
138
+ | ELAN EAF `Symbolic_Association` | `SYMBOLIC_ASSOCIATION` |
139
+ | ELAN EAF `Included_In` | `INCLUDED_IN` |
140
+ | WebVTT cues | `NONE` |
141
+ | W3C Web Annotation | `NONE` (use `motivation` for tier semantics) |
142
+
143
+ ## Provenance on import
144
+
145
+ Every imported annotation gets:
146
+
147
+ ```python
148
+ Provenance(
149
+ was_generated_by=f"adapter:{adapter_name}",
150
+ was_attributed_to=kwargs.get("attribution", "anonymous"),
151
+ was_derived_from=[source_asset_id], # content hash of the source file
152
+ generated_at_time=RationalTime.now(),
153
+ activity="import",
154
+ )
155
+ ```
156
+
157
+ If the source format encodes its own provenance (W3C Web Annotation has
158
+ `creator` + `created`), preserve it under `was_derived_from` chain — don't
159
+ overwrite.
160
+
161
+ ## Common pitfalls (catch in review)
162
+
163
+ 1. **Off-by-one on closed-vs-open intervals.** Praat is closed, lacing is half-open. Document the conversion.
164
+ 2. **Encoding assumptions.** TextGrid can be Latin-1 or UTF-8 with BOM. Always sniff.
165
+ 3. **Rate mismatch within a single file.** ELAN's `TIME_ORDER` defines named anchors with one rate; don't assume project rate.
166
+ 4. **String labels with embedded delimiters.** WebVTT cues with `-->` in the body. Use the format's escape rules, not naive split.
167
+ 5. **Empty intervals.** Some formats forbid `start == end`; lacing allows it. Document drop / convert.
168
+ 6. **Lossy round-trip not flagged.** If the format can't represent confidence/provenance, the adapter must say so in its docstring **and** in the dump return metadata.
169
+
170
+ ## Adapter checklist
171
+
172
+ - [ ] `load` and `dump` signatures match the contract.
173
+ - [ ] `register_adapter` called at module import.
174
+ - [ ] License of any underlying parser is MIT/BSD/Apache.
175
+ - [ ] All time conversions go through `_to_rational` at the boundary.
176
+ - [ ] Tier stereotype mapping documented.
177
+ - [ ] Provenance set with `was_generated_by="adapter:<name>"`.
178
+ - [ ] Round-trip test on at least two real-world samples.
179
+ - [ ] Lossy fields documented in module docstring.
180
+ - [ ] `body_schema_uri` declared and registered.
181
+ - [ ] No core imports leaked into the adapter (it depends on `lacing.model`, not the other way).
182
+
183
+ ## Adapter priority order (Phase 0 → Phase 1)
184
+
185
+ Phase 0: **TextGrid, WebVTT, W3C Web Annotation JSON-LD** — *all three
186
+ landed*. Three formats covering three audiences (linguistics, captions,
187
+ web/scholarly).
188
+
189
+ Phase 1: **ELAN EAF, JAMS, Label Studio JSON, OpenTimelineIO, CoNLL,
190
+ brat standoff, SubRip, TTML, CSV.**
191
+
192
+ Don't skip ahead — Phase 0 adapters validate the data model. If a Phase 0
193
+ adapter exposes a model gap, fix the model before adding more adapters.
194
+
195
+ ## Examples to study
196
+
197
+ The Phase 0/1 adapters are deliberately diverse and demonstrate the pattern:
198
+
199
+ - [lacing/adapters/textgrid.py](../../../lacing/adapters/textgrid.py) — uses an external parser (`praatio`, MIT, optional install), maps both interval and point tiers, raises `ImportError` with the install hint when the extra is missing.
200
+ - [lacing/adapters/webvtt.py](../../../lacing/adapters/webvtt.py) — pure-Python parser, no dependency. Flat cues, no tier hierarchy. Demonstrates the `from_string-or-from-path` source heuristic.
201
+ - [lacing/adapters/web_annotation.py](../../../lacing/adapters/web_annotation.py) — JSON-LD, uses the standard `json` module. Demonstrates discriminated-union round-tripping and creator/provenance preservation.
202
+ - [lacing/adapters/annot.py](../../../lacing/adapters/annot.py) — *lossless* SQLite-based portable file format. Demonstrates the `persistent=True` mode (returns a live `SqliteStore` instead of a `MemoryStore`) and the fast-copy path when the source is already a `SqliteStore`.
203
+ - [lacing/adapters/eaf.py](../../../lacing/adapters/eaf.py) — ELAN EAF via `pympi-ling` (MIT, optional install). The first adapter that exercises the **tier hierarchy** end-to-end: maps EAF's `CONSTRAINTS` strings to lacing's four ELAN stereotypes verbatim, topo-sorts tiers on dump so parents are emitted before children, and pulls the first `MEDIA_DESCRIPTOR/MEDIA_URL` as the default `MediaRef.asset_id`.
204
+
205
+ ## Source pointers
206
+
207
+ - Format catalogue and parser libraries: ANN-DOC §B, §E.
208
+ - Adapter pattern as architecture: ANN-DOC §C ("non-negotiable"); OSS-DOC OTIO `SchemaDef`.
209
+ - Provenance schema: ANN-DOC §C; BACK-DOC §4.5.
210
+ - Round-trip test pattern: BACK-DOC §4.3.
@@ -0,0 +1,97 @@
1
+ ---
2
+ name: lacing-architecture
3
+ description: Use when starting any non-trivial work on the lacing package — adding modules, designing APIs, choosing dependencies, planning a new feature, or making architectural decisions. Triggers on tasks like "implement X in lacing", "design the Y component", "add support for Z format", "where should this code go", or any edit under lacing/, lacing-server/, or lacing-ui/. Loads the ten non-negotiables, the package decomposition, and pointers to the design docs.
4
+ ---
5
+
6
+ # Lacing Architecture Primer
7
+
8
+ `lacing` is a **standoff, interval-keyed annotation system** with a Python
9
+ backend and a TypeScript/React frontend sharing one schema-versioned model.
10
+
11
+ ## Always read first
12
+
13
+ 1. **[misc/docs/Lacing Development Roadmap.md](../../../misc/docs/Lacing%20Development%20Roadmap.md)** — phased plan, cross-referenced to design docs.
14
+ 2. The four design docs in [misc/docs/](../../../misc/docs/):
15
+ - **ANN-DOC** — `Annotation systems - formats, algorithms, architectures, and tooling.md`
16
+ - **BACK-DOC** — `Backend Architecture for Time-Interval Annotation Systems.md`
17
+ - **FRONT-DOC** — `Frontend UI for Multitrack Time-Interval Annotation Editors.md`
18
+ - **OSS-DOC** — `Open-Source Codebase Deep-Dive ...md`
19
+
20
+ If the roadmap and a design doc disagree, **the design doc wins** — fix the roadmap.
21
+
22
+ ## The ten non-negotiables
23
+
24
+ | # | Rule | Source |
25
+ |---|------|--------|
26
+ | 1 | **Time is `RationalTime(value: int, rate: int)`.** Never floats anywhere — wire, storage, UI. Wire as `{v, r}`; Python `fractions.Fraction`; TS `bigint` pair. | OSS-DOC OTIO; BACK-DOC §2.1 |
27
+ | 2 | **Standoff annotations only.** Source media immutable; annotations reference by `(asset_id, interval)`. | ANN-DOC §C |
28
+ | 3 | **One `Annotation` envelope, typed body.** Single shape, `body: dict` validated by `body_schema_uri` (semver). No polymorphic class hierarchy. | BACK-DOC §2.1 |
29
+ | 4 | **Indexes:** `intervaltree` in memory; PostgreSQL `tstzrange` + GiST when persistent; SQLite + R*Tree for `.annot` files. | ANN-DOC §C; BACK-DOC §3.1, §4.2 |
30
+ | 5 | **Public API is a `MutableMapping[TimeInterval, list[Annotation]]` facade** with Allen-relation methods (`intersects`, `during`, `meets`, …). Implemented as a `Protocol` (Python 3.12 forbids `Protocol` inheriting from a non-Protocol ABC); concrete backends like `MemoryStore` implement the full mapping interface structurally. | ANN-DOC §C; BACK-DOC §4.1 |
31
+ | 6 | **ELAN tier stereotypes verbatim:** `NONE`, `TIME_SUBDIVISION`, `INCLUDED_IN`, `SYMBOLIC_SUBDIVISION`, `SYMBOLIC_ASSOCIATION`. | ANN-DOC §C; OSS-DOC tier-2.4 |
32
+ | 7 | **Adapter pattern for I/O.** Core never imports a format module. Every format is a registered plugin. | ANN-DOC §C ("non-negotiable") |
33
+ | 8 | **PROV-O provenance inline on every annotation.** `was_generated_by`, `was_derived_from`, `was_attributed_to`, `generated_at_time`. AI annotations carry `agent:<model>@<hash>`. | ANN-DOC §C; BACK-DOC §4.5 |
34
+ | 9 | **Pydantic v2 → JSON Schema → Zod codegen.** One SoT, two languages. Use `datamodel-code-generator` + `json-schema-to-zod`. | BACK-DOC §6 |
35
+ | 10 | **License hygiene: MIT/BSD/Apache only.** No LGPL, GPL, AGPL, BSL. Banned: `portion`, `praat-parselmouth`, `aeneas`, Peaks.js, Etro, `@theatre/studio`, Remotion. | ANN-DOC §E; FRONT-DOC §1.8; OSS-DOC tier-3 |
36
+
37
+ ## Package layout
38
+
39
+ `lacing` (this repo) is the **core lib only** — no server, no UI, no infra.
40
+ Server and UI live in sibling repos so the data model can be adopted alone.
41
+
42
+ ```
43
+ lacing/ ← THIS REPO: core library
44
+ ├── lacing/
45
+ │ ├── time.py RationalTime, TimeInterval
46
+ │ ├── model.py Annotation, Reference, Provenance
47
+ │ ├── tier.py Tier + 5 ELAN stereotypes
48
+ │ ├── allen.py 13 Allen relations + composition
49
+ │ ├── store/
50
+ │ │ ├── base.py IntervalAnnotationStore facade
51
+ │ │ ├── memory.py intervaltree-backed (Phase 0, done)
52
+ │ │ ├── sqlite.py .annot file format + persistent backend (Phase 1, done)
53
+ │ │ └── postgres.py int8range + GiST + per-tier EXCLUDE (Phase 1, done)
54
+ │ ├── adapters/ plugin-registered I/O
55
+ │ │ ├── textgrid.py Praat (Phase 0, done)
56
+ │ │ ├── webvtt.py captions (Phase 0, done)
57
+ │ │ ├── web_annotation.py W3C JSON-LD (Phase 0, done)
58
+ │ │ ├── annot.py .annot SQLite (Phase 1, done)
59
+ │ │ └── eaf.py ELAN EAF (Phase 1, done)
60
+ │ ├── cli.py argh-based CLI (Phase 1, done)
61
+ │ ├── quality.py IAA: kappa, Krippendorff α, IoU, DER (Phase 0, done)
62
+ │ └── schema.py body_schema registry; JSON Schema export (TODO)
63
+ └── misc/docs/ design docs + roadmap
64
+
65
+ lacing-server/ ← sibling repo (FastAPI + Arq + MCP + Yjs bridge)
66
+ lacing-ui/ ← sibling repo (React + zustand + wavesurfer + dnd-timeline)
67
+ ```
68
+
69
+ ## Phase awareness
70
+
71
+ When asked to implement something, identify which phase from the roadmap:
72
+
73
+ - **Phase 0** — Core: time, model, store, Allen relations, three adapters (TextGrid, WebVTT, W3C), quality metrics. **Done.**
74
+ - **Phase 1** — Persistence (SQLite/Postgres) + more adapters + CLI. **Mostly done:** `SqliteStore` + `.annot` file format adapter, ELAN EAF adapter, **`PostgresStore` with `int8range`/GiST/per-tier EXCLUDE**, and `lacing` CLI (`convert`, `query`, `validate`, `list-formats`) are in. **Remaining:** more adapters (JAMS, Label Studio JSON, OTIO, CoNLL, brat, SubRip, TTML, CSV) and `schema.py` (body-schema registry + JSON Schema export).
75
+ - **Phase 2** — FastAPI server + Arq workers + MCP + OpenTelemetry.
76
+ - **Phase 3** — Frontend MVP (waveform + dialogue tier + viseme tier + monitor + inspector).
77
+ - **Phase 4** — Yjs awareness, then full collab; WebCodecs; tier view.
78
+ - **Phase 5** — Differentiators (full Allen API, soft labels, generator timing, MCP-native).
79
+
80
+ If a request leapfrogs phases (e.g. "let's add Yjs collab" while Phase 1 isn't done), surface that and confirm before proceeding.
81
+
82
+ ## Module sizing rules
83
+
84
+ - Helper used by ONE function → inner function.
85
+ - Helper within SAME module → `_` prefix.
86
+ - Helper used across modules → no prefix.
87
+ - Prefer functional style; OOP only for facades/orchestrators.
88
+ - Keyword-only after the 3rd argument; from the 2nd if it improves readability.
89
+ - No magic numbers — externalize as kwargs with smart defaults.
90
+
91
+ ## When in doubt
92
+
93
+ - If a decision touches **time or intervals**, also load `lacing-time-and-intervals`.
94
+ - If you're **adding/modifying a format adapter**, load `lacing-adapter-authoring`.
95
+ - If you're **changing the data model or schemas**, load `lacing-schema-codegen`.
96
+ - If you're choosing a **dependency**, run the license through the rule-10 banlist before adding.
97
+ - If you're tempted to add a custom interval CRDT — don't. BACK-DOC §4.4 explicitly says compose Yjs primitives, custom code count = zero.
@@ -0,0 +1,183 @@
1
+ ---
2
+ name: lacing-schema-codegen
3
+ description: Use when modifying lacing's data model, body schemas, or the Pydantic→JSON-Schema→Zod codegen pipeline. Triggers on edits to lacing/model.py, lacing/schema.py, lacing/tier.py, body_schema_uri, schema migrations, schema versioning, or anything in lacing-ui/packages/core/ that mirrors a Python type. Encodes the single-source-of-truth rule (Pydantic v2 is SoT), the additive-by-default versioning rule, the migration-registration pattern, and the codegen wiring (`datamodel-code-generator` + `json-schema-to-zod`).
4
+ ---
5
+
6
+ # Lacing — Schema Codegen and Versioning
7
+
8
+ The Python `Annotation` model is the **single source of truth**. JSON Schema
9
+ is generated from it. Zod schemas are generated from the JSON Schema. The
10
+ TypeScript frontend never hand-writes types that mirror Python.
11
+
12
+ ## The pipeline
13
+
14
+ ```
15
+ lacing/model.py (Pydantic v2 — single source of truth)
16
+
17
+ │ pydantic.BaseModel.model_json_schema()
18
+
19
+ lacing/schema/<name>/v<N>.json (JSON Schema artifacts, committed)
20
+
21
+ ├─► Python validation: Pydantic at runtime (server boundary)
22
+
23
+ └─► TypeScript codegen:
24
+ json-schema-to-zod → lacing-ui/packages/core/zod/<name>.ts
25
+ (Zod schema, committed)
26
+
27
+ └─► z.infer<typeof S> for TS types
28
+ ```
29
+
30
+ Why **commit the generated artifacts** in JSON Schema and Zod:
31
+ - Diffs become readable in PRs.
32
+ - Frontend can build without running Python.
33
+ - Schema migrations have clear before/after.
34
+
35
+ ## Where each piece lives
36
+
37
+ | Concern | Location |
38
+ |---------|----------|
39
+ | Annotation envelope (`Annotation`, `Reference`, `Provenance`) | `lacing/model.py` |
40
+ | Tier types + 5 ELAN stereotypes | `lacing/tier.py` |
41
+ | Body schemas (per-domain payloads — phoneme, viseme, named-entity, etc.) | `lacing/bodies/<name>.py` |
42
+ | Body schema registry | `lacing/schema.py` |
43
+ | JSON Schema artifacts (committed) | `lacing/schema/<name>/v<N>.json` |
44
+ | Zod artifacts (committed) | `lacing-ui/packages/core/zod/<name>.ts` |
45
+ | Migrations | `lacing/migrations/<name>/v<N>_to_v<N+1>.py` |
46
+
47
+ ## body_schema_uri convention
48
+
49
+ Every annotation's `body` is validated by the schema named in `body_schema_uri`:
50
+
51
+ ```
52
+ annot://schema/<name>/v<major>
53
+ ```
54
+
55
+ - `name` is `kebab-case`, matches the body file: `annot://schema/named-entity/v1` ↔ `lacing/bodies/named_entity.py`.
56
+ - Only the **major** version is in the URI. Minor/patch bumps must remain backward-compatible.
57
+ - The URI is part of every annotation's wire format. Validators look up the schema by URI.
58
+
59
+ ## Versioning: additive by default
60
+
61
+ **Allowed without a major bump:**
62
+ - Add an *optional* field with a sensible default.
63
+ - Add a value to a string-enum-like field — but only if consumers ignore unknown values gracefully (document this contract per body).
64
+ - Tighten a docstring or description.
65
+
66
+ **Requires a major bump + migration:**
67
+ - Remove or rename a field.
68
+ - Change a field's type.
69
+ - Make an optional field required.
70
+ - Tighten a constraint (regex, range) that would invalidate existing data.
71
+ - Change semantic meaning of an existing field.
72
+
73
+ ## Migration registration
74
+
75
+ Every major bump ships a migration:
76
+
77
+ ```python
78
+ # lacing/migrations/named_entity/v1_to_v2.py
79
+ from lacing.schema import register_migration
80
+
81
+ @register_migration(
82
+ schema_name="named-entity",
83
+ from_version=1,
84
+ to_version=2,
85
+ )
86
+ def upgrade(body: dict) -> dict:
87
+ """v1 used `type`; v2 renames it to `entity_type` and adds optional `confidence`."""
88
+ return {
89
+ **{k: v for k, v in body.items() if k != "type"},
90
+ "entity_type": body["type"],
91
+ }
92
+ ```
93
+
94
+ - Migrations are **forward-only** by convention. If you need a downgrade, register it explicitly as a separate migration.
95
+ - Migrations run lazily on read OR eagerly during a registered batch operation. Don't write code that assumes one or the other.
96
+ - Every migration has a unit test with a v(N) sample → v(N+1) expected output.
97
+
98
+ ## The Pydantic v2 patterns we use
99
+
100
+ ```python
101
+ from pydantic import BaseModel, Field, model_validator
102
+
103
+ class NamedEntityBody(BaseModel):
104
+ """Body for named-entity annotations.
105
+
106
+ body_schema_uri: annot://schema/named-entity/v1
107
+ """
108
+ model_config = {"frozen": True, "extra": "forbid"}
109
+
110
+ entity_type: str = Field(..., description="ENTITY type (PER, ORG, LOC, ...)")
111
+ text: str = Field(..., description="Surface form")
112
+ confidence: float | None = Field(None, ge=0.0, le=1.0)
113
+ ```
114
+
115
+ - `model_config = {"frozen": True}` — annotations are immutable; replace, don't mutate.
116
+ - `"extra": "forbid"` — unknown fields are an error, not silently accepted (catches typos and stale clients).
117
+ - Use `Field(..., description=...)` — descriptions land in JSON Schema and Zod, then in the auto-generated Inspector form (FRONT-DOC §6.3).
118
+ - Prefer `| None` over `Optional[...]` (Python 3.10+).
119
+ - Validation logic via `@model_validator(mode="after")`, never side effects.
120
+
121
+ ## Codegen invocation
122
+
123
+ The exact tooling (per BACK-DOC §6):
124
+
125
+ ```bash
126
+ # Step 1: Pydantic → JSON Schema
127
+ python -m lacing.schema.export --out lacing/schema/
128
+
129
+ # Step 2: JSON Schema → Zod
130
+ npx json-schema-to-zod -i lacing/schema/named-entity/v1.json -o lacing-ui/packages/core/zod/named-entity.ts
131
+ ```
132
+
133
+ Wire this into a single `make codegen` (or equivalent) target. Both
134
+ artifacts are committed. CI verifies they're up to date by re-running
135
+ codegen and diffing.
136
+
137
+ ## The boundary between envelope and body
138
+
139
+ ```
140
+ Annotation ← envelope: id, tier, reference, body, body_schema_uri, provenance
141
+ └── body: dict ← validated by the schema at body_schema_uri
142
+ └── (NamedEntityBody, PhonemeBody, ChordBody, ...)
143
+ ```
144
+
145
+ - The **envelope** is one model in `lacing/model.py`, single version, evolves rarely.
146
+ - **Bodies** are many small models in `lacing/bodies/`, each with its own version.
147
+ - Don't put domain fields in the envelope. Don't put generic fields in the body.
148
+ - If a field needs to be queryable across all annotations (e.g., `confidence`, `author`), it goes in the envelope or in `Provenance`. If it's domain-specific (`pitch_hz`, `speaker_id`), it goes in the body.
149
+
150
+ ## Inspector form generation
151
+
152
+ The frontend Inspector auto-generates a form from each body's Zod schema
153
+ (FRONT-DOC §6.3) using `react-hook-form` + `@hookform/resolvers/zod`.
154
+
155
+ - Field types come from JSON Schema → Zod.
156
+ - Field labels come from the Pydantic `Field(..., description=...)`.
157
+ - This is why descriptions matter — they're not just docs, they're UX strings.
158
+
159
+ ## Frontend mirrors of envelope-level types
160
+
161
+ `RationalTime`, `TimeInterval`, `Reference`, `Provenance`, the 5 tier
162
+ stereotypes — these are codegened. **Don't hand-write TS versions.** A
163
+ hand-written TS type that drifts from Python is the #1 codegen failure mode.
164
+
165
+ ## Checklist before merging a model change
166
+
167
+ - [ ] Is this additive (no version bump) or breaking (major bump + migration)?
168
+ - [ ] If breaking: migration written and tested?
169
+ - [ ] `body_schema_uri` follows `annot://schema/<name>/v<major>` exactly.
170
+ - [ ] `model_config = {"frozen": True, "extra": "forbid"}` set.
171
+ - [ ] All `Field(...)` have descriptions (these become Inspector labels).
172
+ - [ ] JSON Schema regenerated and committed.
173
+ - [ ] Zod regenerated and committed.
174
+ - [ ] If envelope changed, frontend store/selectors checked for breakage.
175
+ - [ ] No envelope ↔ body field bleed (queryable cross-cutting fields stay in envelope; domain fields stay in body).
176
+
177
+ ## Source pointers
178
+
179
+ - Pydantic v2 model definitions: BACK-DOC §2.1, §4.1.
180
+ - JSON-Schema-to-Zod codegen tooling: BACK-DOC §6 (`datamodel-code-generator` + `json-schema-to-zod`).
181
+ - Schema versioning + additive-only default: ANN-DOC §C "Schema versioning"; BACK-DOC §4.5.
182
+ - Inspector form auto-generation: FRONT-DOC §6.3 `AnnotationLayerSpec<T>`.
183
+ - Migration as a registered processor: BACK-DOC §4.5.
@@ -0,0 +1,139 @@
1
+ ---
2
+ name: lacing-time-and-intervals
3
+ description: Use when writing or reviewing any code in the lacing project that touches time, intervals, durations, rates, frame numbers, sample positions, timestamps, or interval queries. Triggers on `RationalTime`, `TimeInterval`, `fractions.Fraction`, `intervaltree`, Allen relations (overlap/during/meets/before), interval boundaries, half-open ranges, snapping, or any conversion between seconds/ticks/frames/samples. Catches the most common landmines: float drift, closed-vs-open boundaries, mixed rates, and ad-hoc overlap predicates.
4
+ ---
5
+
6
+ # Lacing — Time and Intervals
7
+
8
+ Time correctness is the #1 source of subtle bugs in annotation systems. All
9
+ four design docs converge on the same rules. Follow them or annotations
10
+ silently desync over long timelines.
11
+
12
+ ## Rule 1 — Time is rational, never float
13
+
14
+ ```python
15
+ from fractions import Fraction
16
+ from dataclasses import dataclass
17
+
18
+ @dataclass(frozen=True, slots=True)
19
+ class RationalTime:
20
+ value: int # numerator
21
+ rate: int = 24000 # denominator (ticks per second)
22
+
23
+ def to_fraction(self) -> Fraction:
24
+ return Fraction(self.value, self.rate)
25
+
26
+ def to_seconds(self) -> float:
27
+ # ONLY for display / external systems that demand float
28
+ return self.value / self.rate
29
+ ```
30
+
31
+ - **Default rate: 24000.** LCM 1008000 if you need exact representation of all common video rates simultaneously.
32
+ - **Wire format:** `{"v": int, "r": int}`. TS mirrors with `bigint`.
33
+ - Use `fractions.Fraction` for arithmetic. `float` only at the *very* edge (display, third-party libs that demand it).
34
+ - `to_seconds()` returns float **for display only.** Never round-trip through it.
35
+
36
+ **Banned patterns:**
37
+ - `time_in_seconds: float` anywhere in the model, wire, or storage layer.
38
+ - `start + duration` where any operand is a float.
39
+ - `time1 == time2` on floats — use `Fraction` equality.
40
+
41
+ ## Rule 2 — Intervals are half-open `[start, end)`
42
+
43
+ ```python
44
+ @dataclass(frozen=True, slots=True)
45
+ class TimeInterval:
46
+ start: RationalTime
47
+ end: RationalTime # exclusive
48
+
49
+ @property
50
+ def is_point(self) -> bool:
51
+ return self.start == self.end # zero-length = point annotation
52
+ ```
53
+
54
+ - `start <= end` always. `start == end` is a **valid point annotation**, not a degenerate case.
55
+ - Match OTIO's `end_time_exclusive` naming. If you ever expose `end_time_inclusive`, name it explicitly.
56
+ - Two intervals with identical bounds are **equal**, not "touching."
57
+
58
+ ## Rule 3 — Mixed rates: convert, don't compare
59
+
60
+ Two `RationalTime`s with different rates are comparable via `Fraction`, but
61
+ **adding/subtracting them across rates is a smell**. Either:
62
+ - Keep one canonical rate per project, OR
63
+ - Convert explicitly at the boundary via `t.to_rate(new_rate)` and document the rate.
64
+
65
+ The actual implementation is in `lacing.time`:
66
+ - `RationalTime.from_seconds(value, rate=...)` accepts `int | float | str | Fraction` and raises `LossyTimeConversionError` if the value can't be exactly represented at `rate`. Use **strings** for ingest from text formats (`"0.001"` is exact; `0.001` isn't).
67
+ - `RationalTime.to_rate(new_rate)` raises `LossyTimeConversionError` on loss — never rounds silently.
68
+ - `__add__` / `__sub__` operate on `Fraction` then quantize back at `self.rate`. They raise `LossyTimeConversionError` if the sum isn't exact at that rate. **This is stricter than you might expect** — adding `Fraction(1,3)` to a rate-2 `RationalTime` raises rather than rounding.
69
+
70
+ Lossy conversion always raises, never rounds.
71
+
72
+ ## Rule 4 — Allen's 13 relations are the public predicate API
73
+
74
+ Don't write ad-hoc `if a.start < b.end and b.start < a.end:` predicates.
75
+ Use the registered Allen relations (in `lacing/allen.py`):
76
+
77
+ | Relation | Symbol | Predicate |
78
+ |----------|--------|-----------|
79
+ | `before` | `<` | `a.end < b.start` |
80
+ | `after` | `>` | `a.start > b.end` |
81
+ | `meets` | `m` | `a.end == b.start` |
82
+ | `met_by` | `mi` | `a.start == b.end` |
83
+ | `overlaps` | `o` | `a.start < b.start < a.end < b.end` |
84
+ | `overlapped_by` | `oi` | (mirror) |
85
+ | `starts` | `s` | `a.start == b.start and a.end < b.end` |
86
+ | `started_by` | `si` | (mirror) |
87
+ | `during` | `d` | `a.start > b.start and a.end < b.end` |
88
+ | `contains` | `di` | (mirror) |
89
+ | `finishes` | `f` | `a.start > b.start and a.end == b.end` |
90
+ | `finished_by` | `fi` | (mirror) |
91
+ | `equals` | `=` | `a.start == b.start and a.end == b.end` |
92
+
93
+ For "any kind of overlap" use the disjunction `overlaps | overlapped_by | during | contains | starts | started_by | finishes | finished_by | equals` — exposed as `intersects(a, b)`. Don't reinvent it.
94
+
95
+ ## Rule 5 — Use the right index for the query
96
+
97
+ | Query shape | In-memory | Persistent |
98
+ |-------------|-----------|------------|
99
+ | Point query / overlap with a single interval | `intervaltree.IntervalTree` (Apache-2.0) | `PostgresStore` (`int8range` + GiST) or `SqliteStore` (R*Tree, embedded) |
100
+ | Aggregate over fixed range (count, sum) | segment tree with lazy propagation | PostgreSQL with materialized view |
101
+ | Many concurrent reads, append-mostly writes | `intervaltree` is fine | GiST |
102
+ | Bulk batch analytics | `pyranges` v1 (Rust/Polars) | Parquet/Arrow IPC export |
103
+ | Embedded / single-file | n/a | SQLite + R*Tree (`.annot` format) |
104
+ | Multi-dim (time × channel) | `rtree` (libspatialindex) | composite GiST |
105
+
106
+ **Banned:** `portion` (LGPL-3.0). Use `intervaltree` for in-memory work.
107
+
108
+ ## Rule 6 — UI and storage layers can use different units
109
+
110
+ The frontend doc allows **integer microseconds** at the UI layer for
111
+ arithmetic speed, but **only**:
112
+ - Convert at the wire boundary (`RationalTime ↔ µs`).
113
+ - Reject non-exact conversions (raise on lossy).
114
+ - Document the unit on every UI variable name (`pos_us`, `dur_us`).
115
+
116
+ The Python model never speaks microseconds. It speaks `RationalTime`.
117
+
118
+ ## Rule 7 — Snapping is rate-aware
119
+
120
+ Snap targets (playhead, in/out points, clip edges, markers, grid) are all
121
+ `RationalTime`. Snapping logic stays in `Fraction` arithmetic. **Never**
122
+ snap by rounding floats.
123
+
124
+ ## Quick checklist before commit
125
+
126
+ - [ ] No `float` in any signature/field except display layers and external library bridges.
127
+ - [ ] Every `TimeInterval` is half-open; point intervals (`start == end`) handled.
128
+ - [ ] Overlap/containment predicates go through `lacing/allen.py`, not ad-hoc.
129
+ - [ ] Any rate conversion has an explicit lossy-→-raise path.
130
+ - [ ] No `portion` import (LGPL).
131
+ - [ ] In-memory store uses `intervaltree`; persistent goes through `PostgresStore` (`int8range`/GiST) or `SqliteStore` (R*Tree).
132
+
133
+ ## Source pointers
134
+
135
+ - Concrete Pydantic models: BACK-DOC §2.1.
136
+ - Algorithm choices and complexity: ANN-DOC §C–D.
137
+ - 13 relations + ORD-Horn tractable subalgebra: ANN-DOC §A.
138
+ - UI µs convention: FRONT-DOC §3 "Time representation".
139
+ - OTIO `RationalTime` parity: OSS-DOC OTIO section.
@@ -0,0 +1 @@
1
+ *.ipynb linguist-documentation