videobook-engine 4.2.0 → 5.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +84 -39
- package/dist/artifacts.d.ts +5 -10
- package/dist/artifacts.d.ts.map +1 -1
- package/dist/artifacts.js +79 -142
- package/dist/artifacts.js.map +1 -1
- package/dist/books.d.ts +1 -1
- package/dist/books.d.ts.map +1 -1
- package/dist/books.js +16 -9
- package/dist/books.js.map +1 -1
- package/dist/cas.d.ts +2 -1
- package/dist/cas.d.ts.map +1 -1
- package/dist/cas.js +33 -4
- package/dist/cas.js.map +1 -1
- package/dist/catalog-gc-state.d.ts +4 -0
- package/dist/catalog-gc-state.d.ts.map +1 -0
- package/dist/catalog-gc-state.js +49 -0
- package/dist/catalog-gc-state.js.map +1 -0
- package/dist/catalog-metadata.d.ts +2 -2
- package/dist/catalog-metadata.js +3 -3
- package/dist/catalog-metadata.js.map +1 -1
- package/dist/context.d.ts +5 -5
- package/dist/context.d.ts.map +1 -1
- package/dist/context.js +17 -49
- package/dist/context.js.map +1 -1
- package/dist/domain.d.ts +35 -2
- package/dist/domain.d.ts.map +1 -1
- package/dist/domain.js +131 -161
- package/dist/domain.js.map +1 -1
- package/dist/edits.js +2 -2
- package/dist/edits.js.map +1 -1
- package/dist/engine-types.d.ts +159 -39
- package/dist/engine-types.d.ts.map +1 -1
- package/dist/engine-types.js +2 -1
- package/dist/engine-types.js.map +1 -1
- package/dist/engine.d.ts +30 -20
- package/dist/engine.d.ts.map +1 -1
- package/dist/engine.js +46 -4
- package/dist/engine.js.map +1 -1
- package/dist/files.d.ts +10 -0
- package/dist/files.d.ts.map +1 -1
- package/dist/files.js +49 -2
- package/dist/files.js.map +1 -1
- package/dist/fork.d.ts +14 -30
- package/dist/fork.d.ts.map +1 -1
- package/dist/fork.js +18 -27
- package/dist/fork.js.map +1 -1
- package/dist/generations.d.ts +9 -0
- package/dist/generations.d.ts.map +1 -0
- package/dist/generations.js +175 -0
- package/dist/generations.js.map +1 -0
- package/dist/history.d.ts +2 -1
- package/dist/history.d.ts.map +1 -1
- package/dist/history.js +30 -57
- package/dist/history.js.map +1 -1
- package/dist/index.d.ts +8 -6
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +5 -5
- package/dist/index.js.map +1 -1
- package/dist/inline-temporal-models.d.ts +43 -0
- package/dist/inline-temporal-models.d.ts.map +1 -0
- package/dist/inline-temporal-models.js +236 -0
- package/dist/inline-temporal-models.js.map +1 -0
- package/dist/isolated-models.d.ts +8 -0
- package/dist/isolated-models.d.ts.map +1 -0
- package/dist/isolated-models.js +25 -0
- package/dist/isolated-models.js.map +1 -0
- package/dist/job-queue.d.ts +1 -1
- package/dist/job-queue.d.ts.map +1 -1
- package/dist/job-queue.js +20 -6
- package/dist/job-queue.js.map +1 -1
- package/dist/media-image.d.ts +8 -0
- package/dist/media-image.d.ts.map +1 -0
- package/dist/media-image.js +41 -0
- package/dist/media-image.js.map +1 -0
- package/dist/media-process.d.ts +13 -0
- package/dist/media-process.d.ts.map +1 -0
- package/dist/media-process.js +89 -0
- package/dist/media-process.js.map +1 -0
- package/dist/media.d.ts +3 -3
- package/dist/media.d.ts.map +1 -1
- package/dist/media.js +5 -5
- package/dist/media.js.map +1 -1
- package/dist/merge-policy.d.ts +16 -36
- package/dist/merge-policy.d.ts.map +1 -1
- package/dist/merge-policy.js +13 -125
- package/dist/merge-policy.js.map +1 -1
- package/dist/metadata.d.ts +1 -0
- package/dist/metadata.d.ts.map +1 -1
- package/dist/metadata.js +13 -3
- package/dist/metadata.js.map +1 -1
- package/dist/migrate-grid-text.d.ts +17 -0
- package/dist/migrate-grid-text.d.ts.map +1 -0
- package/dist/migrate-grid-text.js +117 -0
- package/dist/migrate-grid-text.js.map +1 -0
- package/dist/migrate-grid-v22.d.ts +12 -0
- package/dist/migrate-grid-v22.d.ts.map +1 -0
- package/dist/migrate-grid-v22.js +166 -0
- package/dist/migrate-grid-v22.js.map +1 -0
- package/dist/migrate-grid-v23.d.ts +10 -0
- package/dist/migrate-grid-v23.d.ts.map +1 -0
- package/dist/migrate-grid-v23.js +31 -0
- package/dist/migrate-grid-v23.js.map +1 -0
- package/dist/migration-notebooks.d.ts +25 -0
- package/dist/migration-notebooks.d.ts.map +1 -0
- package/dist/migration-notebooks.js +131 -0
- package/dist/migration-notebooks.js.map +1 -0
- package/dist/migration-source.d.ts +6 -0
- package/dist/migration-source.d.ts.map +1 -0
- package/dist/migration-source.js +37 -0
- package/dist/migration-source.js.map +1 -0
- package/dist/migration-timeline.d.ts +36 -0
- package/dist/migration-timeline.d.ts.map +1 -0
- package/dist/migration-timeline.js +187 -0
- package/dist/migration-timeline.js.map +1 -0
- package/dist/migration-validation.d.ts +12 -0
- package/dist/migration-validation.d.ts.map +1 -0
- package/dist/migration-validation.js +68 -0
- package/dist/migration-validation.js.map +1 -0
- package/dist/migration.d.ts.map +1 -1
- package/dist/migration.js +220 -119
- package/dist/migration.js.map +1 -1
- package/dist/model-cache-paths.d.ts +2 -0
- package/dist/model-cache-paths.d.ts.map +1 -0
- package/dist/model-cache-paths.js +5 -0
- package/dist/model-cache-paths.js.map +1 -0
- package/dist/model-checksums.json +338 -0
- package/dist/model-file-resolver.d.ts +30 -0
- package/dist/model-file-resolver.d.ts.map +1 -0
- package/dist/model-file-resolver.js +304 -0
- package/dist/model-file-resolver.js.map +1 -0
- package/dist/model-identity.d.ts +13 -0
- package/dist/model-identity.d.ts.map +1 -0
- package/dist/model-identity.js +44 -0
- package/dist/model-identity.js.map +1 -0
- package/dist/model-worker-errors.d.ts +3 -0
- package/dist/model-worker-errors.d.ts.map +1 -0
- package/dist/model-worker-errors.js +12 -0
- package/dist/model-worker-errors.js.map +1 -0
- package/dist/model-worker-pool.d.ts +32 -0
- package/dist/model-worker-pool.d.ts.map +1 -0
- package/dist/model-worker-pool.js +269 -0
- package/dist/model-worker-pool.js.map +1 -0
- package/dist/model-worker-protocol.d.ts +42 -0
- package/dist/model-worker-protocol.d.ts.map +1 -0
- package/dist/model-worker-protocol.js +2 -0
- package/dist/model-worker-protocol.js.map +1 -0
- package/dist/model-worker.d.ts +2 -0
- package/dist/model-worker.d.ts.map +1 -0
- package/dist/model-worker.js +91 -0
- package/dist/model-worker.js.map +1 -0
- package/dist/mvp-contract-fixtures.js +1 -1
- package/dist/mvp-contract-fixtures.js.map +1 -1
- package/dist/mvp-contracts.d.ts +32 -8
- package/dist/mvp-contracts.d.ts.map +1 -1
- package/dist/mvp-contracts.js +1 -1
- package/dist/mvp-contracts.js.map +1 -1
- package/dist/notebook/types.d.ts +1 -29
- package/dist/notebook/types.d.ts.map +1 -1
- package/dist/notebook-grid.d.ts +46 -0
- package/dist/notebook-grid.d.ts.map +1 -0
- package/dist/notebook-grid.js +187 -0
- package/dist/notebook-grid.js.map +1 -0
- package/dist/notebook-media-artifact.d.ts +23 -0
- package/dist/notebook-media-artifact.d.ts.map +1 -0
- package/dist/notebook-media-artifact.js +67 -0
- package/dist/notebook-media-artifact.js.map +1 -0
- package/dist/notebook-mentions.d.ts +20 -0
- package/dist/notebook-mentions.d.ts.map +1 -0
- package/dist/notebook-mentions.js +54 -0
- package/dist/notebook-mentions.js.map +1 -0
- package/dist/runtime-services.js +4 -9
- package/dist/runtime-services.js.map +1 -1
- package/dist/schema.d.ts +4 -19
- package/dist/schema.d.ts.map +1 -1
- package/dist/schema.js +66 -92
- package/dist/schema.js.map +1 -1
- package/dist/search-provider-access.d.ts +11 -0
- package/dist/search-provider-access.d.ts.map +1 -0
- package/dist/search-provider-access.js +42 -0
- package/dist/search-provider-access.js.map +1 -0
- package/dist/similarity.d.ts +60 -1
- package/dist/similarity.d.ts.map +1 -1
- package/dist/similarity.js +174 -154
- package/dist/similarity.js.map +1 -1
- package/dist/status.js +8 -10
- package/dist/status.js.map +1 -1
- package/dist/store.d.ts +16 -2
- package/dist/store.d.ts.map +1 -1
- package/dist/store.js +171 -17
- package/dist/store.js.map +1 -1
- package/dist/temporal-index-cache.d.ts +13 -0
- package/dist/temporal-index-cache.d.ts.map +1 -0
- package/dist/temporal-index-cache.js +59 -0
- package/dist/temporal-index-cache.js.map +1 -0
- package/dist/temporal-model-manifests.d.ts +17 -0
- package/dist/temporal-model-manifests.d.ts.map +1 -0
- package/dist/temporal-model-manifests.js +33 -0
- package/dist/temporal-model-manifests.js.map +1 -0
- package/dist/temporal-models.d.ts +20 -35
- package/dist/temporal-models.d.ts.map +1 -1
- package/dist/temporal-models.js +23 -257
- package/dist/temporal-models.js.map +1 -1
- package/dist/temporal-search.d.ts +6 -3
- package/dist/temporal-search.d.ts.map +1 -1
- package/dist/temporal-search.js +438 -122
- package/dist/temporal-search.js.map +1 -1
- package/dist/temporal-vector-index.d.ts +46 -0
- package/dist/temporal-vector-index.d.ts.map +1 -0
- package/dist/temporal-vector-index.js +220 -0
- package/dist/temporal-vector-index.js.map +1 -0
- package/dist/third-party/@huggingface-jinja.LICENSE +21 -0
- package/dist/third-party/@huggingface-tokenizers.LICENSE +201 -0
- package/dist/third-party/@huggingface-transformers.LICENSE +202 -0
- package/dist/third-party/onnxruntime-common.LICENSE +21 -0
- package/dist/third-party/versions.json +23 -0
- package/dist/transformers-runtime.d.ts +2 -0
- package/dist/transformers-runtime.d.ts.map +1 -0
- package/dist/transformers-runtime.js +33014 -0
- package/dist/transformers-runtime.js.LEGAL.txt +12 -0
- package/dist/transformers-runtime.js.map +7 -0
- package/docs/doltlite-staging.md +69 -0
- package/docs/edit-performance.md +83 -0
- package/docs/engine-layout.md +56 -50
- package/docs/media-limits.md +57 -0
- package/docs/model-download-policy.md +15 -0
- package/docs/model-integrity.md +72 -0
- package/docs/model-isolation.md +72 -0
- package/docs/model-revisions.md +54 -0
- package/docs/mvp-prd.md +2 -2
- package/docs/release-evidence.md +300 -0
- package/docs/search-provider-consent.md +67 -0
- package/docs/semantic-durability.md +55 -0
- package/docs/temporal-search-performance.md +260 -0
- package/docs/v4-migration.md +83 -0
- package/fixtures/v5/contract-fixtures.json +4 -4
- package/package.json +31 -13
- package/dist/artifact-names.d.ts +0 -6
- package/dist/artifact-names.d.ts.map +0 -1
- package/dist/artifact-names.js +0 -56
- package/dist/artifact-names.js.map +0 -1
- package/dist/resolver.d.ts +0 -15
- package/dist/resolver.d.ts.map +0 -1
- package/dist/resolver.js +0 -81
- package/dist/resolver.js.map +0 -1
|
@@ -0,0 +1,260 @@
|
|
|
1
|
+
# Temporal search scale measurements
|
|
2
|
+
|
|
3
|
+
Run the performance fixture through the public engine API:
|
|
4
|
+
|
|
5
|
+
```bash
|
|
6
|
+
npm run benchmark:temporal -- --output /tmp/temporal-search.json --assert
|
|
7
|
+
```
|
|
8
|
+
|
|
9
|
+
The default workload creates 1,000 synthetic video artifacts and 100,000
|
|
10
|
+
one-second moments with deterministic normalized 512-dimensional vectors.
|
|
11
|
+
Each artifact has indexed description text as well as visual embeddings.
|
|
12
|
+
The fixture uses synthetic bytes and declared stream metadata; it does not
|
|
13
|
+
run media probing or model inference and cannot measure indexing-model
|
|
14
|
+
throughput, recall, relevance, multilingual quality, or boundary accuracy.
|
|
15
|
+
Those release gates require the separate frozen, rights-cleared corpus.
|
|
16
|
+
|
|
17
|
+
Indexing uses batches of at most 30 moments by default. After every batch,
|
|
18
|
+
the harness reads the public index plan and verifies cumulative coverage
|
|
19
|
+
and its persisted next cursor. It activates the first partial batch and
|
|
20
|
+
checks that a reference query retrieves the source before indexing finishes.
|
|
21
|
+
After indexing, the engine closes and reopens the catalog and checks the
|
|
22
|
+
stored moment and indexed-artifact counts.
|
|
23
|
+
Each committed batch also calls `temporalSearch.prepare` with periodic
|
|
24
|
+
checkpointing, so native graph construction is included in indexing time.
|
|
25
|
+
|
|
26
|
+
The report records the first query for each mode, then repeats that query
|
|
27
|
+
outside the timing samples to check deterministic results. It measures
|
|
28
|
+
50 further queries per mode, varying the source across the book, and checks
|
|
29
|
+
that image and ordered-video queries retrieve their own source. Hybrid
|
|
30
|
+
queries combine indexed text matching, injected text vectors, ranking,
|
|
31
|
+
overlap collapse, and coverage reporting. No remote providers or embedding
|
|
32
|
+
models are loaded. Query timings include the complete public search call.
|
|
33
|
+
A first query is not a cold OS page-cache measurement, and the modes run
|
|
34
|
+
in image, video, hybrid order.
|
|
35
|
+
|
|
36
|
+
`peakRssBytes` is the process high-water RSS reported by Node, converted from
|
|
37
|
+
KiB, including the engine, index creation, and queries in a fresh fixture
|
|
38
|
+
run. Reusing a fixture measures query-process RSS without the original
|
|
39
|
+
seeding process. The report keeps these runs distinct and records actual
|
|
40
|
+
CPU, memory, Node version, source commit, dirty state, dimensions, raw timing
|
|
41
|
+
samples, coverage cadence, and whether the full-size repeated workload ran.
|
|
42
|
+
The PRD reference device is an M2 Pro with 16 GB RAM; measurements on another
|
|
43
|
+
device must retain that qualification.
|
|
44
|
+
|
|
45
|
+
For a short harness check:
|
|
46
|
+
|
|
47
|
+
```bash
|
|
48
|
+
npm run benchmark:temporal -- --moments 100 --artifacts 5 --reads 3
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
To compare retrieval implementations without recreating the fixture, use
|
|
52
|
+
`--retain-fixture` on the first run and `--fixture <printed-path>` on later
|
|
53
|
+
runs. Only fixtures with this harness's marker and manifest are accepted.
|
|
54
|
+
Reused fixture seeding metrics describe the original build, not the later
|
|
55
|
+
measurement. A short run intentionally fails the `fullScale` assertion.
|
|
56
|
+
|
|
57
|
+
## Initial 100,000-moment baseline
|
|
58
|
+
|
|
59
|
+
The [recorded baseline](../benchmarks/results/temporal-100k-baseline.json)
|
|
60
|
+
measured engine commit `c151c6e` on September 5, 2026, using an Apple M1 Pro
|
|
61
|
+
with 16 GB RAM and Node 24.10.0. Five warm queries per mode were sufficient
|
|
62
|
+
to establish a large performance failure; this is diagnostic evidence, not
|
|
63
|
+
the required 50-query acceptance run.
|
|
64
|
+
|
|
65
|
+
| Mode | Warm p50 | Warm p95 |
|
|
66
|
+
| --- | ---: | ---: |
|
|
67
|
+
| Prepared image | 2,475 ms | 2,515 ms |
|
|
68
|
+
| Prepared eight-second video | 14,344 ms | 14,425 ms |
|
|
69
|
+
| Hybrid text and visual | 2,800 ms | 2,851 ms |
|
|
70
|
+
|
|
71
|
+
Indexing took 138.8 seconds in 4,000 batches, each covering at most 30 seconds
|
|
72
|
+
of source time. First searchable coverage appeared after 49 ms. All 4,000
|
|
73
|
+
persisted resume cursors were verified. Open plus book/search summary took
|
|
74
|
+
963 ms, and the process peak RSS was 2.83 GiB.
|
|
75
|
+
|
|
76
|
+
This baseline did not meet the latency requirement. Its query path loaded all
|
|
77
|
+
segments and embeddings and evaluated every possible video window. Faster
|
|
78
|
+
timings alone cannot satisfy VE-NFR-012 if every query still scans the full
|
|
79
|
+
vector collection.
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
## Indexed candidate retrieval
|
|
83
|
+
|
|
84
|
+
The engine now caches segment metadata and vectors for the active generations
|
|
85
|
+
and uses a USearch HNSW index with F16 storage to select visual/audio/text
|
|
86
|
+
vector candidates. It recomputes shortlisted cosine scores from the original
|
|
87
|
+
F32 vectors before reciprocal-rank fusion. Each modality contributes up to
|
|
88
|
+
1,000 vector candidates, independently of the requested page size or cursor.
|
|
89
|
+
Pagination traverses the combined ranked candidate set, including lexical and
|
|
90
|
+
exact/near-fingerprint evidence. Restrictive filters are applied to candidate
|
|
91
|
+
eligibility; small eligible subsets are scored directly, and broader filters
|
|
92
|
+
expand the ANN shortlist until enough eligible candidates are found.
|
|
93
|
+
|
|
94
|
+
Ordered video searches use timed neighbors from every reference sample,
|
|
95
|
+
propose nearby start positions, then score bounded windows with timestamp
|
|
96
|
+
alignment and temporal coherence. At most 512 neighbor seeds produce at most
|
|
97
|
+
1,536 start positions. Each window uses at most 16 target samples; cached
|
|
98
|
+
query interpolation and prefix sums of continuity keep long-window scoring
|
|
99
|
+
bounded without treating skipped samples as gaps. Duration filters apply to
|
|
100
|
+
the resulting window. Still-image neighbors cannot displace timed candidates.
|
|
101
|
+
|
|
102
|
+
Committed indexing batches refresh only affected cache entries. Semantic
|
|
103
|
+
changes refresh metadata and remove deleted sources while preserving native
|
|
104
|
+
vectors, so a rename does not rebuild the graph. Generation changes and engine
|
|
105
|
+
close discard the relevant in-memory state. Regression tests cover native ANN
|
|
106
|
+
pagination across page sizes, vector/text replacement, source and label
|
|
107
|
+
filters, rename/deletion, ordered action, differing sampling densities, bounded
|
|
108
|
+
durations, and a long video reference surrounded by matching still images.
|
|
109
|
+
|
|
110
|
+
The [full indexed run](../benchmarks/results/temporal-100k-indexed.json) measures
|
|
111
|
+
the source committed as `db996f2` on the same M1 Pro/16 GB machine, with a newly
|
|
112
|
+
created 1,000-artifact/100,000-moment fixture and 50 warm queries per mode:
|
|
113
|
+
|
|
114
|
+
| Mode | Warm p50 | Warm p95 |
|
|
115
|
+
| --- | ---: | ---: |
|
|
116
|
+
| Prepared image | 54 ms | 63 ms |
|
|
117
|
+
| Prepared eight-second video | 105 ms | 140 ms |
|
|
118
|
+
| Hybrid text and visual | 284 ms | 331 ms |
|
|
119
|
+
|
|
120
|
+
This run passed all harness gates. Process peak RSS, including fixture
|
|
121
|
+
creation and index/query structures, was 3.26 GiB. It committed 4,000 batches
|
|
122
|
+
covering at most 30 source seconds each, verified every resume cursor, and
|
|
123
|
+
published the first searchable partial coverage after 45 ms. Open plus
|
|
124
|
+
book/search summary after closing the newly built fixture took 1.35 seconds.
|
|
125
|
+
|
|
126
|
+
Two limitations remained in that run. First, creating the native graph on the
|
|
127
|
+
first query took 139 seconds; warm-query success does not make that suitable
|
|
128
|
+
for an interactive request. `ve-s84.4` tracks preparing and persisting the
|
|
129
|
+
index through indexing work and validating fast query readiness after reopen.
|
|
130
|
+
Publication depends on that issue. Second, independent fresh-process opens
|
|
131
|
+
have spent over four seconds in automatic catalog GC; `ve-ovz.2` tracks that
|
|
132
|
+
cost, which the same-process reopen measurement does not consistently expose.
|
|
133
|
+
These measurements also do not replace the frozen corpus quality gate.
|
|
134
|
+
|
|
135
|
+
## Persisted preparation
|
|
136
|
+
|
|
137
|
+
Activate a generation after committing its first partial batch, then prepare
|
|
138
|
+
its search index inside a cancellable indexing job:
|
|
139
|
+
|
|
140
|
+
```ts
|
|
141
|
+
const activated = engine.temporalSearch.activate(manifestId, generation);
|
|
142
|
+
if (!activated.ok) throw new Error(activated.error.message);
|
|
143
|
+
const prepared = await engine.temporalSearch.prepare({
|
|
144
|
+
manifestId,
|
|
145
|
+
generation,
|
|
146
|
+
signal,
|
|
147
|
+
checkpoint: "periodic",
|
|
148
|
+
});
|
|
149
|
+
if (!prepared.ok) throw new Error(prepared.error.message);
|
|
150
|
+
```
|
|
151
|
+
|
|
152
|
+
Preparation yields between bounded native insertions. Already prepared
|
|
153
|
+
indexes update incrementally as committed batches are consumed. Periodic
|
|
154
|
+
checkpoints write after 5,000 changes; the default preparation mode and engine
|
|
155
|
+
close flush outstanding changes. Omitting the manifest and generation
|
|
156
|
+
prepares every active index, which supports background recovery of old books.
|
|
157
|
+
Collections of at most 1,000 vectors use direct scoring without a native graph.
|
|
158
|
+
|
|
159
|
+
Disposable snapshots live under `data/runtime-search`. Their identity includes
|
|
160
|
+
the full manifest, generation, modality, dimensions, native version, and index
|
|
161
|
+
configuration. A SHA-256 checksum protects the native bytes before loading;
|
|
162
|
+
stable vector keys and content digests validate the mapping against committed
|
|
163
|
+
vectors. Metadata is published by rename after a complete graph is saved.
|
|
164
|
+
A stale checkpoint can be reconciled from changed vectors; a corrupt or
|
|
165
|
+
incompatible checkpoint is rebuilt. Cancellation never publishes a partial
|
|
166
|
+
graph. The source vectors remain in the catalog.
|
|
167
|
+
|
|
168
|
+
Deleted keys are filtered from queries immediately but retain occupied native
|
|
169
|
+
slots until preparation rebuilds a fresh graph. Preparation compacts when more
|
|
170
|
+
than one quarter of native keys are inactive. This avoids a USearch 2.26.0
|
|
171
|
+
crash reproduced by saving a graph with removed slots, reopening it, and adding
|
|
172
|
+
replacement vectors. Changed vectors replace their native slots immediately;
|
|
173
|
+
a failed replacement discards the in-memory graph. Snapshot format v2 rejects
|
|
174
|
+
older, potentially unsafe graphs before loading them. Old books therefore need
|
|
175
|
+
one background preparation before large-index searches can resume.
|
|
176
|
+
|
|
177
|
+
An interactive query loads a snapshot containing all current vectors, but never
|
|
178
|
+
constructs a missing large graph. It returns `NOT_READY` with
|
|
179
|
+
`error.details.requiresIndexPreparation === true` when preparation is needed.
|
|
180
|
+
Consumers should queue preparation, show its progress, and retry after the job
|
|
181
|
+
finishes. The index's preparation result separates loaded indexes, changed
|
|
182
|
+
vectors, and persisted indexes. `--prepare-existing` in the benchmark measures
|
|
183
|
+
explicit recovery before closing and reopening a retained older fixture.
|
|
184
|
+
|
|
185
|
+
The [safe-cache regression run](../benchmarks/results/temporal-100k-safe-cache.json)
|
|
186
|
+
rebuilt the retained 100,000-vector fixture from the old format in 152.6 seconds.
|
|
187
|
+
After reopening, the first image query took 3.03 seconds; warm p95 was 72 ms for
|
|
188
|
+
image, 151 ms for video, and 371 ms for hybrid. Peak process RSS was 2.27 GiB and
|
|
189
|
+
all harness gates passed. This run measures cache rebuilding and queries; its
|
|
190
|
+
indexing fields are inherited from the retained fixture, not a new indexing run.
|
|
191
|
+
|
|
192
|
+
The [full persisted run](../benchmarks/results/temporal-100k-persisted.json)
|
|
193
|
+
measured the preparation implementation committed as `a7c3703` on the same
|
|
194
|
+
M1 Pro/16 GB device. Indexing and preparation took 291.4 seconds across 4,000
|
|
195
|
+
batches; preparation accounted for 148.3 seconds of that total. Initial
|
|
196
|
+
searchable coverage appeared after 40 ms, and every resume cursor passed.
|
|
197
|
+
After close/reopen, open plus summary took 919 ms and the first image query,
|
|
198
|
+
including verified graph loading, took 2.90 seconds. The 50-query warm p95s
|
|
199
|
+
were 62 ms (image), 162 ms (video), and 361 ms (hybrid); peak RSS was 3.58 GiB.
|
|
200
|
+
All harness gates passed. This includes incremental construction and persisted
|
|
201
|
+
reopen, but does not remove the separate fresh-process catalog GC or corpus
|
|
202
|
+
quality release gates.
|
|
203
|
+
|
|
204
|
+
## Healthy catalog reopen in a fresh process
|
|
205
|
+
|
|
206
|
+
The [fresh-process run](../benchmarks/results/temporal-100k-fresh-process.json)
|
|
207
|
+
reuses the persisted 100,000-moment fixture after recording successful catalog
|
|
208
|
+
compaction. It runs in a new Node process on the same M1 Pro/16 GB device.
|
|
209
|
+
Open plus book/search summary took 1.14 seconds, compared with 4.76 seconds
|
|
210
|
+
when the same fixture first required compaction. The first image query took
|
|
211
|
+
4.14 seconds including vector and verified graph loading. Fifty warm queries
|
|
212
|
+
per mode passed with p95s of 95 ms (image), 154 ms (video), and 378 ms (hybrid).
|
|
213
|
+
This query process peaked at 2.02 GiB and passed all harness gates; its indexing
|
|
214
|
+
metrics describe the earlier fixture creation, not new indexing work.
|
|
215
|
+
|
|
216
|
+
The store now skips size-triggered GC only when its disposable compaction
|
|
217
|
+
record matches the catalog file. Writes invalidate the record, and missing or
|
|
218
|
+
damaged metadata falls back to GC. This resolves the repeated healthy-catalog
|
|
219
|
+
compaction measured above; frozen-corpus quality remains a separate gate.
|
|
220
|
+
|
|
221
|
+
## Current-source acceptance runs
|
|
222
|
+
|
|
223
|
+
Two runs on September 6, 2026 measure the working tree on top of engine
|
|
224
|
+
`8b235f2` (the NFR measurement changes touch only benchmarks, tests and docs)
|
|
225
|
+
on the same M1 Pro/16 GB device with Node 24.10.0. The
|
|
226
|
+
[fresh build](../benchmarks/results/temporal-100k-current-source.json) created
|
|
227
|
+
a new 1,000-artifact/100,000-moment fixture: indexing plus incremental
|
|
228
|
+
preparation took 332.8 seconds across 4,000 batches of at most 30 source
|
|
229
|
+
seconds, every resume cursor passed, first searchable coverage appeared after
|
|
230
|
+
45 ms, and open plus summary in the same process took 206 ms. Fifty warm
|
|
231
|
+
queries per mode measured p50/p95 of 62/67 ms (image), 117/159 ms (video) and
|
|
232
|
+
333/391 ms (hybrid); peak RSS was 2.43 GiB. The
|
|
233
|
+
[fresh-process reopen](../benchmarks/results/temporal-100k-current-source-reopen.json)
|
|
234
|
+
of that retained fixture took 960 ms for open plus summary, with warm p50/p95
|
|
235
|
+
of 58/62 ms (image), 115/148 ms (video) and 322/359 ms (hybrid) and a 2.32 GiB
|
|
236
|
+
peak. Both runs passed every harness gate. The first image query after reopen
|
|
237
|
+
took 3.1 seconds including verified snapshot loading; the M2 Pro reference
|
|
238
|
+
device measurement remains outstanding.
|
|
239
|
+
|
|
240
|
+
## DoltLite 0.50.6 candidate
|
|
241
|
+
|
|
242
|
+
Engine `c4f1d89` (5.3.2) was measured from a clean committed tree on an Apple
|
|
243
|
+
M1 Pro with 16 GB RAM and Node 24.10.0. The
|
|
244
|
+
[fresh build](../benchmarks/results/temporal-100k-doltlite-0506.json) and
|
|
245
|
+
[fresh-process reopen](../benchmarks/results/temporal-100k-doltlite-0506-reopen.json)
|
|
246
|
+
retain all samples for 100,000 moments, 1,000 artifacts, 512 dimensions, and
|
|
247
|
+
50 warm queries per mode. Every harness gate passes.
|
|
248
|
+
|
|
249
|
+
| Run | Open + summary | Image p50 / p95 | Video p50 / p95 | Hybrid p50 / p95 | Peak RSS |
|
|
250
|
+
| --- | ---: | ---: | ---: | ---: | ---: |
|
|
251
|
+
| Fresh build, same-process reopen | 171 ms | 59 / 66 ms | 113 / 165 ms | 332 / 411 ms | 3.03 GiB |
|
|
252
|
+
| Fresh process, retained fixture | 1,714 ms | 60 / 73 ms | 117 / 179 ms | 310 / 388 ms | 2.08 GiB |
|
|
253
|
+
|
|
254
|
+
Fresh indexing and preparation take 319.9 seconds in 4,000 batches of at most
|
|
255
|
+
30 source seconds. All durable resume cursors are verified, and first searchable
|
|
256
|
+
coverage appears after 47 ms. The first query after reopen includes snapshot
|
|
257
|
+
loading: 3.03 seconds in the build process and 3.12 seconds in the fresh process;
|
|
258
|
+
these are separate from warm query latency. The reused fixture inherits indexing
|
|
259
|
+
metrics and does not measure another build. The reference M2 Pro measurement
|
|
260
|
+
remains `ve-ovz.22`; the synthetic vectors do not replace the frozen quality corpus.
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
# Copy-forward migration from schema v4
|
|
2
|
+
|
|
3
|
+
Keep the legacy book closed while migrating. `dryRunV4Migration(root)` and
|
|
4
|
+
`readV4BookIdentity(root)` inspect a disposable copy of its catalog. Migration
|
|
5
|
+
also reads a copy, leaving the original catalog and content bytes untouched.
|
|
6
|
+
The destination must be separate, including through symbolic-link aliases.
|
|
7
|
+
|
|
8
|
+
```ts
|
|
9
|
+
const preview = dryRunV4Migration(sourceRoot);
|
|
10
|
+
if (!preview.ok) throw new Error(preview.error.message);
|
|
11
|
+
|
|
12
|
+
const result = await migrateV4({
|
|
13
|
+
sourceRoot,
|
|
14
|
+
destinationRoot,
|
|
15
|
+
dryRun: false,
|
|
16
|
+
expectedSourceBookId: preview.value.sourceBookId,
|
|
17
|
+
expectedSourceHead: preview.value.sourceHeadRevision,
|
|
18
|
+
expectedMigrationKey: preview.value.migrationKey,
|
|
19
|
+
signal: controller.signal,
|
|
20
|
+
onProgress: ({ phase, completed, total }) => reportProgress(phase, completed, total),
|
|
21
|
+
});
|
|
22
|
+
```
|
|
23
|
+
|
|
24
|
+
The migration key covers the converter version, destination schema, source
|
|
25
|
+
head, and current semantic rows. It detects uncommitted changes even when IDs
|
|
26
|
+
and row counts are unchanged. The source is checked again before publication.
|
|
27
|
+
Dry runs report missing files, incorrect object sizes, invalid references,
|
|
28
|
+
unrepresentable notebook state, unsupported media, and required probing and
|
|
29
|
+
reindexing. Full object hashes and media probes are verified during migration
|
|
30
|
+
before a destination can be published. Timed conversion requires `ffprobe` on
|
|
31
|
+
PATH, or an explicit `ffprobePath`; it does not require model downloads.
|
|
32
|
+
|
|
33
|
+
Current artifact/file hashes, entities, notebook IDs, cell IDs, edge IDs, run
|
|
34
|
+
records, prompts, messages, waveforms, and compatible metadata are preserved.
|
|
35
|
+
Notebook canvas coordinates become deterministic row-major slots, ordered by
|
|
36
|
+
Y, X, and cell ID. A notebook exceeding the 64-by-8 grid is rejected. Legacy
|
|
37
|
+
scene cells become prompt cells with their entity references intact; asset
|
|
38
|
+
cells use their referenced artifact's current media type. Legacy `image` and
|
|
39
|
+
`video` cells were generation nodes; they become `generate_image` and
|
|
40
|
+
`generate_video`, with their existing output artifact IDs and edges intact.
|
|
41
|
+
Their model strings are retained as explicit tool selections. When no model
|
|
42
|
+
was configured, migration records the legacy GPT Image 2 text-to-image or
|
|
43
|
+
Wan 2.7 text-to-video default, rather than changing to the application's current
|
|
44
|
+
default. Retired or unavailable tools remain visible for user reconfiguration;
|
|
45
|
+
migration never submits generation jobs. The report includes original model
|
|
46
|
+
values, resolved tool names, and whether each choice was explicit or defaulted.
|
|
47
|
+
Compatible notebook
|
|
48
|
+
fields and execution state are restored through the engine API. Original
|
|
49
|
+
coordinates, cell types, and all legacy notebook properties remain in the
|
|
50
|
+
machine-readable report.
|
|
51
|
+
|
|
52
|
+
Image slots retain the legacy three-second duration. Video slots use probed
|
|
53
|
+
stream duration and follow legacy `(ordinal, slot_id)` order on the first video
|
|
54
|
+
track. Audio clips retain their 30-fps start and duration on the first audio
|
|
55
|
+
track, including overlaps. They play at native speed, trimming the source when
|
|
56
|
+
the requested clip is shorter; a longer placement retains the legacy trailing
|
|
57
|
+
silence. Percentage volume becomes gain in dB, zero volume becomes mute, and
|
|
58
|
+
millisecond fades become frame counts. Fades exceeding the clip duration are
|
|
59
|
+
rejected instead of being silently shortened. Render orientation sets the
|
|
60
|
+
primary sequence dimensions. Every generated placement passes edit preview
|
|
61
|
+
and commit validation.
|
|
62
|
+
|
|
63
|
+
The report records source-head and current-table digests, conversion decisions,
|
|
64
|
+
new stream profiles, and the legacy source location. Legacy action/operation
|
|
65
|
+
history remains in that source; it is not injected into current Dolt history.
|
|
66
|
+
Legacy terminal job records remain part of the source audit, while current
|
|
67
|
+
runtime job IDs start fresh. Old similarity rows are discarded. The application
|
|
68
|
+
must queue indexing under its configured current manifests after switching.
|
|
69
|
+
|
|
70
|
+
Migration builds in a uniquely named sibling temporary directory and publishes
|
|
71
|
+
by rename only after its catalog, objects, notebook state, timeline, report,
|
|
72
|
+
and audit commit have succeeded. Cancellation or an ordinary failure removes
|
|
73
|
+
that temporary directory. Process termination can leave an unpublished
|
|
74
|
+
`<destination>.migrating-*` directory; retry safely creates a fresh candidate
|
|
75
|
+
and cannot duplicate destination rows. An identical completed migration returns
|
|
76
|
+
its verified receipt. Unrelated or incompatible nonempty destinations are never
|
|
77
|
+
overwritten. Migration itself never switches the application's book root.
|
|
78
|
+
|
|
79
|
+
Tests use the pinned schema from engine commit `580fc0e`, real JPEG/H.264/AAC/WAV
|
|
80
|
+
media, an empty book, a 1,001-media-artifact catalog with 1,536 notebook cells,
|
|
81
|
+
missing/corrupt objects, cancellation, process termination, source changes,
|
|
82
|
+
and repeated completed requests. Source catalog preservation is checked byte
|
|
83
|
+
for byte as well as through its head and semantic-state digest.
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"compatibility": {
|
|
3
3
|
"contractVersion": 1,
|
|
4
|
-
"schemaVersion":
|
|
4
|
+
"schemaVersion": 24,
|
|
5
5
|
"minimumReaderContractVersion": 1,
|
|
6
6
|
"legacySchemaVersions": [
|
|
7
7
|
4,
|
|
@@ -1378,7 +1378,7 @@
|
|
|
1378
1378
|
"hits": [
|
|
1379
1379
|
{
|
|
1380
1380
|
"artifactId": "0197-artifact-video",
|
|
1381
|
-
"
|
|
1381
|
+
"artifactLabel": "Red Van",
|
|
1382
1382
|
"artifactKind": "video",
|
|
1383
1383
|
"location": {
|
|
1384
1384
|
"kind": "timed",
|
|
@@ -1481,7 +1481,7 @@
|
|
|
1481
1481
|
"migrationDryRun": {
|
|
1482
1482
|
"contractVersion": 1,
|
|
1483
1483
|
"sourceSchemaVersion": 4,
|
|
1484
|
-
"destinationSchemaVersion":
|
|
1484
|
+
"destinationSchemaVersion": 24,
|
|
1485
1485
|
"sourceBookId": "0197-book-v4",
|
|
1486
1486
|
"sourceHeadRevision": "dolt-v4-head",
|
|
1487
1487
|
"artifactCount": 8,
|
|
@@ -1503,7 +1503,7 @@
|
|
|
1503
1503
|
"migrationResult": {
|
|
1504
1504
|
"contractVersion": 1,
|
|
1505
1505
|
"sourceSchemaVersion": 4,
|
|
1506
|
-
"destinationSchemaVersion":
|
|
1506
|
+
"destinationSchemaVersion": 24,
|
|
1507
1507
|
"sourceBookId": "0197-book-v4",
|
|
1508
1508
|
"sourceHeadRevision": "dolt-v4-head",
|
|
1509
1509
|
"artifactCount": 8,
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "videobook-engine",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "5.3.2",
|
|
4
4
|
"description": "Dolt-native local-first storage engine for Videobook",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"exports": {
|
|
@@ -20,6 +20,18 @@
|
|
|
20
20
|
"import": "./dist/edit-policy.js",
|
|
21
21
|
"types": "./dist/edit-policy.d.ts"
|
|
22
22
|
},
|
|
23
|
+
"./notebook-grid": {
|
|
24
|
+
"import": "./dist/notebook-grid.js",
|
|
25
|
+
"types": "./dist/notebook-grid.d.ts"
|
|
26
|
+
},
|
|
27
|
+
"./notebook-media-artifact": {
|
|
28
|
+
"import": "./dist/notebook-media-artifact.js",
|
|
29
|
+
"types": "./dist/notebook-media-artifact.d.ts"
|
|
30
|
+
},
|
|
31
|
+
"./notebook-mentions": {
|
|
32
|
+
"import": "./dist/notebook-mentions.js",
|
|
33
|
+
"types": "./dist/notebook-mentions.d.ts"
|
|
34
|
+
},
|
|
23
35
|
"./package.json": "./package.json",
|
|
24
36
|
"./fixtures/v5": "./fixtures/v5/contract-fixtures.json"
|
|
25
37
|
},
|
|
@@ -31,29 +43,33 @@
|
|
|
31
43
|
"fixtures"
|
|
32
44
|
],
|
|
33
45
|
"scripts": {
|
|
46
|
+
"test:package": "node scripts/package-smoke.mjs",
|
|
34
47
|
"benchmark:api": "tsx benchmarks/run-api-benchmark.ts",
|
|
35
48
|
"benchmark:api:smoke": "tsx benchmarks/run-api-benchmark.ts --quick",
|
|
36
49
|
"clean": "node -e \"require('node:fs').rmSync('dist', { recursive: true, force: true })\"",
|
|
37
|
-
"build": "npm run clean && tsc",
|
|
50
|
+
"build": "npm run clean && tsc && node scripts/build-transformers.mjs",
|
|
38
51
|
"prepare": "npm run build",
|
|
39
52
|
"prepack": "npm run build",
|
|
40
53
|
"typecheck": "tsc --noEmit && tsc -p tsconfig.benchmarks.json",
|
|
41
54
|
"lint": "knip",
|
|
55
|
+
"pretest": "node scripts/patch-transformers.mjs",
|
|
42
56
|
"test": "vitest run",
|
|
43
57
|
"test:watch": "vitest",
|
|
44
58
|
"example:basic": "npm run build && tsx examples/basic-usage.ts",
|
|
45
|
-
"examples": "npm run example:basic"
|
|
59
|
+
"examples": "npm run example:basic",
|
|
60
|
+
"benchmark:temporal": "tsx benchmarks/run-temporal-benchmark.ts",
|
|
61
|
+
"benchmark:edits": "tsx benchmarks/run-edit-benchmark.ts"
|
|
46
62
|
},
|
|
47
63
|
"engines": {
|
|
48
64
|
"node": ">=22"
|
|
49
65
|
},
|
|
50
66
|
"dependencies": {
|
|
51
|
-
"@dolthub/doltlite": "
|
|
52
|
-
"@huggingface/transformers": "4.2.0",
|
|
67
|
+
"@dolthub/doltlite": "0.50.6",
|
|
53
68
|
"node-gyp": "12.1.0",
|
|
54
|
-
"sharp": "0.35.
|
|
69
|
+
"sharp": "0.35.4",
|
|
55
70
|
"usearch": "2.26.0",
|
|
56
|
-
"uuid": "^14.0.0"
|
|
71
|
+
"uuid": "^14.0.0",
|
|
72
|
+
"onnxruntime-node": "1.29.0"
|
|
57
73
|
},
|
|
58
74
|
"devDependencies": {
|
|
59
75
|
"@types/node": "^20.0.0",
|
|
@@ -61,20 +77,22 @@
|
|
|
61
77
|
"knip": "^6.11.0",
|
|
62
78
|
"tsx": "^4.19.0",
|
|
63
79
|
"typescript": "^5.5.0",
|
|
64
|
-
"vitest": "^4.1.8"
|
|
80
|
+
"vitest": "^4.1.8",
|
|
81
|
+
"@huggingface/transformers": "4.2.0",
|
|
82
|
+
"esbuild": "0.28.1"
|
|
65
83
|
},
|
|
66
84
|
"overrides": {
|
|
67
|
-
"adm-zip": "0.6.0",
|
|
68
85
|
"esbuild": "0.28.1",
|
|
69
86
|
"postcss": "^8.5.23",
|
|
70
|
-
"sharp": "
|
|
87
|
+
"sharp": "$sharp",
|
|
88
|
+
"onnxruntime-node": "$onnxruntime-node"
|
|
71
89
|
},
|
|
72
90
|
"allowScripts": {
|
|
73
|
-
"@dolthub/doltlite@0.
|
|
91
|
+
"@dolthub/doltlite@0.50.6": true,
|
|
74
92
|
"esbuild@0.28.1": true,
|
|
75
93
|
"fsevents@2.3.3": true,
|
|
76
|
-
"onnxruntime-node@1.24.3": true,
|
|
77
94
|
"protobufjs@7.6.5": true,
|
|
78
|
-
"usearch@2.26.0": true
|
|
95
|
+
"usearch@2.26.0": true,
|
|
96
|
+
"onnxruntime-node@1.29.0": true
|
|
79
97
|
}
|
|
80
98
|
}
|
package/dist/artifact-names.d.ts
DELETED
|
@@ -1,6 +0,0 @@
|
|
|
1
|
-
import type { ArtifactKind } from "./engine-types.js";
|
|
2
|
-
export declare function isArtifactNameStopWord(word: string): boolean;
|
|
3
|
-
export declare function artifactKindPrefix(kind: ArtifactKind): string;
|
|
4
|
-
export declare function artifactNameSlug(kind: ArtifactKind, name: string, maxWords?: number): string;
|
|
5
|
-
export declare function humanizeArtifactSlug(slug: string): string;
|
|
6
|
-
//# sourceMappingURL=artifact-names.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"artifact-names.d.ts","sourceRoot":"","sources":["../src/artifact-names.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAyBtD,wBAAgB,sBAAsB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAE5D;AAED,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,YAAY,GAAG,MAAM,CAE7D;AAED,wBAAgB,gBAAgB,CAC9B,IAAI,EAAE,YAAY,EAClB,IAAI,EAAE,MAAM,EACZ,QAAQ,SAAI,GACX,MAAM,CAiBR;AAED,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CASzD"}
|
package/dist/artifact-names.js
DELETED
|
@@ -1,56 +0,0 @@
|
|
|
1
|
-
const STOP_WORDS = new Set([
|
|
2
|
-
"a", "an", "the", "and", "or", "but", "in", "on", "at", "to", "for", "of",
|
|
3
|
-
"with", "by", "from", "as", "is", "was", "are", "were", "been", "be",
|
|
4
|
-
"have", "has", "had", "do", "does", "did", "will", "would", "could",
|
|
5
|
-
"should", "may", "might", "must", "shall", "can", "need", "this", "that",
|
|
6
|
-
"these", "those", "it", "its", "my", "your", "his", "her", "our", "their",
|
|
7
|
-
"what", "which", "who", "whom", "whose", "where", "when", "why", "how",
|
|
8
|
-
"all", "each", "every", "both", "few", "more", "most", "other", "some",
|
|
9
|
-
"such", "no", "nor", "not", "only", "own", "same", "so", "than", "too",
|
|
10
|
-
"very", "just", "also",
|
|
11
|
-
]);
|
|
12
|
-
const KIND_PREFIXES = {
|
|
13
|
-
video: "vid",
|
|
14
|
-
image: "img",
|
|
15
|
-
audio: "aud",
|
|
16
|
-
script: "script",
|
|
17
|
-
character: "char",
|
|
18
|
-
prompt: "prompt",
|
|
19
|
-
scene: "scene",
|
|
20
|
-
final: "final",
|
|
21
|
-
};
|
|
22
|
-
export function isArtifactNameStopWord(word) {
|
|
23
|
-
return STOP_WORDS.has(word.toLowerCase());
|
|
24
|
-
}
|
|
25
|
-
export function artifactKindPrefix(kind) {
|
|
26
|
-
return KIND_PREFIXES[kind];
|
|
27
|
-
}
|
|
28
|
-
export function artifactNameSlug(kind, name, maxWords = 4) {
|
|
29
|
-
if (kind === "final")
|
|
30
|
-
return "final";
|
|
31
|
-
const prefix = artifactKindPrefix(kind);
|
|
32
|
-
const words = name
|
|
33
|
-
.toLowerCase()
|
|
34
|
-
.replace(/[^a-z0-9\s-]/g, "")
|
|
35
|
-
.split(/[\s-]+/)
|
|
36
|
-
.filter((word) => word.length > 0
|
|
37
|
-
&& word !== prefix
|
|
38
|
-
&& !isArtifactNameStopWord(word));
|
|
39
|
-
const deduped = [];
|
|
40
|
-
for (const word of words) {
|
|
41
|
-
if (deduped.at(-1) !== word)
|
|
42
|
-
deduped.push(word);
|
|
43
|
-
}
|
|
44
|
-
const slug = deduped.slice(0, Math.max(1, maxWords)).join("-");
|
|
45
|
-
return `${prefix}-${slug || "untitled"}`;
|
|
46
|
-
}
|
|
47
|
-
export function humanizeArtifactSlug(slug) {
|
|
48
|
-
return slug
|
|
49
|
-
.replace(/^[a-z]+-/, "")
|
|
50
|
-
.split("-")
|
|
51
|
-
.map((word) => word.length === 0
|
|
52
|
-
? word
|
|
53
|
-
: word.charAt(0).toUpperCase() + word.slice(1))
|
|
54
|
-
.join(" ");
|
|
55
|
-
}
|
|
56
|
-
//# sourceMappingURL=artifact-names.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"artifact-names.js","sourceRoot":"","sources":["../src/artifact-names.ts"],"names":[],"mappings":"AAEA,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC;IACzB,GAAG,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI;IACzE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI;IACpE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO;IACnE,QAAQ,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM;IACxE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO;IACzE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK;IACtE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM;IACtE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK;IACtE,MAAM,EAAE,MAAM,EAAE,MAAM;CACvB,CAAC,CAAC;AAEH,MAAM,aAAa,GAAG;IACpB,KAAK,EAAE,KAAK;IACZ,KAAK,EAAE,KAAK;IACZ,KAAK,EAAE,KAAK;IACZ,MAAM,EAAE,QAAQ;IAChB,SAAS,EAAE,MAAM;IACjB,MAAM,EAAE,QAAQ;IAChB,KAAK,EAAE,OAAO;IACd,KAAK,EAAE,OAAO;CACiC,CAAC;AAElD,MAAM,UAAU,sBAAsB,CAAC,IAAY;IACjD,OAAO,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;AAC5C,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,IAAkB;IACnD,OAAO,aAAa,CAAC,IAAI,CAAC,CAAC;AAC7B,CAAC;AAED,MAAM,UAAU,gBAAgB,CAC9B,IAAkB,EAClB,IAAY,EACZ,QAAQ,GAAG,CAAC;IAEZ,IAAI,IAAI,KAAK,OAAO;QAAE,OAAO,OAAO,CAAC;IACrC,MAAM,MAAM,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAC;IACxC,MAAM,KAAK,GAAG,IAAI;SACf,WAAW,EAAE;SACb,OAAO,CAAC,eAAe,EAAE,EAAE,CAAC;SAC5B,KAAK,CAAC,QAAQ,CAAC;SACf,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CACf,IAAI,CAAC,MAAM,GAAG,CAAC;WACZ,IAAI,KAAK,MAAM;WACf,CAAC,sBAAsB,CAAC,IAAI,CAAC,CAAC,CAAC;IACtC,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,IAAI;YAAE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAClD,CAAC;IACD,MAAM,IAAI,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC/D,OAAO,GAAG,MAAM,IAAI,IAAI,IAAI,UAAU,EAAE,CAAC;AAC3C,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,IAAY;IAC/C,OAAO,IAAI;SACR,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC;SACvB,KAAK,CAAC,GAAG,CAAC;SACV,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CACZ,IAAI,CAAC,MAAM,KAAK,CAAC;QACf,CAAC,CAAC,IAAI;QACN,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;SAClD,IAAI,CAAC,GAAG,CAAC,CAAC;AACf,CAAC"}
|
package/dist/resolver.d.ts
DELETED
|
@@ -1,15 +0,0 @@
|
|
|
1
|
-
import type { EngineError, ResolvedArtifact, Result } from "./engine-types.js";
|
|
2
|
-
import { EngineContext } from "./context.js";
|
|
3
|
-
export declare function createResolverApi(context: EngineContext): {
|
|
4
|
-
parseTags: typeof parseArtifactTags;
|
|
5
|
-
resolveAll: (text: string) => Promise<Result<{
|
|
6
|
-
resolved: ResolvedArtifact[];
|
|
7
|
-
unresolved: string[];
|
|
8
|
-
}, EngineError>>;
|
|
9
|
-
expandSlotRefs: typeof expandSlotRefs;
|
|
10
|
-
};
|
|
11
|
-
export declare function parseArtifactTags(text: string): string[];
|
|
12
|
-
export declare function expandSlotRefs(text: string, slots: Array<{
|
|
13
|
-
slug: string;
|
|
14
|
-
}>): string;
|
|
15
|
-
//# sourceMappingURL=resolver.d.ts.map
|
package/dist/resolver.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"resolver.d.ts","sourceRoot":"","sources":["../src/resolver.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EACV,WAAW,EACX,gBAAgB,EAChB,MAAM,EACP,MAAM,mBAAmB,CAAC;AAE3B,OAAO,EAAE,aAAa,EAAY,MAAM,cAAc,CAAC;AAOvD,wBAAgB,iBAAiB,CAAC,OAAO,EAAE,aAAa;;uBAI5C,MAAM,KACX,OAAO,CACR,MAAM,CACJ;QAAE,QAAQ,EAAE,gBAAgB,EAAE,CAAC;QAAC,UAAU,EAAE,MAAM,EAAE,CAAA;KAAE,EACtD,WAAW,CACZ,CACF;;EAGJ;AAED,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,CAMxD;AAED,wBAAgB,cAAc,CAC5B,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,KAAK,CAAC;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,CAAC,GAC7B,MAAM,CAKR"}
|
package/dist/resolver.js
DELETED
|
@@ -1,81 +0,0 @@
|
|
|
1
|
-
import * as path from "node:path";
|
|
2
|
-
import { ok } from "./engine-types.js";
|
|
3
|
-
import { resultOf } from "./context.js";
|
|
4
|
-
import { createFilesApi } from "./files.js";
|
|
5
|
-
import { findPrimaryMediaFile } from "./media.js";
|
|
6
|
-
const TAG_PATTERN = /@([a-zA-Z0-9]+-[a-zA-Z0-9_-]+)/g;
|
|
7
|
-
const SLOT_PATTERN = /@s(\d{2})/g;
|
|
8
|
-
export function createResolverApi(context) {
|
|
9
|
-
return {
|
|
10
|
-
parseTags: parseArtifactTags,
|
|
11
|
-
resolveAll: (text) => resolveAllArtifacts(context, text),
|
|
12
|
-
expandSlotRefs,
|
|
13
|
-
};
|
|
14
|
-
}
|
|
15
|
-
export function parseArtifactTags(text) {
|
|
16
|
-
const tags = [];
|
|
17
|
-
for (const match of text.matchAll(TAG_PATTERN)) {
|
|
18
|
-
if (match[1])
|
|
19
|
-
tags.push(match[1]);
|
|
20
|
-
}
|
|
21
|
-
return [...new Set(tags)];
|
|
22
|
-
}
|
|
23
|
-
export function expandSlotRefs(text, slots) {
|
|
24
|
-
return text.replace(SLOT_PATTERN, (match, digits) => {
|
|
25
|
-
const slot = slots[Number.parseInt(digits, 10) - 1];
|
|
26
|
-
return slot?.slug ? `@${slot.slug}` : match;
|
|
27
|
-
});
|
|
28
|
-
}
|
|
29
|
-
async function resolveAllArtifacts(context, text) {
|
|
30
|
-
return resultOf(async () => {
|
|
31
|
-
const tags = parseArtifactTags(text);
|
|
32
|
-
if (tags.length === 0) {
|
|
33
|
-
return ok({ resolved: [], unresolved: [] });
|
|
34
|
-
}
|
|
35
|
-
const rows = context.store.db
|
|
36
|
-
.prepare(`SELECT artifact_id, slug, kind, created_at
|
|
37
|
-
FROM artifacts`)
|
|
38
|
-
.all();
|
|
39
|
-
const bySlug = new Map(rows.map((row) => [row.slug, row]));
|
|
40
|
-
const files = createFilesApi(context);
|
|
41
|
-
const resolved = [];
|
|
42
|
-
const unresolved = [];
|
|
43
|
-
for (const tag of tags) {
|
|
44
|
-
const candidates = [
|
|
45
|
-
tag,
|
|
46
|
-
`img-${tag}`,
|
|
47
|
-
`vid-${tag}`,
|
|
48
|
-
`aud-${tag}`,
|
|
49
|
-
`script-${tag}`,
|
|
50
|
-
`char-${tag}`,
|
|
51
|
-
`prompt-${tag}`,
|
|
52
|
-
`scene-${tag}`,
|
|
53
|
-
];
|
|
54
|
-
const row = candidates
|
|
55
|
-
.map((candidate) => bySlug.get(candidate))
|
|
56
|
-
.find((candidate) => candidate !== undefined);
|
|
57
|
-
if (!row) {
|
|
58
|
-
unresolved.push(tag);
|
|
59
|
-
continue;
|
|
60
|
-
}
|
|
61
|
-
const manifest = await files.manifest(row.artifact_id);
|
|
62
|
-
if (!manifest.ok)
|
|
63
|
-
return manifest;
|
|
64
|
-
const primary = findPrimaryMediaFile(manifest.value.files, row.slug) ??
|
|
65
|
-
manifest.value.files[0] ??
|
|
66
|
-
null;
|
|
67
|
-
resolved.push({
|
|
68
|
-
tag: `@${tag}`,
|
|
69
|
-
artifactId: row.artifact_id,
|
|
70
|
-
artifactSlug: row.slug,
|
|
71
|
-
artifactType: row.kind,
|
|
72
|
-
filePath: primary
|
|
73
|
-
? path.join(manifest.value.path, primary.name)
|
|
74
|
-
: null,
|
|
75
|
-
workspacePath: manifest.value.path,
|
|
76
|
-
});
|
|
77
|
-
}
|
|
78
|
-
return ok({ resolved, unresolved });
|
|
79
|
-
});
|
|
80
|
-
}
|
|
81
|
-
//# sourceMappingURL=resolver.js.map
|
package/dist/resolver.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"resolver.js","sourceRoot":"","sources":["../src/resolver.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAOlC,OAAO,EAAE,EAAE,EAAE,MAAM,mBAAmB,CAAC;AACvC,OAAO,EAAiB,QAAQ,EAAE,MAAM,cAAc,CAAC;AACvD,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAC5C,OAAO,EAAE,oBAAoB,EAAE,MAAM,YAAY,CAAC;AAElD,MAAM,WAAW,GAAG,iCAAiC,CAAC;AACtD,MAAM,YAAY,GAAG,YAAY,CAAC;AAElC,MAAM,UAAU,iBAAiB,CAAC,OAAsB;IACtD,OAAO;QACL,SAAS,EAAE,iBAAiB;QAC5B,UAAU,EAAE,CACV,IAAY,EAMZ,EAAE,CAAC,mBAAmB,CAAC,OAAO,EAAE,IAAI,CAAC;QACvC,cAAc;KACf,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,IAAY;IAC5C,MAAM,IAAI,GAAa,EAAE,CAAC;IAC1B,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;QAC/C,IAAI,KAAK,CAAC,CAAC,CAAC;YAAE,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IACpC,CAAC;IACD,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;AAC5B,CAAC;AAED,MAAM,UAAU,cAAc,CAC5B,IAAY,EACZ,KAA8B;IAE9B,OAAO,IAAI,CAAC,OAAO,CAAC,YAAY,EAAE,CAAC,KAAK,EAAE,MAAc,EAAE,EAAE;QAC1D,MAAM,IAAI,GAAG,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;QACpD,OAAO,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;IAC9C,CAAC,CAAC,CAAC;AACL,CAAC;AAED,KAAK,UAAU,mBAAmB,CAChC,OAAsB,EACtB,IAAY;IAOZ,OAAO,QAAQ,CAAC,KAAK,IAAI,EAAE;QACzB,MAAM,IAAI,GAAG,iBAAiB,CAAC,IAAI,CAAC,CAAC;QACrC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACtB,OAAO,EAAE,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,UAAU,EAAE,EAAE,EAAE,CAAC,CAAC;QAC9C,CAAC;QACD,MAAM,IAAI,GAAG,OAAO,CAAC,KAAK,CAAC,EAAE;aAC1B,OAAO,CACN;wBACgB,CACjB;aACA,GAAG,EAKJ,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;QAC3D,MAAM,KAAK,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;QACtC,MAAM,QAAQ,GAAuB,EAAE,CAAC;QACxC,MAAM,UAAU,GAAa,EAAE,CAAC;QAEhC,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,MAAM,UAAU,GAAG;gBACjB,GAAG;gBACH,OAAO,GAAG,EAAE;gBACZ,OAAO,GAAG,EAAE;gBACZ,OAAO,GAAG,EAAE;gBACZ,UAAU,GAAG,EAAE;gBACf,QAAQ,GAAG,EAAE;gBACb,UAAU,GAAG,EAAE;gBACf,SAAS,GAAG,EAAE;aACf,CAAC;YACF,MAAM,GAAG,GAAG,UAAU;iBACnB,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;iBACzC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC;YAChD,IAAI,CAAC,GAAG,EAAE,CAAC;gBACT,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACrB,SAAS;YACX,CAAC;YACD,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;YACvD,IAAI,CAAC,QAAQ,CAAC,EAAE;gBAAE,OAAO,QAAQ,CAAC;YAClC,MAAM,OAAO,GACX,oBAAoB,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,IAAI,CAAC;gBACpD,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;gBACvB,IAAI,CAAC;YACP,QAAQ,CAAC,IAAI,CAAC;gBACZ,GAAG,EAAE,IAAI,GAAG,EAAE;gBACd,UAAU,EAAE,GAAG,CAAC,WAAW;gBAC3B,YAAY,EAAE,GAAG,CAAC,IAAI;gBACtB,YAAY,EAAE,GAAG,CAAC,IAAI;gBACtB,QAAQ,EAAE,OAAO;oBACf,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,EAAE,OAAO,CAAC,IAAI,CAAC;oBAC9C,CAAC,CAAC,IAAI;gBACR,aAAa,EAAE,QAAQ,CAAC,KAAK,CAAC,IAAI;aACnC,CAAC,CAAC;QACL,CAAC;QACD,OAAO,EAAE,CAAC,EAAE,QAAQ,EAAE,UAAU,EAAE,CAAC,CAAC;IACtC,CAAC,CAAC,CAAC;AACL,CAAC"}
|