videobook-engine 2.0.1 → 5.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +99 -68
- package/dist/artifacts.d.ts +4 -9
- package/dist/artifacts.d.ts.map +1 -1
- package/dist/artifacts.js +78 -138
- package/dist/artifacts.js.map +1 -1
- package/dist/books.d.ts +1 -1
- package/dist/books.d.ts.map +1 -1
- package/dist/books.js +18 -10
- package/dist/books.js.map +1 -1
- package/dist/cas.d.ts +27 -1
- package/dist/cas.d.ts.map +1 -1
- package/dist/cas.js +78 -6
- package/dist/cas.js.map +1 -1
- package/dist/catalog-metadata.d.ts +56 -0
- package/dist/catalog-metadata.d.ts.map +1 -0
- package/dist/catalog-metadata.js +42 -0
- package/dist/catalog-metadata.js.map +1 -0
- package/dist/communications.js +4 -2
- package/dist/communications.js.map +1 -1
- package/dist/context.d.ts +9 -6
- package/dist/context.d.ts.map +1 -1
- package/dist/context.js +42 -51
- package/dist/context.js.map +1 -1
- package/dist/domain.d.ts +39 -2
- package/dist/domain.d.ts.map +1 -1
- package/dist/domain.js +846 -64
- package/dist/domain.js.map +1 -1
- package/dist/edit-policy.d.ts +3 -0
- package/dist/edit-policy.d.ts.map +1 -0
- package/dist/edit-policy.js +6 -0
- package/dist/edit-policy.js.map +1 -0
- package/dist/edits.d.ts +10 -0
- package/dist/edits.d.ts.map +1 -0
- package/dist/edits.js +1392 -0
- package/dist/edits.js.map +1 -0
- package/dist/engine-types.d.ts +171 -86
- package/dist/engine-types.d.ts.map +1 -1
- package/dist/engine-types.js +32 -0
- package/dist/engine-types.js.map +1 -1
- package/dist/engine.d.ts +89 -26
- package/dist/engine.d.ts.map +1 -1
- package/dist/engine.js +27 -21
- package/dist/engine.js.map +1 -1
- package/dist/files.d.ts.map +1 -1
- package/dist/files.js +32 -28
- package/dist/files.js.map +1 -1
- package/dist/fork.d.ts +148 -0
- package/dist/fork.d.ts.map +1 -0
- package/dist/fork.js +602 -0
- package/dist/fork.js.map +1 -0
- package/dist/generations.d.ts +9 -0
- package/dist/generations.d.ts.map +1 -0
- package/dist/generations.js +175 -0
- package/dist/generations.js.map +1 -0
- package/dist/history.d.ts +2 -5
- package/dist/history.d.ts.map +1 -1
- package/dist/history.js +183 -640
- package/dist/history.js.map +1 -1
- package/dist/index.d.ts +21 -7
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +15 -5
- package/dist/index.js.map +1 -1
- package/dist/job-queue.d.ts +2 -0
- package/dist/job-queue.d.ts.map +1 -1
- package/dist/job-queue.js +29 -2
- package/dist/job-queue.js.map +1 -1
- package/dist/media.d.ts +3 -3
- package/dist/media.d.ts.map +1 -1
- package/dist/media.js +6 -5
- package/dist/media.js.map +1 -1
- package/dist/merge-policy.d.ts +83 -0
- package/dist/merge-policy.d.ts.map +1 -0
- package/dist/merge-policy.js +203 -0
- package/dist/merge-policy.js.map +1 -0
- package/dist/metadata.d.ts.map +1 -1
- package/dist/metadata.js +17 -23
- package/dist/metadata.js.map +1 -1
- package/dist/migration.d.ts +9 -0
- package/dist/migration.d.ts.map +1 -0
- package/dist/migration.js +564 -0
- package/dist/migration.js.map +1 -0
- package/dist/mvp-contract-fixtures.d.ts +244 -0
- package/dist/mvp-contract-fixtures.d.ts.map +1 -0
- package/dist/mvp-contract-fixtures.js +556 -0
- package/dist/mvp-contract-fixtures.js.map +1 -0
- package/dist/mvp-contracts.d.ts +761 -0
- package/dist/mvp-contracts.d.ts.map +1 -0
- package/dist/mvp-contracts.js +59 -0
- package/dist/mvp-contracts.js.map +1 -0
- package/dist/mvp-time.d.ts +54 -0
- package/dist/mvp-time.d.ts.map +1 -0
- package/dist/mvp-time.js +148 -0
- package/dist/mvp-time.js.map +1 -0
- package/dist/notebook/types.d.ts +85 -8
- package/dist/notebook/types.d.ts.map +1 -1
- package/dist/notebook-grid.d.ts +37 -0
- package/dist/notebook-grid.d.ts.map +1 -0
- package/dist/notebook-grid.js +160 -0
- package/dist/notebook-grid.js.map +1 -0
- package/dist/notebook-media-artifact.d.ts +23 -0
- package/dist/notebook-media-artifact.d.ts.map +1 -0
- package/dist/notebook-media-artifact.js +67 -0
- package/dist/notebook-media-artifact.js.map +1 -0
- package/dist/notebook-mentions.d.ts +20 -0
- package/dist/notebook-mentions.d.ts.map +1 -0
- package/dist/notebook-mentions.js +53 -0
- package/dist/notebook-mentions.js.map +1 -0
- package/dist/order-keys.d.ts +22 -0
- package/dist/order-keys.d.ts.map +1 -0
- package/dist/order-keys.js +262 -0
- package/dist/order-keys.js.map +1 -0
- package/dist/runtime-services.js +8 -11
- package/dist/runtime-services.js.map +1 -1
- package/dist/schema.d.ts +8 -5
- package/dist/schema.d.ts.map +1 -1
- package/dist/schema.js +624 -141
- package/dist/schema.js.map +1 -1
- package/dist/search-benchmark.d.ts +4 -0
- package/dist/search-benchmark.d.ts.map +1 -0
- package/dist/search-benchmark.js +144 -0
- package/dist/search-benchmark.js.map +1 -0
- package/dist/sequences.d.ts +20 -0
- package/dist/sequences.d.ts.map +1 -0
- package/dist/sequences.js +825 -0
- package/dist/sequences.js.map +1 -0
- package/dist/similarity.js +10 -10
- package/dist/similarity.js.map +1 -1
- package/dist/status.js +8 -10
- package/dist/status.js.map +1 -1
- package/dist/storage.d.ts +3 -1
- package/dist/storage.d.ts.map +1 -1
- package/dist/storage.js +388 -7
- package/dist/storage.js.map +1 -1
- package/dist/store.d.ts +64 -3
- package/dist/store.d.ts.map +1 -1
- package/dist/store.js +354 -68
- package/dist/store.js.map +1 -1
- package/dist/streams.d.ts +10 -0
- package/dist/streams.d.ts.map +1 -0
- package/dist/streams.js +183 -0
- package/dist/streams.js.map +1 -0
- package/dist/temporal-models.d.ts +48 -0
- package/dist/temporal-models.d.ts.map +1 -0
- package/dist/temporal-models.js +278 -0
- package/dist/temporal-models.js.map +1 -0
- package/dist/temporal-search.d.ts +29 -0
- package/dist/temporal-search.d.ts.map +1 -0
- package/dist/temporal-search.js +1551 -0
- package/dist/temporal-search.js.map +1 -0
- package/dist/transcripts.d.ts +16 -0
- package/dist/transcripts.d.ts.map +1 -0
- package/dist/transcripts.js +519 -0
- package/dist/transcripts.js.map +1 -0
- package/docs/engine-layout.md +345 -110
- package/docs/mvp-prd.md +918 -0
- package/fixtures/media/vancat.mp4 +0 -0
- package/fixtures/media/vancat_profile.jpg +0 -0
- package/fixtures/v5/contract-fixtures.json +1532 -0
- package/package.json +42 -5
- package/dist/history-types.d.ts +0 -74
- package/dist/history-types.d.ts.map +0 -1
- package/dist/history-types.js +0 -2
- package/dist/history-types.js.map +0 -1
- package/dist/resolver.d.ts +0 -15
- package/dist/resolver.d.ts.map +0 -1
- package/dist/resolver.js +0 -81
- package/dist/resolver.js.map +0 -1
- package/dist/timeline.d.ts +0 -9
- package/dist/timeline.d.ts.map +0 -1
- package/dist/timeline.js +0 -236
- package/dist/timeline.js.map +0 -1
|
@@ -0,0 +1,1551 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { normalizeSourcePoint, normalizeSourceRange, rationalEquals, sourceRangeEndTick, } from "./mvp-time.js";
|
|
3
|
+
import { resultOf, syncResultOf } from "./context.js";
|
|
4
|
+
import { canonicalJson, EngineFault, parseJson } from "./store.js";
|
|
5
|
+
const INDEX_PHASES = [
|
|
6
|
+
"probe",
|
|
7
|
+
"segment",
|
|
8
|
+
"transcript",
|
|
9
|
+
"ocr",
|
|
10
|
+
"visual",
|
|
11
|
+
"audio",
|
|
12
|
+
"lexical",
|
|
13
|
+
"activate",
|
|
14
|
+
];
|
|
15
|
+
export function createTemporalSearchApi(context) {
|
|
16
|
+
const providers = new Map();
|
|
17
|
+
return {
|
|
18
|
+
providers: {
|
|
19
|
+
register: (provider) => {
|
|
20
|
+
requiredText(provider.manifestId, "Provider manifest ID");
|
|
21
|
+
providers.set(provider.manifestId, provider);
|
|
22
|
+
},
|
|
23
|
+
list: () => [...providers.keys()].sort(),
|
|
24
|
+
},
|
|
25
|
+
manifests: {
|
|
26
|
+
register: (manifest) => syncResultOf(() => registerManifest(context, manifest)),
|
|
27
|
+
list: () => listManifests(context),
|
|
28
|
+
get: (manifestId) => syncResultOf(() => requiredManifest(context, manifestId)),
|
|
29
|
+
},
|
|
30
|
+
plan: (artifact, objectHash, manifestId, generation) => syncResultOf(() => indexPlan(context, artifact, objectHash, manifestId, generation)),
|
|
31
|
+
commitBatch: (input) => syncResultOf(() => commitIndexBatch(context, input)),
|
|
32
|
+
activate: (manifestId, generation) => syncResultOf(() => activateGeneration(context, manifestId, generation)),
|
|
33
|
+
coverage: (artifact) => syncResultOf(() => searchCoverage(context, artifact)),
|
|
34
|
+
query: (query) => queryTemporalIndex(context, providers, query),
|
|
35
|
+
queryPrepared: (query, reference, options = {}) => queryTemporalIndex(context, providers, query, { reference, options }),
|
|
36
|
+
invalidate: (artifact, objectHash) => syncResultOf(() => invalidateCoverage(context, artifact, objectHash)),
|
|
37
|
+
cleanup: () => syncResultOf(() => cleanupRetired(context)),
|
|
38
|
+
stats: () => temporalStats(context),
|
|
39
|
+
};
|
|
40
|
+
}
|
|
41
|
+
function registerManifest(context, manifest) {
|
|
42
|
+
validateManifest(manifest);
|
|
43
|
+
const existing = context.store.db
|
|
44
|
+
.prepare("SELECT * FROM runtime_index_manifests WHERE manifest_id=?")
|
|
45
|
+
.get(manifest.manifestId);
|
|
46
|
+
if (existing) {
|
|
47
|
+
const projected = manifestFromRow(existing);
|
|
48
|
+
if (canonicalJson(projected) !== canonicalJson(manifest)) {
|
|
49
|
+
throw new EngineFault({
|
|
50
|
+
code: "MANIFEST_INCOMPATIBLE",
|
|
51
|
+
message: `Index manifest is immutable: ${manifest.manifestId}`,
|
|
52
|
+
});
|
|
53
|
+
}
|
|
54
|
+
return projected;
|
|
55
|
+
}
|
|
56
|
+
context.store.runtime(() => {
|
|
57
|
+
context.store.db
|
|
58
|
+
.prepare(`INSERT INTO runtime_index_manifests(
|
|
59
|
+
manifest_id, provider, model_id, model_revision, license,
|
|
60
|
+
embedding_space, dimensions, modalities_json,
|
|
61
|
+
supported_languages_json, preprocessing_version,
|
|
62
|
+
extractor_version, created_at
|
|
63
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
|
|
64
|
+
.run(manifest.manifestId, manifest.provider, manifest.modelId, manifest.modelRevision, manifest.license ?? null, manifest.embeddingSpace, manifest.dimensions, canonicalJson(manifest.modalities), canonicalJson(manifest.supportedLanguages), manifest.preprocessingVersion, manifest.extractorVersion, manifest.createdAt);
|
|
65
|
+
});
|
|
66
|
+
return manifest;
|
|
67
|
+
}
|
|
68
|
+
function validateManifest(manifest) {
|
|
69
|
+
requiredText(manifest.manifestId, "Manifest ID");
|
|
70
|
+
requiredText(manifest.provider, "Manifest provider");
|
|
71
|
+
requiredText(manifest.modelId, "Manifest model ID");
|
|
72
|
+
requiredText(manifest.modelRevision, "Manifest model revision");
|
|
73
|
+
requiredText(manifest.embeddingSpace, "Manifest embedding space");
|
|
74
|
+
safeIntegerAtLeast(manifest.dimensions, 1, "Manifest dimensions");
|
|
75
|
+
if (manifest.modalities.length === 0) {
|
|
76
|
+
throw new Error("Manifest must support at least one modality");
|
|
77
|
+
}
|
|
78
|
+
requiredText(manifest.preprocessingVersion, "Preprocessing version");
|
|
79
|
+
requiredText(manifest.extractorVersion, "Extractor version");
|
|
80
|
+
safeIntegerAtLeast(manifest.createdAt, 0, "Manifest createdAt");
|
|
81
|
+
}
|
|
82
|
+
function listManifests(context) {
|
|
83
|
+
return context.store.db
|
|
84
|
+
.prepare(`SELECT * FROM runtime_index_manifests
|
|
85
|
+
ORDER BY created_at, manifest_id`)
|
|
86
|
+
.all().map(manifestFromRow);
|
|
87
|
+
}
|
|
88
|
+
function requiredManifest(context, manifestId) {
|
|
89
|
+
const row = context.store.db
|
|
90
|
+
.prepare("SELECT * FROM runtime_index_manifests WHERE manifest_id=?")
|
|
91
|
+
.get(manifestId);
|
|
92
|
+
if (!row) {
|
|
93
|
+
throw new EngineFault({
|
|
94
|
+
code: "NOT_FOUND",
|
|
95
|
+
message: `Index manifest not found: ${manifestId}`,
|
|
96
|
+
});
|
|
97
|
+
}
|
|
98
|
+
return manifestFromRow(row);
|
|
99
|
+
}
|
|
100
|
+
function manifestFromRow(row) {
|
|
101
|
+
return {
|
|
102
|
+
manifestId: row.manifest_id,
|
|
103
|
+
provider: row.provider,
|
|
104
|
+
modelId: row.model_id,
|
|
105
|
+
modelRevision: row.model_revision,
|
|
106
|
+
...(row.license ? { license: row.license } : {}),
|
|
107
|
+
embeddingSpace: row.embedding_space,
|
|
108
|
+
dimensions: row.dimensions,
|
|
109
|
+
modalities: parseJson(row.modalities_json, []),
|
|
110
|
+
supportedLanguages: parseJson(row.supported_languages_json, []),
|
|
111
|
+
preprocessingVersion: row.preprocessing_version,
|
|
112
|
+
extractorVersion: row.extractor_version,
|
|
113
|
+
createdAt: row.created_at,
|
|
114
|
+
};
|
|
115
|
+
}
|
|
116
|
+
function indexPlan(context, artifactReference, objectHash, manifestId, generation) {
|
|
117
|
+
const artifact = context.artifactRow(artifactReference);
|
|
118
|
+
assertArtifactObject(context, artifact.artifact_id, objectHash);
|
|
119
|
+
requiredManifest(context, manifestId);
|
|
120
|
+
requiredText(generation, "Index generation");
|
|
121
|
+
const coverage = coverageRows(context, "WHERE artifact_id=? AND object_hash=? AND manifest_id=? AND generation=?", [artifact.artifact_id, objectHash, manifestId, generation]);
|
|
122
|
+
const completed = new Set(coverage
|
|
123
|
+
.filter((item) => item.state === "ready")
|
|
124
|
+
.map((item) => item.phase));
|
|
125
|
+
return {
|
|
126
|
+
artifactId: artifact.artifact_id,
|
|
127
|
+
objectHash,
|
|
128
|
+
manifestId,
|
|
129
|
+
generation,
|
|
130
|
+
phases: INDEX_PHASES,
|
|
131
|
+
pendingPhases: INDEX_PHASES.filter((phase) => !completed.has(phase)),
|
|
132
|
+
coverage,
|
|
133
|
+
};
|
|
134
|
+
}
|
|
135
|
+
function commitIndexBatch(context, input) {
|
|
136
|
+
const artifact = context.artifactRow(input.artifactId);
|
|
137
|
+
const manifest = requiredManifest(context, input.manifestId);
|
|
138
|
+
if (artifact.artifact_id !== input.artifactId) {
|
|
139
|
+
throw new EngineFault({
|
|
140
|
+
code: "INVALID_INPUT",
|
|
141
|
+
message: "Index batches require a canonical artifact ID",
|
|
142
|
+
});
|
|
143
|
+
}
|
|
144
|
+
assertArtifactObject(context, artifact.artifact_id, input.objectHash);
|
|
145
|
+
requiredText(input.generation, "Index generation");
|
|
146
|
+
safeIntegerAtLeast(input.maxUnits, 1, "Index batch max units");
|
|
147
|
+
if (input.observations.length > input.maxUnits) {
|
|
148
|
+
throw new EngineFault({
|
|
149
|
+
code: "RESOURCE_EXHAUSTED",
|
|
150
|
+
message: "Index batch exceeds maxUnits",
|
|
151
|
+
});
|
|
152
|
+
}
|
|
153
|
+
const batchKey = hashJson({
|
|
154
|
+
artifactId: input.artifactId,
|
|
155
|
+
objectHash: input.objectHash,
|
|
156
|
+
manifestId: input.manifestId,
|
|
157
|
+
generation: input.generation,
|
|
158
|
+
phase: input.phase,
|
|
159
|
+
cursor: input.cursor ?? null,
|
|
160
|
+
observations: input.observations,
|
|
161
|
+
});
|
|
162
|
+
const existing = context.store.db
|
|
163
|
+
.prepare("SELECT result_json FROM runtime_index_batches WHERE batch_key=?")
|
|
164
|
+
.get(batchKey);
|
|
165
|
+
if (existing) {
|
|
166
|
+
return parseJson(existing.result_json, {
|
|
167
|
+
artifactId: input.artifactId,
|
|
168
|
+
manifestId: input.manifestId,
|
|
169
|
+
phase: input.phase,
|
|
170
|
+
committedUnits: 0,
|
|
171
|
+
coveredRanges: [],
|
|
172
|
+
complete: false,
|
|
173
|
+
generation: input.generation,
|
|
174
|
+
});
|
|
175
|
+
}
|
|
176
|
+
const streamCache = new Map();
|
|
177
|
+
const observations = input.observations.map((observation, index) => normalizeObservation(context, manifest, input, observation, index, streamCache));
|
|
178
|
+
const coveredRanges = input.coveredRanges.map(normalizeSourceRange);
|
|
179
|
+
const result = {
|
|
180
|
+
artifactId: input.artifactId,
|
|
181
|
+
manifestId: input.manifestId,
|
|
182
|
+
phase: input.phase,
|
|
183
|
+
committedUnits: observations.length,
|
|
184
|
+
coveredRanges,
|
|
185
|
+
...(input.nextCursor ? { nextCursor: input.nextCursor } : {}),
|
|
186
|
+
complete: input.complete,
|
|
187
|
+
generation: input.generation,
|
|
188
|
+
};
|
|
189
|
+
context.store.runtime((now) => {
|
|
190
|
+
ensureGeneration(context, manifest, input.generation, now);
|
|
191
|
+
const statements = prepareObservationStatements(context);
|
|
192
|
+
const existingSegments = existingSegmentIds(context, observations);
|
|
193
|
+
const writtenSegments = new Set();
|
|
194
|
+
for (const observation of observations) {
|
|
195
|
+
upsertObservation(input, observation, now, statements, existingSegments.has(observation.segmentId)
|
|
196
|
+
|| writtenSegments.has(observation.segmentId));
|
|
197
|
+
writtenSegments.add(observation.segmentId);
|
|
198
|
+
}
|
|
199
|
+
const prior = context.store.db
|
|
200
|
+
.prepare(`SELECT indexed_units FROM runtime_index_coverage
|
|
201
|
+
WHERE artifact_id=? AND object_hash=? AND manifest_id=?
|
|
202
|
+
AND generation=? AND phase=?`)
|
|
203
|
+
.get(input.artifactId, input.objectHash, input.manifestId, input.generation, input.phase);
|
|
204
|
+
context.store.db
|
|
205
|
+
.prepare(`INSERT INTO runtime_index_coverage(
|
|
206
|
+
artifact_id, object_hash, manifest_id, generation, phase,
|
|
207
|
+
state, covered_ranges_json, indexed_units, total_units,
|
|
208
|
+
retryable, error_json, cursor, updated_at
|
|
209
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 1, NULL, ?, ?)
|
|
210
|
+
ON CONFLICT(
|
|
211
|
+
artifact_id, object_hash, manifest_id, generation, phase
|
|
212
|
+
) DO UPDATE SET
|
|
213
|
+
state=excluded.state,
|
|
214
|
+
covered_ranges_json=excluded.covered_ranges_json,
|
|
215
|
+
indexed_units=excluded.indexed_units,
|
|
216
|
+
total_units=excluded.total_units,
|
|
217
|
+
retryable=1,
|
|
218
|
+
error_json=NULL,
|
|
219
|
+
cursor=excluded.cursor,
|
|
220
|
+
updated_at=excluded.updated_at`)
|
|
221
|
+
.run(input.artifactId, input.objectHash, input.manifestId, input.generation, input.phase, input.complete ? "ready" : "partial", canonicalJson(coveredRanges), (prior?.indexed_units ?? 0) + observations.length, input.totalUnits ?? null, input.nextCursor ?? null, now);
|
|
222
|
+
context.store.db
|
|
223
|
+
.prepare(`INSERT INTO runtime_index_batches(
|
|
224
|
+
batch_key, artifact_id, object_hash, manifest_id, generation,
|
|
225
|
+
phase, cursor, result_json, committed_at
|
|
226
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`)
|
|
227
|
+
.run(batchKey, input.artifactId, input.objectHash, input.manifestId, input.generation, input.phase, input.cursor ?? null, canonicalJson(result), now);
|
|
228
|
+
});
|
|
229
|
+
return result;
|
|
230
|
+
}
|
|
231
|
+
function normalizeObservation(context, manifest, batch, input, index, streamCache) {
|
|
232
|
+
if (input.artifactId !== batch.artifactId
|
|
233
|
+
|| input.objectHash !== batch.objectHash) {
|
|
234
|
+
throw new EngineFault({
|
|
235
|
+
code: "INVALID_INPUT",
|
|
236
|
+
message: "Index observation source does not match its batch",
|
|
237
|
+
});
|
|
238
|
+
}
|
|
239
|
+
const segmentId = input.segmentId
|
|
240
|
+
?? deterministicUuid(batch.generation, `${batch.artifactId}:${batch.phase}:${batch.cursor ?? "start"}:${index}`);
|
|
241
|
+
const range = input.range ? normalizeSourceRange(input.range) : undefined;
|
|
242
|
+
if (range) {
|
|
243
|
+
if (range.objectHash !== batch.objectHash
|
|
244
|
+
|| (input.streamId && range.streamId !== input.streamId)) {
|
|
245
|
+
throw new EngineFault({
|
|
246
|
+
code: "INVALID_RANGE",
|
|
247
|
+
message: "Observation range does not match its stream identity",
|
|
248
|
+
});
|
|
249
|
+
}
|
|
250
|
+
let stream = streamCache.get(range.streamId);
|
|
251
|
+
if (!streamCache.has(range.streamId)) {
|
|
252
|
+
stream = context.store.db
|
|
253
|
+
.prepare(`SELECT artifact_id, object_hash, duration_ticks,
|
|
254
|
+
time_base_numerator, time_base_denominator
|
|
255
|
+
FROM artifact_streams WHERE stream_id=?`)
|
|
256
|
+
.get(range.streamId);
|
|
257
|
+
streamCache.set(range.streamId, stream);
|
|
258
|
+
}
|
|
259
|
+
if (!stream
|
|
260
|
+
|| stream.artifact_id !== batch.artifactId
|
|
261
|
+
|| stream.object_hash !== batch.objectHash
|
|
262
|
+
|| sourceRangeEndTick(range) > stream.duration_ticks
|
|
263
|
+
|| !rationalEquals(range.timeBase, {
|
|
264
|
+
numerator: stream.time_base_numerator,
|
|
265
|
+
denominator: stream.time_base_denominator,
|
|
266
|
+
})) {
|
|
267
|
+
throw new EngineFault({
|
|
268
|
+
code: "INVALID_RANGE",
|
|
269
|
+
message: "Observation range exceeds or mismatches its source stream",
|
|
270
|
+
});
|
|
271
|
+
}
|
|
272
|
+
}
|
|
273
|
+
else if (!input.sourcePath) {
|
|
274
|
+
throw new EngineFault({
|
|
275
|
+
code: "INVALID_INPUT",
|
|
276
|
+
message: "Untimed observations require a source path",
|
|
277
|
+
});
|
|
278
|
+
}
|
|
279
|
+
requiredText(input.segmentationVersion, "Segmentation version");
|
|
280
|
+
for (const text of input.texts) {
|
|
281
|
+
requiredText(text.text, "Segment text");
|
|
282
|
+
if (text.startUtf8Byte !== undefined
|
|
283
|
+
|| text.endUtf8Byte !== undefined) {
|
|
284
|
+
safeIntegerAtLeast(text.startUtf8Byte ?? -1, 0, "Text start byte");
|
|
285
|
+
safeIntegerAtLeast(text.endUtf8Byte ?? -1, 1, "Text end byte");
|
|
286
|
+
if ((text.endUtf8Byte ?? 0) <= (text.startUtf8Byte ?? 0)) {
|
|
287
|
+
throw new EngineFault({
|
|
288
|
+
code: "INVALID_RANGE",
|
|
289
|
+
message: "Text byte range must be positive and half-open",
|
|
290
|
+
});
|
|
291
|
+
}
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
for (const embedding of input.embeddings) {
|
|
295
|
+
if (embedding.embeddingSpace !== manifest.embeddingSpace) {
|
|
296
|
+
throw new EngineFault({
|
|
297
|
+
code: "MANIFEST_INCOMPATIBLE",
|
|
298
|
+
message: "Observation embedding space does not match its manifest",
|
|
299
|
+
});
|
|
300
|
+
}
|
|
301
|
+
if (embedding.vector.length !== manifest.dimensions) {
|
|
302
|
+
throw new EngineFault({
|
|
303
|
+
code: "MANIFEST_INCOMPATIBLE",
|
|
304
|
+
message: "Observation vector dimensions do not match its manifest",
|
|
305
|
+
});
|
|
306
|
+
}
|
|
307
|
+
if (!embedding.vector.every(Number.isFinite)) {
|
|
308
|
+
throw new Error("Observation vectors must contain finite numbers");
|
|
309
|
+
}
|
|
310
|
+
}
|
|
311
|
+
return {
|
|
312
|
+
...input,
|
|
313
|
+
segmentId,
|
|
314
|
+
...(range ? { range } : {}),
|
|
315
|
+
};
|
|
316
|
+
}
|
|
317
|
+
function ensureGeneration(context, manifest, generation, now) {
|
|
318
|
+
const existing = context.store.db
|
|
319
|
+
.prepare(`SELECT manifest_id FROM runtime_index_generations
|
|
320
|
+
WHERE embedding_space=? AND generation=?`)
|
|
321
|
+
.get(manifest.embeddingSpace, generation);
|
|
322
|
+
if (existing && existing.manifest_id !== manifest.manifestId) {
|
|
323
|
+
throw new EngineFault({
|
|
324
|
+
code: "MANIFEST_INCOMPATIBLE",
|
|
325
|
+
message: "Generation is already bound to another manifest",
|
|
326
|
+
});
|
|
327
|
+
}
|
|
328
|
+
context.store.db
|
|
329
|
+
.prepare(`INSERT OR IGNORE INTO runtime_index_generations(
|
|
330
|
+
embedding_space, generation, manifest_id, state,
|
|
331
|
+
activated_at, created_at
|
|
332
|
+
) VALUES (?, ?, ?, 'building', NULL, ?)`)
|
|
333
|
+
.run(manifest.embeddingSpace, generation, manifest.manifestId, now);
|
|
334
|
+
}
|
|
335
|
+
function upsertObservation(batch, observation, now, statements, replaceChildren) {
|
|
336
|
+
statements.upsertSegment.run(observation.segmentId, observation.artifactId, observation.streamId ?? observation.range?.streamId ?? null, observation.objectHash, observation.range ? canonicalJson(observation.range) : null, observation.sourcePath ?? null, observation.kind, observation.representativeTick ?? null, observation.segmentationVersion, batch.generation, batch.manifestId, now);
|
|
337
|
+
if (replaceChildren) {
|
|
338
|
+
statements.deleteText.run(observation.segmentId);
|
|
339
|
+
statements.deleteEmbeddings.run(observation.segmentId);
|
|
340
|
+
statements.deleteFingerprints.run(observation.segmentId);
|
|
341
|
+
}
|
|
342
|
+
observation.texts.forEach((text, index) => {
|
|
343
|
+
statements.insertText.run(text.textId ?? deterministicUuid(observation.segmentId, `text:${index}`), observation.segmentId, text.kind, text.language ?? null, text.text, text.startUtf8Byte ?? null, text.endUtf8Byte ?? null, text.confidence ?? null, canonicalJson(text.provenance ?? {}), batch.generation);
|
|
344
|
+
});
|
|
345
|
+
observation.embeddings.forEach((embedding, index) => {
|
|
346
|
+
statements.insertEmbedding.run(embedding.embeddingId
|
|
347
|
+
?? deterministicUuid(observation.segmentId, `embedding:${index}`), observation.segmentId, embedding.modality, embedding.embeddingSpace, embedding.vector.length, vectorToBlob(embedding.vector), embedding.sourceHash, batch.generation, now);
|
|
348
|
+
});
|
|
349
|
+
observation.fingerprints.forEach((fingerprint, index) => {
|
|
350
|
+
statements.insertFingerprint.run(fingerprint.fingerprintId
|
|
351
|
+
?? deterministicUuid(observation.segmentId, `fingerprint:${index}`), observation.segmentId, fingerprint.kind, fingerprint.value, fingerprint.extractorVersion, batch.generation);
|
|
352
|
+
});
|
|
353
|
+
}
|
|
354
|
+
function existingSegmentIds(context, observations) {
|
|
355
|
+
const existing = new Set();
|
|
356
|
+
for (let offset = 0; offset < observations.length; offset += 500) {
|
|
357
|
+
const ids = observations
|
|
358
|
+
.slice(offset, offset + 500)
|
|
359
|
+
.map((observation) => observation.segmentId);
|
|
360
|
+
if (ids.length === 0)
|
|
361
|
+
continue;
|
|
362
|
+
const rows = context.store.db
|
|
363
|
+
.prepare(`SELECT segment_id FROM runtime_media_segments
|
|
364
|
+
WHERE segment_id IN (${ids.map(() => "?").join(",")})`)
|
|
365
|
+
.all(...ids);
|
|
366
|
+
for (const row of rows)
|
|
367
|
+
existing.add(row.segment_id);
|
|
368
|
+
}
|
|
369
|
+
return existing;
|
|
370
|
+
}
|
|
371
|
+
function prepareObservationStatements(context) {
|
|
372
|
+
return {
|
|
373
|
+
upsertSegment: context.store.db.prepare(`INSERT INTO runtime_media_segments(
|
|
374
|
+
segment_id, artifact_id, stream_id, object_hash,
|
|
375
|
+
source_range_json, source_path, segment_kind,
|
|
376
|
+
representative_tick, segmentation_version,
|
|
377
|
+
generation, manifest_id, created_at
|
|
378
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
379
|
+
ON CONFLICT(segment_id) DO UPDATE SET
|
|
380
|
+
source_range_json=excluded.source_range_json,
|
|
381
|
+
source_path=excluded.source_path,
|
|
382
|
+
segment_kind=excluded.segment_kind,
|
|
383
|
+
representative_tick=excluded.representative_tick,
|
|
384
|
+
segmentation_version=excluded.segmentation_version`),
|
|
385
|
+
deleteText: context.store.db.prepare("DELETE FROM runtime_segment_text WHERE segment_id=?"),
|
|
386
|
+
deleteEmbeddings: context.store.db.prepare("DELETE FROM runtime_segment_embeddings WHERE segment_id=?"),
|
|
387
|
+
deleteFingerprints: context.store.db.prepare("DELETE FROM runtime_segment_fingerprints WHERE segment_id=?"),
|
|
388
|
+
insertText: context.store.db.prepare(`INSERT INTO runtime_segment_text(
|
|
389
|
+
text_id, segment_id, kind, language, text,
|
|
390
|
+
start_utf8_byte, end_utf8_byte, confidence,
|
|
391
|
+
provenance_json, generation
|
|
392
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`),
|
|
393
|
+
insertEmbedding: context.store.db.prepare(`INSERT INTO runtime_segment_embeddings(
|
|
394
|
+
embedding_id, segment_id, modality, embedding_space,
|
|
395
|
+
dimensions, vector_blob, source_hash, generation, updated_at
|
|
396
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`),
|
|
397
|
+
insertFingerprint: context.store.db.prepare(`INSERT INTO runtime_segment_fingerprints(
|
|
398
|
+
fingerprint_id, segment_id, kind, value,
|
|
399
|
+
extractor_version, generation
|
|
400
|
+
) VALUES (?, ?, ?, ?, ?, ?)`),
|
|
401
|
+
};
|
|
402
|
+
}
|
|
403
|
+
function activateGeneration(context, manifestId, generation) {
|
|
404
|
+
const manifest = requiredManifest(context, manifestId);
|
|
405
|
+
const row = context.store.db
|
|
406
|
+
.prepare(`SELECT state FROM runtime_index_generations
|
|
407
|
+
WHERE embedding_space=? AND generation=? AND manifest_id=?`)
|
|
408
|
+
.get(manifest.embeddingSpace, generation, manifestId);
|
|
409
|
+
if (!row) {
|
|
410
|
+
throw new EngineFault({
|
|
411
|
+
code: "NOT_FOUND",
|
|
412
|
+
message: `Index generation not found: ${generation}`,
|
|
413
|
+
});
|
|
414
|
+
}
|
|
415
|
+
const ready = context.store.db
|
|
416
|
+
.prepare(`SELECT 1 AS present FROM runtime_index_coverage
|
|
417
|
+
WHERE manifest_id=? AND generation=?
|
|
418
|
+
AND state IN ('ready','partial') LIMIT 1`)
|
|
419
|
+
.get(manifestId, generation);
|
|
420
|
+
if (!ready) {
|
|
421
|
+
throw new EngineFault({
|
|
422
|
+
code: "INDEX_INCOMPLETE",
|
|
423
|
+
message: "Generation has no searchable coverage",
|
|
424
|
+
});
|
|
425
|
+
}
|
|
426
|
+
context.store.runtime((now) => {
|
|
427
|
+
context.store.db
|
|
428
|
+
.prepare(`UPDATE runtime_index_generations
|
|
429
|
+
SET state='retired'
|
|
430
|
+
WHERE embedding_space=? AND state='active' AND generation<>?`)
|
|
431
|
+
.run(manifest.embeddingSpace, generation);
|
|
432
|
+
context.store.db
|
|
433
|
+
.prepare(`UPDATE runtime_index_generations
|
|
434
|
+
SET state='active', activated_at=?
|
|
435
|
+
WHERE embedding_space=? AND generation=?`)
|
|
436
|
+
.run(now, manifest.embeddingSpace, generation);
|
|
437
|
+
});
|
|
438
|
+
return { manifestId, generation };
|
|
439
|
+
}
|
|
440
|
+
async function queryTemporalIndex(context, providers, query, prepared) {
|
|
441
|
+
return resultOf(async () => {
|
|
442
|
+
validateQuery(query, prepared?.reference);
|
|
443
|
+
if (prepared)
|
|
444
|
+
validatePreparedReference(prepared.reference, prepared.options);
|
|
445
|
+
const active = activeGenerations(context);
|
|
446
|
+
const generationKey = hashJson(active.map((item) => [item.embedding_space, item.generation]));
|
|
447
|
+
const offset = decodeCursor(query.cursor, generationKey);
|
|
448
|
+
const segments = querySegments(context, active, query);
|
|
449
|
+
const candidates = new Map(segments.map((segment) => [
|
|
450
|
+
segment.segment_id,
|
|
451
|
+
{ segment, signals: [], score: 0 },
|
|
452
|
+
]));
|
|
453
|
+
if (query.text) {
|
|
454
|
+
if (allowsLexical(query.modalities)) {
|
|
455
|
+
addLexicalSignals(context, candidates, query.text, active);
|
|
456
|
+
}
|
|
457
|
+
if (allowsSemanticVector(query.modalities)) {
|
|
458
|
+
await addTextVectorSignals(context, providers, candidates, query.text, active);
|
|
459
|
+
}
|
|
460
|
+
}
|
|
461
|
+
if (query.reference) {
|
|
462
|
+
addReferenceSignals(context, candidates, query.reference, active);
|
|
463
|
+
}
|
|
464
|
+
if (prepared) {
|
|
465
|
+
addPreparedReferenceSignals(context, candidates, prepared.reference, prepared.options, active);
|
|
466
|
+
}
|
|
467
|
+
const ranked = [...candidates.values()]
|
|
468
|
+
.filter((candidate) => (!query.sourceArtifactIds
|
|
469
|
+
|| query.sourceArtifactIds.includes(candidate.segment.artifact_id))
|
|
470
|
+
&& (!query.artifactKinds
|
|
471
|
+
|| query.artifactKinds.includes(candidate.segment.artifact_kind)))
|
|
472
|
+
.map((candidate) => ({
|
|
473
|
+
...candidate,
|
|
474
|
+
signals: candidate.signals.filter((signal) => allowsSignal(query.modalities, signal.kind)),
|
|
475
|
+
}))
|
|
476
|
+
.filter((candidate) => candidate.signals.length > 0)
|
|
477
|
+
.map(finalizeCandidate)
|
|
478
|
+
.filter((candidate) => candidate.score >= (query.minScore ?? 0))
|
|
479
|
+
.sort(compareCandidates);
|
|
480
|
+
const deduplicated = collapseOverlaps(ranked);
|
|
481
|
+
const limit = Math.min(query.limit ?? 20, 100);
|
|
482
|
+
const page = deduplicated.slice(offset, offset + limit);
|
|
483
|
+
return {
|
|
484
|
+
hits: page.map(candidateToHit),
|
|
485
|
+
...(offset + limit < deduplicated.length
|
|
486
|
+
? {
|
|
487
|
+
nextCursor: encodeCursor(generationKey, offset + limit),
|
|
488
|
+
}
|
|
489
|
+
: {}),
|
|
490
|
+
coverage: searchCoverage(context),
|
|
491
|
+
};
|
|
492
|
+
});
|
|
493
|
+
}
|
|
494
|
+
function allowsLexical(modalities) {
|
|
495
|
+
return (!modalities
|
|
496
|
+
|| modalities.includes("auto")
|
|
497
|
+
|| modalities.some((item) => item === "speech" || item === "ocr" || item === "metadata"));
|
|
498
|
+
}
|
|
499
|
+
function allowsSemanticVector(modalities) {
|
|
500
|
+
return (!modalities
|
|
501
|
+
|| modalities.includes("auto")
|
|
502
|
+
|| modalities.some((item) => item === "visual"
|
|
503
|
+
|| item === "audio"
|
|
504
|
+
|| item === "speech"
|
|
505
|
+
|| item === "ocr"));
|
|
506
|
+
}
|
|
507
|
+
function allowsSignal(modalities, signal) {
|
|
508
|
+
if (!modalities || modalities.includes("auto"))
|
|
509
|
+
return true;
|
|
510
|
+
if (signal === "exact" || signal === "near")
|
|
511
|
+
return true;
|
|
512
|
+
return modalities.includes(signal);
|
|
513
|
+
}
|
|
514
|
+
function activeGenerations(context) {
|
|
515
|
+
return context.store.db
|
|
516
|
+
.prepare(`SELECT embedding_space, generation, manifest_id, state
|
|
517
|
+
FROM runtime_index_generations
|
|
518
|
+
WHERE state='active'
|
|
519
|
+
ORDER BY embedding_space, generation`)
|
|
520
|
+
.all();
|
|
521
|
+
}
|
|
522
|
+
function querySegments(context, active, query) {
|
|
523
|
+
if (active.length === 0)
|
|
524
|
+
return [];
|
|
525
|
+
const generations = active.map((item) => item.generation);
|
|
526
|
+
const placeholders = generations.map(() => "?").join(",");
|
|
527
|
+
const rows = context.store.db
|
|
528
|
+
.prepare(`SELECT s.segment_id, s.artifact_id, a.label AS artifact_label,
|
|
529
|
+
a.kind AS artifact_kind, a.created_at AS artifact_created_at,
|
|
530
|
+
s.stream_id, s.object_hash,
|
|
531
|
+
s.source_range_json, s.source_path, s.segment_kind,
|
|
532
|
+
s.representative_tick, s.generation, s.manifest_id
|
|
533
|
+
FROM runtime_media_segments s
|
|
534
|
+
JOIN artifacts a ON a.artifact_id=s.artifact_id
|
|
535
|
+
WHERE s.generation IN (${placeholders})
|
|
536
|
+
ORDER BY s.artifact_id, s.stream_id, s.representative_tick, s.segment_id`)
|
|
537
|
+
.all(...generations);
|
|
538
|
+
return rows.filter((row) => {
|
|
539
|
+
if (query.artifactKinds && !query.artifactKinds.includes(row.artifact_kind)) {
|
|
540
|
+
return false;
|
|
541
|
+
}
|
|
542
|
+
if (query.sourceArtifactIds
|
|
543
|
+
&& !query.sourceArtifactIds.includes(row.artifact_id)) {
|
|
544
|
+
return false;
|
|
545
|
+
}
|
|
546
|
+
if (query.createdAfter !== undefined
|
|
547
|
+
&& row.artifact_created_at <= query.createdAfter) {
|
|
548
|
+
return false;
|
|
549
|
+
}
|
|
550
|
+
if (query.createdBefore !== undefined
|
|
551
|
+
&& row.artifact_created_at >= query.createdBefore) {
|
|
552
|
+
return false;
|
|
553
|
+
}
|
|
554
|
+
if (query.labels && !artifactHasLabels(context, row.artifact_id, query.labels)) {
|
|
555
|
+
return false;
|
|
556
|
+
}
|
|
557
|
+
if (query.orientations
|
|
558
|
+
&& !query.orientations.includes(artifactOrientation(context, row.artifact_id))) {
|
|
559
|
+
return false;
|
|
560
|
+
}
|
|
561
|
+
if (query.indexingStates
|
|
562
|
+
&& !query.indexingStates.includes(artifactGenerationState(context, row.artifact_id, row.generation))) {
|
|
563
|
+
return false;
|
|
564
|
+
}
|
|
565
|
+
if (query.durationMs) {
|
|
566
|
+
const range = segmentRange(row);
|
|
567
|
+
if (range) {
|
|
568
|
+
const durationMs = (range.durationTicks * range.timeBase.numerator * 1_000)
|
|
569
|
+
/ range.timeBase.denominator;
|
|
570
|
+
if ((query.durationMs.min !== undefined
|
|
571
|
+
&& durationMs < query.durationMs.min)
|
|
572
|
+
|| (query.durationMs.max !== undefined
|
|
573
|
+
&& durationMs > query.durationMs.max)) {
|
|
574
|
+
return false;
|
|
575
|
+
}
|
|
576
|
+
}
|
|
577
|
+
}
|
|
578
|
+
return true;
|
|
579
|
+
});
|
|
580
|
+
}
|
|
581
|
+
function artifactHasLabels(context, artifactId, requested) {
|
|
582
|
+
const row = context.store.db
|
|
583
|
+
.prepare(`SELECT value_json FROM artifact_metadata
|
|
584
|
+
WHERE artifact_id=? AND key='labels'`)
|
|
585
|
+
.get(artifactId);
|
|
586
|
+
const labels = row ? parseJson(row.value_json, []) : [];
|
|
587
|
+
return requested.every((label) => labels.includes(label));
|
|
588
|
+
}
|
|
589
|
+
function artifactOrientation(context, artifactId) {
|
|
590
|
+
const row = context.store.db
|
|
591
|
+
.prepare(`SELECT profile_json FROM artifact_streams
|
|
592
|
+
WHERE artifact_id=? AND kind='video'
|
|
593
|
+
ORDER BY stream_index LIMIT 1`)
|
|
594
|
+
.get(artifactId);
|
|
595
|
+
const profile = row
|
|
596
|
+
? parseJson(row.profile_json, {})
|
|
597
|
+
: {};
|
|
598
|
+
const width = profile.video?.width ?? 1;
|
|
599
|
+
const height = profile.video?.height ?? 1;
|
|
600
|
+
if (width === height)
|
|
601
|
+
return "square";
|
|
602
|
+
return width > height ? "landscape" : "portrait";
|
|
603
|
+
}
|
|
604
|
+
function artifactGenerationState(context, artifactId, generation) {
|
|
605
|
+
const rows = context.store.db
|
|
606
|
+
.prepare(`SELECT state FROM runtime_index_coverage
|
|
607
|
+
WHERE artifact_id=? AND generation=?`)
|
|
608
|
+
.all(artifactId, generation);
|
|
609
|
+
return aggregateCoverageState(rows);
|
|
610
|
+
}
|
|
611
|
+
function addLexicalSignals(context, candidates, text, active) {
|
|
612
|
+
const generations = active.map((item) => item.generation);
|
|
613
|
+
if (generations.length === 0)
|
|
614
|
+
return;
|
|
615
|
+
const rows = context.store.db
|
|
616
|
+
.prepare(`SELECT segment_id, kind, text
|
|
617
|
+
FROM runtime_segment_text
|
|
618
|
+
WHERE generation IN (${generations.map(() => "?").join(",")})
|
|
619
|
+
ORDER BY segment_id, kind, text`)
|
|
620
|
+
.all(...generations);
|
|
621
|
+
const normalized = text.trim().toLocaleLowerCase();
|
|
622
|
+
const quoted = [...text.matchAll(/"([^"]+)"/g)]
|
|
623
|
+
.map((match) => match[1]?.trim().toLocaleLowerCase())
|
|
624
|
+
.filter((item) => Boolean(item));
|
|
625
|
+
const terms = normalized
|
|
626
|
+
.replaceAll('"', "")
|
|
627
|
+
.split(/\s+/)
|
|
628
|
+
.filter((term) => term.length > 1);
|
|
629
|
+
const matches = rows
|
|
630
|
+
.map((row) => {
|
|
631
|
+
const haystack = row.text.toLocaleLowerCase();
|
|
632
|
+
const exact = quoted.some((phrase) => haystack.includes(phrase));
|
|
633
|
+
const termCount = terms.filter((term) => haystack.includes(term)).length;
|
|
634
|
+
return {
|
|
635
|
+
row,
|
|
636
|
+
exact,
|
|
637
|
+
score: (exact ? 2 : 0)
|
|
638
|
+
+ (terms.length === 0 ? 0 : termCount / terms.length),
|
|
639
|
+
};
|
|
640
|
+
})
|
|
641
|
+
.filter((item) => item.score > 0)
|
|
642
|
+
.sort((left, right) => right.score - left.score
|
|
643
|
+
|| left.row.segment_id.localeCompare(right.row.segment_id));
|
|
644
|
+
matches.forEach((match, index) => {
|
|
645
|
+
const candidate = candidates.get(match.row.segment_id);
|
|
646
|
+
if (!candidate)
|
|
647
|
+
return;
|
|
648
|
+
candidate.signals.push({
|
|
649
|
+
kind: match.row.kind === "ocr"
|
|
650
|
+
? "ocr"
|
|
651
|
+
: match.row.kind === "transcript"
|
|
652
|
+
? "speech"
|
|
653
|
+
: "metadata",
|
|
654
|
+
rank: index + 1,
|
|
655
|
+
score: match.score,
|
|
656
|
+
explanation: match.exact
|
|
657
|
+
? "Exact quoted text matched this moment"
|
|
658
|
+
: "Query terms matched indexed text",
|
|
659
|
+
});
|
|
660
|
+
if (!candidate.excerpt || match.exact)
|
|
661
|
+
candidate.excerpt = match.row.text;
|
|
662
|
+
});
|
|
663
|
+
}
|
|
664
|
+
async function addTextVectorSignals(context, providers, candidates, text, active) {
|
|
665
|
+
for (const generation of active) {
|
|
666
|
+
const provider = providers.get(generation.manifest_id);
|
|
667
|
+
if (!provider)
|
|
668
|
+
continue;
|
|
669
|
+
const manifest = requiredManifest(context, generation.manifest_id);
|
|
670
|
+
await provider.prepare();
|
|
671
|
+
const queryVector = provider.embedText(text);
|
|
672
|
+
const resolved = await queryVector;
|
|
673
|
+
if (resolved.length !== manifest.dimensions) {
|
|
674
|
+
throw new EngineFault({
|
|
675
|
+
code: "MANIFEST_INCOMPATIBLE",
|
|
676
|
+
message: "Query provider returned incompatible vector dimensions",
|
|
677
|
+
});
|
|
678
|
+
}
|
|
679
|
+
for (const modality of manifest.modalities) {
|
|
680
|
+
if (modality === "metadata")
|
|
681
|
+
continue;
|
|
682
|
+
rankVectorRows(context, candidates, generation, resolved, modality, `Text-to-${modality} semantic similarity`);
|
|
683
|
+
}
|
|
684
|
+
}
|
|
685
|
+
}
|
|
686
|
+
function addReferenceSignals(context, candidates, reference, active) {
|
|
687
|
+
for (const generation of active) {
|
|
688
|
+
if (reference.kind === "video") {
|
|
689
|
+
addOrderedVideoSignals(context, candidates, reference.range, generation);
|
|
690
|
+
continue;
|
|
691
|
+
}
|
|
692
|
+
const queryRows = referenceEmbeddingRows(context, reference, generation);
|
|
693
|
+
if (queryRows.length === 0)
|
|
694
|
+
continue;
|
|
695
|
+
const queryVector = averageVectors(queryRows.map((row) => blobToVector(row.vector_blob, row.dimensions)));
|
|
696
|
+
const modality = reference.kind === "audio" ? "audio" : "visual";
|
|
697
|
+
rankVectorRows(context, candidates, generation, queryVector, modality, `${reference.kind} reference similarity`);
|
|
698
|
+
addFingerprintSignals(context, candidates, queryRows, generation);
|
|
699
|
+
}
|
|
700
|
+
}
|
|
701
|
+
function addOrderedVideoSignals(context, candidates, rangeInput, generation) {
|
|
702
|
+
const range = normalizeSourceRange(rangeInput);
|
|
703
|
+
const query = orderedEmbeddings(context, generation, (segmentRangeValue) => segmentRangeValue.streamId === range.streamId
|
|
704
|
+
&& segmentRangeValue.objectHash === range.objectHash
|
|
705
|
+
&& rangesOverlap(segmentRangeValue.startTick, sourceRangeEndTick(segmentRangeValue), range.startTick, sourceRangeEndTick(range)));
|
|
706
|
+
if (query.length === 0)
|
|
707
|
+
return;
|
|
708
|
+
const sample = evenlySample(query, Math.min(query.length, 16));
|
|
709
|
+
addOrderedVideoVectorSignals(context, candidates, sample, generation);
|
|
710
|
+
}
|
|
711
|
+
function addPreparedReferenceSignals(context, candidates, reference, options, active) {
|
|
712
|
+
for (const generation of active) {
|
|
713
|
+
if (generation.embedding_space !== reference.embeddingSpace)
|
|
714
|
+
continue;
|
|
715
|
+
const manifest = requiredManifest(context, generation.manifest_id);
|
|
716
|
+
if (reference.kind === "image") {
|
|
717
|
+
const queryVector = preparedVector(reference.vector, manifest.dimensions);
|
|
718
|
+
rankVectorRows(context, candidates, generation, queryVector, "visual", "Prepared image reference similarity");
|
|
719
|
+
addPreparedFingerprintSignals(context, candidates, reference.fingerprints ?? [], generation);
|
|
720
|
+
continue;
|
|
721
|
+
}
|
|
722
|
+
const samples = preparedVideoSamples(reference, options);
|
|
723
|
+
for (const sample of samples) {
|
|
724
|
+
preparedVector(sample.vector, manifest.dimensions);
|
|
725
|
+
}
|
|
726
|
+
addOrderedVideoVectorSignals(context, candidates, samples.map((sample) => ({
|
|
727
|
+
vector: Float32Array.from(sample.vector),
|
|
728
|
+
})), generation);
|
|
729
|
+
}
|
|
730
|
+
}
|
|
731
|
+
function addOrderedVideoVectorSignals(context, candidates, sample, generation) {
|
|
732
|
+
if (sample.length === 0)
|
|
733
|
+
return;
|
|
734
|
+
const all = orderedEmbeddings(context, generation, () => true);
|
|
735
|
+
const byStream = new Map();
|
|
736
|
+
for (const item of all) {
|
|
737
|
+
const values = byStream.get(item.range.streamId) ?? [];
|
|
738
|
+
values.push(item);
|
|
739
|
+
byStream.set(item.range.streamId, values);
|
|
740
|
+
}
|
|
741
|
+
const windows = [];
|
|
742
|
+
for (const values of byStream.values()) {
|
|
743
|
+
values.sort((left, right) => left.range.startTick - right.range.startTick
|
|
744
|
+
|| left.segment.segment_id.localeCompare(right.segment.segment_id));
|
|
745
|
+
if (values.length < sample.length)
|
|
746
|
+
continue;
|
|
747
|
+
for (let start = 0; start <= values.length - sample.length; start += 1) {
|
|
748
|
+
const window = values.slice(start, start + sample.length);
|
|
749
|
+
const aligned = sample.reduce((sum, item, index) => sum + cosineSimilarity(item.vector, window[index].vector), 0) / sample.length;
|
|
750
|
+
const coherence = transitionCoherence(sample, window) * 0.75
|
|
751
|
+
+ temporalContinuity(window) * 0.25;
|
|
752
|
+
windows.push({
|
|
753
|
+
values: window,
|
|
754
|
+
score: aligned * 0.75 + coherence * 0.25,
|
|
755
|
+
coherence,
|
|
756
|
+
});
|
|
757
|
+
}
|
|
758
|
+
}
|
|
759
|
+
windows
|
|
760
|
+
.sort((left, right) => right.score - left.score
|
|
761
|
+
|| left.values[0].segment.segment_id.localeCompare(right.values[0].segment.segment_id))
|
|
762
|
+
.forEach((window, index) => {
|
|
763
|
+
const first = window.values[0];
|
|
764
|
+
const last = window.values.at(-1);
|
|
765
|
+
const combined = {
|
|
766
|
+
...first.range,
|
|
767
|
+
durationTicks: sourceRangeEndTick(last.range) - first.range.startTick,
|
|
768
|
+
};
|
|
769
|
+
const segmentId = `ordered:${generation.generation}:${first.segment.segment_id}:${last.segment.segment_id}`;
|
|
770
|
+
const candidate = {
|
|
771
|
+
segment: {
|
|
772
|
+
...first.segment,
|
|
773
|
+
segment_id: segmentId,
|
|
774
|
+
source_range_json: canonicalJson(combined),
|
|
775
|
+
representative_tick: first.range.startTick + Math.floor(combined.durationTicks / 2),
|
|
776
|
+
},
|
|
777
|
+
signals: [
|
|
778
|
+
...(isExactOrderedWindow(sample, window.values)
|
|
779
|
+
? [{
|
|
780
|
+
kind: "exact",
|
|
781
|
+
rank: 1,
|
|
782
|
+
score: 1,
|
|
783
|
+
explanation: "Exact source bytes and ordered range matched",
|
|
784
|
+
}]
|
|
785
|
+
: []),
|
|
786
|
+
{
|
|
787
|
+
kind: "visual",
|
|
788
|
+
rank: index + 1,
|
|
789
|
+
score: window.score,
|
|
790
|
+
explanation: `Ordered video alignment with temporal coherence ${window.coherence.toFixed(3)}`,
|
|
791
|
+
},
|
|
792
|
+
],
|
|
793
|
+
score: 0,
|
|
794
|
+
};
|
|
795
|
+
candidates.set(segmentId, candidate);
|
|
796
|
+
});
|
|
797
|
+
}
|
|
798
|
+
function orderedEmbeddings(context, generation, include) {
|
|
799
|
+
const rows = context.store.db
|
|
800
|
+
.prepare(`SELECT s.segment_id, s.artifact_id, a.label AS artifact_label,
|
|
801
|
+
a.kind AS artifact_kind, a.created_at AS artifact_created_at,
|
|
802
|
+
s.stream_id, s.object_hash, s.source_range_json,
|
|
803
|
+
s.source_path, s.segment_kind, s.representative_tick,
|
|
804
|
+
s.generation, s.manifest_id, e.dimensions, e.vector_blob
|
|
805
|
+
FROM runtime_media_segments s
|
|
806
|
+
JOIN artifacts a ON a.artifact_id=s.artifact_id
|
|
807
|
+
JOIN runtime_segment_embeddings e ON e.segment_id=s.segment_id
|
|
808
|
+
WHERE s.generation=? AND e.embedding_space=?
|
|
809
|
+
AND e.modality='visual' AND s.source_range_json IS NOT NULL
|
|
810
|
+
ORDER BY s.stream_id, s.representative_tick, s.segment_id`)
|
|
811
|
+
.all(generation.generation, generation.embedding_space);
|
|
812
|
+
return rows.flatMap((row) => {
|
|
813
|
+
const range = segmentRange(row);
|
|
814
|
+
return range && include(range)
|
|
815
|
+
? [{
|
|
816
|
+
segment: row,
|
|
817
|
+
range,
|
|
818
|
+
vector: blobToVector(row.vector_blob, row.dimensions),
|
|
819
|
+
}]
|
|
820
|
+
: [];
|
|
821
|
+
});
|
|
822
|
+
}
|
|
823
|
+
function transitionCoherence(query, candidate) {
|
|
824
|
+
if (query.length < 2)
|
|
825
|
+
return 1;
|
|
826
|
+
let score = 0;
|
|
827
|
+
for (let index = 1; index < query.length; index += 1) {
|
|
828
|
+
score += cosineSimilarity(vectorDifference(query[index].vector, query[index - 1].vector), vectorDifference(candidate[index].vector, candidate[index - 1].vector));
|
|
829
|
+
}
|
|
830
|
+
return score / (query.length - 1);
|
|
831
|
+
}
|
|
832
|
+
function temporalContinuity(candidate) {
|
|
833
|
+
if (candidate.length < 2)
|
|
834
|
+
return 1;
|
|
835
|
+
let score = 0;
|
|
836
|
+
for (let index = 1; index < candidate.length; index += 1) {
|
|
837
|
+
const previous = candidate[index - 1].range;
|
|
838
|
+
const current = candidate[index].range;
|
|
839
|
+
const gapMs = Math.max(0, sourceTickToMs(current.startTick, current)
|
|
840
|
+
- sourceTickToMs(sourceRangeEndTick(previous), previous));
|
|
841
|
+
const scaleMs = Math.max(1, (sourceTickToMs(previous.durationTicks, previous)
|
|
842
|
+
+ sourceTickToMs(current.durationTicks, current)) / 2);
|
|
843
|
+
score += 1 / (1 + gapMs / scaleMs);
|
|
844
|
+
}
|
|
845
|
+
return score / (candidate.length - 1);
|
|
846
|
+
}
|
|
847
|
+
function isExactOrderedWindow(query, candidate) {
|
|
848
|
+
return query.length === candidate.length && query.every((item, index) => {
|
|
849
|
+
const reference = item.range;
|
|
850
|
+
const match = candidate[index]?.range;
|
|
851
|
+
return Boolean(reference
|
|
852
|
+
&& match
|
|
853
|
+
&& reference.objectHash === match.objectHash
|
|
854
|
+
&& nearlyEqual(sourceTickToMs(reference.startTick, reference), sourceTickToMs(match.startTick, match))
|
|
855
|
+
&& nearlyEqual(sourceTickToMs(reference.durationTicks, reference), sourceTickToMs(match.durationTicks, match)));
|
|
856
|
+
});
|
|
857
|
+
}
|
|
858
|
+
function sourceTickToMs(tick, range) {
|
|
859
|
+
return (tick * range.timeBase.numerator * 1_000) / range.timeBase.denominator;
|
|
860
|
+
}
|
|
861
|
+
function nearlyEqual(left, right) {
|
|
862
|
+
return Math.abs(left - right) <= 1e-6;
|
|
863
|
+
}
|
|
864
|
+
function vectorDifference(left, right) {
|
|
865
|
+
const result = new Float32Array(left.length);
|
|
866
|
+
for (let index = 0; index < left.length; index += 1) {
|
|
867
|
+
result[index] = left[index] - right[index];
|
|
868
|
+
}
|
|
869
|
+
return result;
|
|
870
|
+
}
|
|
871
|
+
function evenlySample(values, count) {
|
|
872
|
+
if (count <= 1)
|
|
873
|
+
return values.length === 0 ? [] : [values[0]];
|
|
874
|
+
if (count >= values.length)
|
|
875
|
+
return values;
|
|
876
|
+
return Array.from({ length: count }, (_, index) => {
|
|
877
|
+
const position = Math.round((index * (values.length - 1)) / (count - 1));
|
|
878
|
+
return values[position];
|
|
879
|
+
});
|
|
880
|
+
}
|
|
881
|
+
function rankVectorRows(context, candidates, generation, queryVector, modality, explanation) {
|
|
882
|
+
const rows = context.store.db
|
|
883
|
+
.prepare(`SELECT segment_id, embedding_space, modality, dimensions, vector_blob
|
|
884
|
+
FROM runtime_segment_embeddings
|
|
885
|
+
WHERE generation=? AND embedding_space=?
|
|
886
|
+
AND modality=?
|
|
887
|
+
ORDER BY segment_id`)
|
|
888
|
+
.all(generation.generation, generation.embedding_space, modality);
|
|
889
|
+
rows
|
|
890
|
+
.map((row) => ({
|
|
891
|
+
row,
|
|
892
|
+
score: cosineSimilarity(queryVector, blobToVector(row.vector_blob, row.dimensions)),
|
|
893
|
+
}))
|
|
894
|
+
.filter((item) => Number.isFinite(item.score))
|
|
895
|
+
.sort((left, right) => right.score - left.score
|
|
896
|
+
|| left.row.segment_id.localeCompare(right.row.segment_id))
|
|
897
|
+
.forEach((item, index) => {
|
|
898
|
+
const candidate = candidates.get(item.row.segment_id);
|
|
899
|
+
if (!candidate)
|
|
900
|
+
return;
|
|
901
|
+
candidate.signals.push({
|
|
902
|
+
kind: modality,
|
|
903
|
+
rank: index + 1,
|
|
904
|
+
score: item.score,
|
|
905
|
+
explanation,
|
|
906
|
+
});
|
|
907
|
+
});
|
|
908
|
+
}
|
|
909
|
+
function referenceEmbeddingRows(context, reference, generation) {
|
|
910
|
+
const segments = referenceSegments(context, reference, generation.generation);
|
|
911
|
+
if (segments.length === 0)
|
|
912
|
+
return [];
|
|
913
|
+
const placeholders = segments.map(() => "?").join(",");
|
|
914
|
+
const modality = reference.kind === "audio" ? "audio" : "visual";
|
|
915
|
+
return context.store.db
|
|
916
|
+
.prepare(`SELECT segment_id, embedding_space, modality, dimensions, vector_blob
|
|
917
|
+
FROM runtime_segment_embeddings
|
|
918
|
+
WHERE segment_id IN (${placeholders})
|
|
919
|
+
AND embedding_space=? AND modality=?
|
|
920
|
+
ORDER BY segment_id`)
|
|
921
|
+
.all(...segments, generation.embedding_space, modality);
|
|
922
|
+
}
|
|
923
|
+
function referenceSegments(context, reference, generation) {
|
|
924
|
+
if (reference.kind === "image") {
|
|
925
|
+
const artifact = context.artifactRow(reference.artifact);
|
|
926
|
+
return context.store.db
|
|
927
|
+
.prepare(`SELECT segment_id FROM runtime_media_segments
|
|
928
|
+
WHERE generation=? AND artifact_id=?
|
|
929
|
+
ORDER BY segment_id`)
|
|
930
|
+
.all(generation, artifact.artifact_id).map((row) => row.segment_id);
|
|
931
|
+
}
|
|
932
|
+
const range = reference.kind === "frame"
|
|
933
|
+
? undefined
|
|
934
|
+
: normalizeSourceRange(reference.range);
|
|
935
|
+
const point = reference.kind === "frame"
|
|
936
|
+
? normalizeSourcePoint(reference.source)
|
|
937
|
+
: undefined;
|
|
938
|
+
const rows = context.store.db
|
|
939
|
+
.prepare(`SELECT segment_id, source_range_json
|
|
940
|
+
FROM runtime_media_segments
|
|
941
|
+
WHERE generation=? AND stream_id=? AND object_hash=?
|
|
942
|
+
ORDER BY representative_tick, segment_id`)
|
|
943
|
+
.all(generation, point?.streamId ?? range?.streamId, point?.objectHash ?? range?.objectHash);
|
|
944
|
+
return rows
|
|
945
|
+
.filter((row) => {
|
|
946
|
+
if (!row.source_range_json)
|
|
947
|
+
return false;
|
|
948
|
+
const candidate = parseJson(row.source_range_json, null);
|
|
949
|
+
if (!candidate)
|
|
950
|
+
return false;
|
|
951
|
+
if (point) {
|
|
952
|
+
return (candidate.startTick <= point.tick
|
|
953
|
+
&& point.tick < sourceRangeEndTick(candidate));
|
|
954
|
+
}
|
|
955
|
+
return range
|
|
956
|
+
? rangesOverlap(candidate.startTick, sourceRangeEndTick(candidate), range.startTick, sourceRangeEndTick(range))
|
|
957
|
+
: false;
|
|
958
|
+
})
|
|
959
|
+
.map((row) => row.segment_id);
|
|
960
|
+
}
|
|
961
|
+
function addFingerprintSignals(context, candidates, queryRows, generation) {
|
|
962
|
+
const segmentIds = unique(queryRows.map((row) => row.segment_id));
|
|
963
|
+
if (segmentIds.length === 0)
|
|
964
|
+
return;
|
|
965
|
+
const fingerprints = context.store.db
|
|
966
|
+
.prepare(`SELECT kind, value FROM runtime_segment_fingerprints
|
|
967
|
+
WHERE segment_id IN (${segmentIds.map(() => "?").join(",")})
|
|
968
|
+
ORDER BY kind, value`)
|
|
969
|
+
.all(...segmentIds);
|
|
970
|
+
addPreparedFingerprintSignals(context, candidates, fingerprints, generation);
|
|
971
|
+
}
|
|
972
|
+
function addPreparedFingerprintSignals(context, candidates, fingerprints, generation) {
|
|
973
|
+
for (const fingerprint of fingerprints) {
|
|
974
|
+
const rows = context.store.db
|
|
975
|
+
.prepare(`SELECT segment_id FROM runtime_segment_fingerprints
|
|
976
|
+
WHERE generation=? AND kind=? AND value=?
|
|
977
|
+
ORDER BY segment_id`)
|
|
978
|
+
.all(generation.generation, fingerprint.kind, fingerprint.value);
|
|
979
|
+
rows.forEach((row, index) => {
|
|
980
|
+
const candidate = candidates.get(row.segment_id);
|
|
981
|
+
if (!candidate)
|
|
982
|
+
return;
|
|
983
|
+
candidate.signals.push({
|
|
984
|
+
kind: fingerprint.kind === "sha256" ? "exact" : "near",
|
|
985
|
+
rank: index + 1,
|
|
986
|
+
score: 1,
|
|
987
|
+
explanation: fingerprint.kind === "sha256"
|
|
988
|
+
? "Exact source fingerprint matched"
|
|
989
|
+
: "Near-duplicate fingerprint matched",
|
|
990
|
+
});
|
|
991
|
+
});
|
|
992
|
+
}
|
|
993
|
+
}
|
|
994
|
+
function finalizeCandidate(candidate) {
|
|
995
|
+
const sortedSignals = [...candidate.signals].sort((left, right) => signalPriority(left.kind) - signalPriority(right.kind)
|
|
996
|
+
|| left.rank - right.rank);
|
|
997
|
+
const rrf = sortedSignals.reduce((sum, signal) => sum + 1 / (60 + signal.rank), 0);
|
|
998
|
+
const exactBoost = sortedSignals.some((signal) => signal.kind === "exact")
|
|
999
|
+
? 1
|
|
1000
|
+
: sortedSignals.some((signal) => signal.kind === "near")
|
|
1001
|
+
? 0.25
|
|
1002
|
+
: 0;
|
|
1003
|
+
return { ...candidate, signals: sortedSignals, score: rrf + exactBoost };
|
|
1004
|
+
}
|
|
1005
|
+
function compareCandidates(left, right) {
|
|
1006
|
+
return (right.score - left.score
|
|
1007
|
+
|| left.segment.artifact_id.localeCompare(right.segment.artifact_id)
|
|
1008
|
+
|| locationKey(left.segment).localeCompare(locationKey(right.segment))
|
|
1009
|
+
|| left.segment.segment_id.localeCompare(right.segment.segment_id));
|
|
1010
|
+
}
|
|
1011
|
+
function collapseOverlaps(candidates) {
|
|
1012
|
+
const accepted = [];
|
|
1013
|
+
const intervals = new Map();
|
|
1014
|
+
for (const candidate of candidates) {
|
|
1015
|
+
const range = segmentRange(candidate.segment);
|
|
1016
|
+
if (!range) {
|
|
1017
|
+
accepted.push(candidate);
|
|
1018
|
+
continue;
|
|
1019
|
+
}
|
|
1020
|
+
const key = `${candidate.segment.artifact_id}\u0000${range.streamId}`;
|
|
1021
|
+
const interval = {
|
|
1022
|
+
start: range.startTick,
|
|
1023
|
+
end: sourceRangeEndTick(range),
|
|
1024
|
+
duration: range.durationTicks,
|
|
1025
|
+
segmentId: candidate.segment.segment_id,
|
|
1026
|
+
};
|
|
1027
|
+
const root = intervals.get(key) ?? null;
|
|
1028
|
+
if (hasDuplicateInterval(root, interval))
|
|
1029
|
+
continue;
|
|
1030
|
+
intervals.set(key, insertInterval(root, interval));
|
|
1031
|
+
accepted.push(candidate);
|
|
1032
|
+
}
|
|
1033
|
+
return accepted;
|
|
1034
|
+
}
|
|
1035
|
+
function hasDuplicateInterval(node, candidate) {
|
|
1036
|
+
if (!node || node.maxEnd <= candidate.start)
|
|
1037
|
+
return false;
|
|
1038
|
+
if (node.left
|
|
1039
|
+
&& node.left.maxEnd > candidate.start
|
|
1040
|
+
&& hasDuplicateInterval(node.left, candidate)) {
|
|
1041
|
+
return true;
|
|
1042
|
+
}
|
|
1043
|
+
const other = node.interval;
|
|
1044
|
+
if (other.start < candidate.end) {
|
|
1045
|
+
const overlap = Math.max(0, Math.min(candidate.end, other.end) - Math.max(candidate.start, other.start));
|
|
1046
|
+
if (overlap / Math.min(candidate.duration, other.duration) >= 0.8) {
|
|
1047
|
+
return true;
|
|
1048
|
+
}
|
|
1049
|
+
}
|
|
1050
|
+
else {
|
|
1051
|
+
return false;
|
|
1052
|
+
}
|
|
1053
|
+
return hasDuplicateInterval(node.right, candidate);
|
|
1054
|
+
}
|
|
1055
|
+
function insertInterval(node, interval) {
|
|
1056
|
+
if (!node) {
|
|
1057
|
+
return {
|
|
1058
|
+
interval,
|
|
1059
|
+
priority: intervalPriority(interval),
|
|
1060
|
+
maxEnd: interval.end,
|
|
1061
|
+
left: null,
|
|
1062
|
+
right: null,
|
|
1063
|
+
};
|
|
1064
|
+
}
|
|
1065
|
+
if (compareIntervals(interval, node.interval) < 0) {
|
|
1066
|
+
node.left = insertInterval(node.left, interval);
|
|
1067
|
+
if (node.left.priority > node.priority)
|
|
1068
|
+
node = rotateIntervalRight(node);
|
|
1069
|
+
}
|
|
1070
|
+
else {
|
|
1071
|
+
node.right = insertInterval(node.right, interval);
|
|
1072
|
+
if (node.right.priority > node.priority)
|
|
1073
|
+
node = rotateIntervalLeft(node);
|
|
1074
|
+
}
|
|
1075
|
+
refreshMaxEnd(node);
|
|
1076
|
+
return node;
|
|
1077
|
+
}
|
|
1078
|
+
function rotateIntervalLeft(node) {
|
|
1079
|
+
const pivot = node.right;
|
|
1080
|
+
if (!pivot)
|
|
1081
|
+
return node;
|
|
1082
|
+
node.right = pivot.left;
|
|
1083
|
+
pivot.left = node;
|
|
1084
|
+
refreshMaxEnd(node);
|
|
1085
|
+
refreshMaxEnd(pivot);
|
|
1086
|
+
return pivot;
|
|
1087
|
+
}
|
|
1088
|
+
function rotateIntervalRight(node) {
|
|
1089
|
+
const pivot = node.left;
|
|
1090
|
+
if (!pivot)
|
|
1091
|
+
return node;
|
|
1092
|
+
node.left = pivot.right;
|
|
1093
|
+
pivot.right = node;
|
|
1094
|
+
refreshMaxEnd(node);
|
|
1095
|
+
refreshMaxEnd(pivot);
|
|
1096
|
+
return pivot;
|
|
1097
|
+
}
|
|
1098
|
+
function refreshMaxEnd(node) {
|
|
1099
|
+
node.maxEnd = Math.max(node.interval.end, node.left?.maxEnd ?? Number.NEGATIVE_INFINITY, node.right?.maxEnd ?? Number.NEGATIVE_INFINITY);
|
|
1100
|
+
}
|
|
1101
|
+
function compareIntervals(left, right) {
|
|
1102
|
+
return (left.start - right.start
|
|
1103
|
+
|| left.end - right.end
|
|
1104
|
+
|| left.segmentId.localeCompare(right.segmentId));
|
|
1105
|
+
}
|
|
1106
|
+
function intervalPriority(interval) {
|
|
1107
|
+
let hash = 2_166_136_261;
|
|
1108
|
+
const key = `${interval.segmentId}:${interval.start}:${interval.end}`;
|
|
1109
|
+
for (let index = 0; index < key.length; index += 1) {
|
|
1110
|
+
hash ^= key.charCodeAt(index);
|
|
1111
|
+
hash = Math.imul(hash, 16_777_619);
|
|
1112
|
+
}
|
|
1113
|
+
return hash >>> 0;
|
|
1114
|
+
}
|
|
1115
|
+
function candidateToHit(candidate) {
|
|
1116
|
+
return {
|
|
1117
|
+
artifactId: candidate.segment.artifact_id,
|
|
1118
|
+
...(candidate.segment.artifact_label
|
|
1119
|
+
? { artifactLabel: candidate.segment.artifact_label }
|
|
1120
|
+
: {}),
|
|
1121
|
+
artifactKind: candidate.segment.artifact_kind,
|
|
1122
|
+
location: segmentLocation(candidate.segment),
|
|
1123
|
+
...(candidate.segment.representative_tick === null
|
|
1124
|
+
? {}
|
|
1125
|
+
: { representativeTick: candidate.segment.representative_tick }),
|
|
1126
|
+
score: candidate.score,
|
|
1127
|
+
signals: candidate.signals,
|
|
1128
|
+
...(candidate.excerpt ? { excerpt: candidate.excerpt } : {}),
|
|
1129
|
+
indexManifestIds: [candidate.segment.manifest_id],
|
|
1130
|
+
};
|
|
1131
|
+
}
|
|
1132
|
+
function segmentLocation(segment) {
|
|
1133
|
+
const range = segmentRange(segment);
|
|
1134
|
+
if (range) {
|
|
1135
|
+
return {
|
|
1136
|
+
kind: "timed",
|
|
1137
|
+
artifactId: segment.artifact_id,
|
|
1138
|
+
range,
|
|
1139
|
+
};
|
|
1140
|
+
}
|
|
1141
|
+
return {
|
|
1142
|
+
kind: segment.segment_kind === "document" ? "document" : "still",
|
|
1143
|
+
artifactId: segment.artifact_id,
|
|
1144
|
+
sourcePath: segment.source_path ?? "unknown",
|
|
1145
|
+
objectHash: segment.object_hash,
|
|
1146
|
+
...(segment.segment_kind === "document"
|
|
1147
|
+
? { startUtf8Byte: 0, endUtf8Byte: 1 }
|
|
1148
|
+
: {}),
|
|
1149
|
+
};
|
|
1150
|
+
}
|
|
1151
|
+
function segmentRange(segment) {
|
|
1152
|
+
return segment.source_range_json
|
|
1153
|
+
? parseJson(segment.source_range_json, null)
|
|
1154
|
+
: null;
|
|
1155
|
+
}
|
|
1156
|
+
function locationKey(segment) {
|
|
1157
|
+
return canonicalJson(segmentLocation(segment));
|
|
1158
|
+
}
|
|
1159
|
+
function searchCoverage(context, artifactReference) {
|
|
1160
|
+
const artifactId = artifactReference
|
|
1161
|
+
? context.artifactRow(artifactReference).artifact_id
|
|
1162
|
+
: undefined;
|
|
1163
|
+
const active = activeGenerations(context);
|
|
1164
|
+
const rows = coverageRows(context, artifactId
|
|
1165
|
+
? `WHERE artifact_id=? AND generation IN (${active.map(() => "?").join(",")})`
|
|
1166
|
+
: active.length > 0
|
|
1167
|
+
? `WHERE generation IN (${active.map(() => "?").join(",")})`
|
|
1168
|
+
: "WHERE 1=0", artifactId
|
|
1169
|
+
? [artifactId, ...active.map((item) => item.generation)]
|
|
1170
|
+
: active.map((item) => item.generation));
|
|
1171
|
+
const allArtifacts = context.store.db
|
|
1172
|
+
.prepare("SELECT COUNT(*) AS count FROM artifacts")
|
|
1173
|
+
.get();
|
|
1174
|
+
const indexedArtifactCount = new Set(rows.map((row) => row.artifactId)).size;
|
|
1175
|
+
const state = aggregateCoverageState(rows);
|
|
1176
|
+
const generation = hashJson(active.map((item) => [item.embedding_space, item.generation]));
|
|
1177
|
+
const modalities = unique(active.flatMap((item) => requiredManifest(context, item.manifest_id).modalities)).map((modality) => {
|
|
1178
|
+
const relevant = rows.filter((row) => phaseForModality(modality) === row.phase);
|
|
1179
|
+
return {
|
|
1180
|
+
modality,
|
|
1181
|
+
state: aggregateCoverageState(relevant),
|
|
1182
|
+
indexedUnits: relevant.reduce((sum, row) => sum + row.indexedUnits, 0),
|
|
1183
|
+
...(relevant.some((row) => row.totalUnits !== undefined)
|
|
1184
|
+
? {
|
|
1185
|
+
totalUnits: relevant.reduce((sum, row) => sum + (row.totalUnits ?? 0), 0),
|
|
1186
|
+
}
|
|
1187
|
+
: {}),
|
|
1188
|
+
languageCoverage: modality === "speech" || modality === "ocr"
|
|
1189
|
+
? "best-effort"
|
|
1190
|
+
: "unsupported",
|
|
1191
|
+
manifestId: active.find((item) => requiredManifest(context, item.manifest_id).modalities.includes(modality))?.manifest_id,
|
|
1192
|
+
};
|
|
1193
|
+
});
|
|
1194
|
+
return {
|
|
1195
|
+
state,
|
|
1196
|
+
generation,
|
|
1197
|
+
modalities,
|
|
1198
|
+
indexedArtifactCount,
|
|
1199
|
+
totalArtifactCount: artifactId ? 1 : allArtifacts.count,
|
|
1200
|
+
partialResults: state !== "ready",
|
|
1201
|
+
};
|
|
1202
|
+
}
|
|
1203
|
+
function coverageRows(context, clause, params) {
|
|
1204
|
+
const rows = context.store.db
|
|
1205
|
+
.prepare(`SELECT artifact_id, object_hash, manifest_id, generation, phase,
|
|
1206
|
+
state, covered_ranges_json, indexed_units, total_units,
|
|
1207
|
+
retryable, error_json, cursor, updated_at
|
|
1208
|
+
FROM runtime_index_coverage ${clause}
|
|
1209
|
+
ORDER BY artifact_id, manifest_id, generation, phase`)
|
|
1210
|
+
.all(...params);
|
|
1211
|
+
return rows.map((row) => ({
|
|
1212
|
+
artifactId: row.artifact_id,
|
|
1213
|
+
objectHash: row.object_hash,
|
|
1214
|
+
manifestId: row.manifest_id,
|
|
1215
|
+
phase: row.phase,
|
|
1216
|
+
state: row.state,
|
|
1217
|
+
coveredRanges: parseJson(row.covered_ranges_json, []),
|
|
1218
|
+
indexedUnits: row.indexed_units,
|
|
1219
|
+
...(row.total_units === null ? {} : { totalUnits: row.total_units }),
|
|
1220
|
+
...(row.cursor === null ? {} : { nextCursor: row.cursor }),
|
|
1221
|
+
retryable: row.retryable === 1,
|
|
1222
|
+
...(row.error_json
|
|
1223
|
+
? { error: parseJson(row.error_json, {
|
|
1224
|
+
code: "INTERNAL_ERROR",
|
|
1225
|
+
message: "Unknown index error",
|
|
1226
|
+
}) }
|
|
1227
|
+
: {}),
|
|
1228
|
+
updatedAt: row.updated_at,
|
|
1229
|
+
}));
|
|
1230
|
+
}
|
|
1231
|
+
function invalidateCoverage(context, artifactReference, objectHash) {
|
|
1232
|
+
const artifact = context.artifactRow(artifactReference);
|
|
1233
|
+
return context.store.runtime((now) => {
|
|
1234
|
+
const result = objectHash
|
|
1235
|
+
? context.store.db
|
|
1236
|
+
.prepare(`UPDATE runtime_index_coverage
|
|
1237
|
+
SET state='stale', updated_at=?
|
|
1238
|
+
WHERE artifact_id=? AND object_hash=?`)
|
|
1239
|
+
.run(now, artifact.artifact_id, objectHash)
|
|
1240
|
+
: context.store.db
|
|
1241
|
+
.prepare(`UPDATE runtime_index_coverage
|
|
1242
|
+
SET state='stale', updated_at=?
|
|
1243
|
+
WHERE artifact_id=?`)
|
|
1244
|
+
.run(now, artifact.artifact_id);
|
|
1245
|
+
return Number(result.changes);
|
|
1246
|
+
});
|
|
1247
|
+
}
|
|
1248
|
+
function cleanupRetired(context) {
|
|
1249
|
+
return context.store.runtime(() => {
|
|
1250
|
+
const retired = context.store.db
|
|
1251
|
+
.prepare(`SELECT generation FROM runtime_index_generations
|
|
1252
|
+
WHERE state='retired'`)
|
|
1253
|
+
.all().map((row) => row.generation);
|
|
1254
|
+
if (retired.length === 0)
|
|
1255
|
+
return { removedSegments: 0 };
|
|
1256
|
+
const placeholders = retired.map(() => "?").join(",");
|
|
1257
|
+
const result = context.store.db
|
|
1258
|
+
.prepare(`DELETE FROM runtime_media_segments
|
|
1259
|
+
WHERE generation IN (${placeholders})`)
|
|
1260
|
+
.run(...retired);
|
|
1261
|
+
context.store.db
|
|
1262
|
+
.prepare(`DELETE FROM runtime_index_coverage
|
|
1263
|
+
WHERE generation IN (${placeholders})`)
|
|
1264
|
+
.run(...retired);
|
|
1265
|
+
context.store.db
|
|
1266
|
+
.prepare(`DELETE FROM runtime_index_batches
|
|
1267
|
+
WHERE generation IN (${placeholders})`)
|
|
1268
|
+
.run(...retired);
|
|
1269
|
+
context.store.db
|
|
1270
|
+
.prepare(`DELETE FROM runtime_index_generations
|
|
1271
|
+
WHERE generation IN (${placeholders})`)
|
|
1272
|
+
.run(...retired);
|
|
1273
|
+
return { removedSegments: Number(result.changes) };
|
|
1274
|
+
});
|
|
1275
|
+
}
|
|
1276
|
+
function temporalStats(context) {
|
|
1277
|
+
const count = (table, clause = "") => context.store.db
|
|
1278
|
+
.prepare(`SELECT COUNT(*) AS count FROM ${table} ${clause}`)
|
|
1279
|
+
.get().count;
|
|
1280
|
+
return {
|
|
1281
|
+
activeGenerations: count("runtime_index_generations", "WHERE state='active'"),
|
|
1282
|
+
segments: count("runtime_media_segments"),
|
|
1283
|
+
textObservations: count("runtime_segment_text"),
|
|
1284
|
+
embeddings: count("runtime_segment_embeddings"),
|
|
1285
|
+
fingerprints: count("runtime_segment_fingerprints"),
|
|
1286
|
+
};
|
|
1287
|
+
}
|
|
1288
|
+
function assertArtifactObject(context, artifactId, objectHash) {
|
|
1289
|
+
const object = context.store.db
|
|
1290
|
+
.prepare(`SELECT 1 AS present FROM artifact_files
|
|
1291
|
+
WHERE artifact_id=? AND object_hash=? LIMIT 1`)
|
|
1292
|
+
.get(artifactId, objectHash);
|
|
1293
|
+
if (!object) {
|
|
1294
|
+
throw new EngineFault({
|
|
1295
|
+
code: "OBJECT_UNAVAILABLE",
|
|
1296
|
+
message: "Artifact does not currently reference the requested object",
|
|
1297
|
+
details: { artifactId, objectHash },
|
|
1298
|
+
});
|
|
1299
|
+
}
|
|
1300
|
+
}
|
|
1301
|
+
function validateQuery(query, prepared) {
|
|
1302
|
+
if (!query.text && !query.reference && !prepared) {
|
|
1303
|
+
throw new EngineFault({
|
|
1304
|
+
code: "INVALID_INPUT",
|
|
1305
|
+
message: "Search requires text, reference media, or both",
|
|
1306
|
+
});
|
|
1307
|
+
}
|
|
1308
|
+
if (query.reference && prepared) {
|
|
1309
|
+
throw new EngineFault({
|
|
1310
|
+
code: "INVALID_INPUT",
|
|
1311
|
+
message: "Search cannot combine artifact and prepared references",
|
|
1312
|
+
});
|
|
1313
|
+
}
|
|
1314
|
+
if (query.text !== undefined)
|
|
1315
|
+
requiredText(query.text, "Search text");
|
|
1316
|
+
if (query.limit !== undefined)
|
|
1317
|
+
safeIntegerAtLeast(query.limit, 1, "Search limit");
|
|
1318
|
+
if (query.minScore !== undefined && !Number.isFinite(query.minScore)) {
|
|
1319
|
+
throw new Error("Search minScore must be finite");
|
|
1320
|
+
}
|
|
1321
|
+
}
|
|
1322
|
+
function validatePreparedReference(reference, options) {
|
|
1323
|
+
requiredText(reference.embeddingSpace, "Prepared reference embedding space");
|
|
1324
|
+
if (reference.kind === "image") {
|
|
1325
|
+
preparedVector(reference.vector);
|
|
1326
|
+
if (options.range) {
|
|
1327
|
+
throw new EngineFault({
|
|
1328
|
+
code: "INVALID_INPUT",
|
|
1329
|
+
message: "Image references do not accept a time range",
|
|
1330
|
+
});
|
|
1331
|
+
}
|
|
1332
|
+
for (const fingerprint of reference.fingerprints ?? []) {
|
|
1333
|
+
requiredText(fingerprint.kind, "Prepared fingerprint kind");
|
|
1334
|
+
requiredText(fingerprint.value, "Prepared fingerprint value");
|
|
1335
|
+
}
|
|
1336
|
+
return;
|
|
1337
|
+
}
|
|
1338
|
+
if (reference.samples.length === 0 || reference.samples.length > 16) {
|
|
1339
|
+
throw new EngineFault({
|
|
1340
|
+
code: "INVALID_INPUT",
|
|
1341
|
+
message: "Prepared video references require between 1 and 16 samples",
|
|
1342
|
+
});
|
|
1343
|
+
}
|
|
1344
|
+
let previousOffset = -1;
|
|
1345
|
+
for (const sample of reference.samples) {
|
|
1346
|
+
if (!Number.isFinite(sample.offsetMs) || sample.offsetMs < 0) {
|
|
1347
|
+
throw new EngineFault({
|
|
1348
|
+
code: "INVALID_INPUT",
|
|
1349
|
+
message: "Prepared video sample offsets must be finite and non-negative",
|
|
1350
|
+
});
|
|
1351
|
+
}
|
|
1352
|
+
if (sample.offsetMs <= previousOffset) {
|
|
1353
|
+
throw new EngineFault({
|
|
1354
|
+
code: "INVALID_INPUT",
|
|
1355
|
+
message: "Prepared video sample offsets must be strictly increasing",
|
|
1356
|
+
});
|
|
1357
|
+
}
|
|
1358
|
+
previousOffset = sample.offsetMs;
|
|
1359
|
+
preparedVector(sample.vector);
|
|
1360
|
+
}
|
|
1361
|
+
const range = options.range;
|
|
1362
|
+
if (range) {
|
|
1363
|
+
if (!Number.isFinite(range.startMs) || range.startMs < 0) {
|
|
1364
|
+
throw new EngineFault({
|
|
1365
|
+
code: "INVALID_RANGE",
|
|
1366
|
+
message: "Prepared video range start must be finite and non-negative",
|
|
1367
|
+
});
|
|
1368
|
+
}
|
|
1369
|
+
if (range.durationMs !== undefined
|
|
1370
|
+
&& (!Number.isFinite(range.durationMs) || range.durationMs <= 0)) {
|
|
1371
|
+
throw new EngineFault({
|
|
1372
|
+
code: "INVALID_RANGE",
|
|
1373
|
+
message: "Prepared video range duration must be finite and positive",
|
|
1374
|
+
});
|
|
1375
|
+
}
|
|
1376
|
+
}
|
|
1377
|
+
preparedVideoSamples(reference, options);
|
|
1378
|
+
}
|
|
1379
|
+
function preparedVideoSamples(reference, options) {
|
|
1380
|
+
const range = options.range;
|
|
1381
|
+
const endMs = range?.durationMs === undefined
|
|
1382
|
+
? Number.POSITIVE_INFINITY
|
|
1383
|
+
: range.startMs + range.durationMs;
|
|
1384
|
+
const samples = range
|
|
1385
|
+
? reference.samples.filter((sample) => sample.offsetMs >= range.startMs && sample.offsetMs <= endMs)
|
|
1386
|
+
: reference.samples;
|
|
1387
|
+
if (samples.length === 0) {
|
|
1388
|
+
throw new EngineFault({
|
|
1389
|
+
code: "INVALID_RANGE",
|
|
1390
|
+
message: "Prepared video range does not contain a sampled frame",
|
|
1391
|
+
});
|
|
1392
|
+
}
|
|
1393
|
+
return samples;
|
|
1394
|
+
}
|
|
1395
|
+
function preparedVector(vector, dimensions) {
|
|
1396
|
+
if (!Array.isArray(vector)
|
|
1397
|
+
|| vector.length === 0
|
|
1398
|
+
|| vector.some((value) => !Number.isFinite(value))) {
|
|
1399
|
+
throw new EngineFault({
|
|
1400
|
+
code: "INVALID_INPUT",
|
|
1401
|
+
message: "Prepared reference vectors must contain finite values",
|
|
1402
|
+
});
|
|
1403
|
+
}
|
|
1404
|
+
if (dimensions !== undefined && vector.length !== dimensions) {
|
|
1405
|
+
throw new EngineFault({
|
|
1406
|
+
code: "MANIFEST_INCOMPATIBLE",
|
|
1407
|
+
message: "Prepared reference vector dimensions do not match the index",
|
|
1408
|
+
});
|
|
1409
|
+
}
|
|
1410
|
+
return Float32Array.from(vector);
|
|
1411
|
+
}
|
|
1412
|
+
function aggregateCoverageState(rows) {
|
|
1413
|
+
if (rows.length === 0)
|
|
1414
|
+
return "not-indexed";
|
|
1415
|
+
if (rows.some((row) => row.state === "failed"))
|
|
1416
|
+
return "failed";
|
|
1417
|
+
if (rows.some((row) => row.state === "stale"))
|
|
1418
|
+
return "stale";
|
|
1419
|
+
if (rows.every((row) => row.state === "ready"))
|
|
1420
|
+
return "ready";
|
|
1421
|
+
return "partial";
|
|
1422
|
+
}
|
|
1423
|
+
function phaseForModality(modality) {
|
|
1424
|
+
if (modality === "visual")
|
|
1425
|
+
return "visual";
|
|
1426
|
+
if (modality === "audio")
|
|
1427
|
+
return "audio";
|
|
1428
|
+
if (modality === "ocr")
|
|
1429
|
+
return "ocr";
|
|
1430
|
+
if (modality === "speech")
|
|
1431
|
+
return "transcript";
|
|
1432
|
+
return "lexical";
|
|
1433
|
+
}
|
|
1434
|
+
function signalPriority(kind) {
|
|
1435
|
+
return ["exact", "near", "speech", "ocr", "visual", "audio", "metadata"].indexOf(kind);
|
|
1436
|
+
}
|
|
1437
|
+
function vectorToBlob(vector) {
|
|
1438
|
+
const values = Float32Array.from(vector);
|
|
1439
|
+
return Buffer.from(values.buffer, values.byteOffset, values.byteLength);
|
|
1440
|
+
}
|
|
1441
|
+
function blobToVector(blob, dimensions) {
|
|
1442
|
+
const bytes = Buffer.from(blob);
|
|
1443
|
+
if (bytes.byteLength !== dimensions * Float32Array.BYTES_PER_ELEMENT) {
|
|
1444
|
+
throw new EngineFault({
|
|
1445
|
+
code: "MANIFEST_INCOMPATIBLE",
|
|
1446
|
+
message: "Stored vector dimensions do not match its byte length",
|
|
1447
|
+
});
|
|
1448
|
+
}
|
|
1449
|
+
const copy = new Uint8Array(bytes);
|
|
1450
|
+
return new Float32Array(copy.buffer);
|
|
1451
|
+
}
|
|
1452
|
+
function cosineSimilarity(left, right) {
|
|
1453
|
+
if (left.length !== right.length || left.length === 0) {
|
|
1454
|
+
throw new EngineFault({
|
|
1455
|
+
code: "MANIFEST_INCOMPATIBLE",
|
|
1456
|
+
message: "Cannot compare incompatible vector dimensions",
|
|
1457
|
+
});
|
|
1458
|
+
}
|
|
1459
|
+
let dot = 0;
|
|
1460
|
+
let leftNorm = 0;
|
|
1461
|
+
let rightNorm = 0;
|
|
1462
|
+
for (let index = 0; index < left.length; index += 1) {
|
|
1463
|
+
const leftValue = left[index];
|
|
1464
|
+
const rightValue = right[index];
|
|
1465
|
+
dot += leftValue * rightValue;
|
|
1466
|
+
leftNorm += leftValue * leftValue;
|
|
1467
|
+
rightNorm += rightValue * rightValue;
|
|
1468
|
+
}
|
|
1469
|
+
if (leftNorm === 0 || rightNorm === 0)
|
|
1470
|
+
return 0;
|
|
1471
|
+
return dot / (Math.sqrt(leftNorm) * Math.sqrt(rightNorm));
|
|
1472
|
+
}
|
|
1473
|
+
function averageVectors(vectors) {
|
|
1474
|
+
const first = vectors[0];
|
|
1475
|
+
if (!first)
|
|
1476
|
+
return new Float32Array();
|
|
1477
|
+
const result = new Float32Array(first.length);
|
|
1478
|
+
for (const vector of vectors) {
|
|
1479
|
+
if (vector.length !== result.length) {
|
|
1480
|
+
throw new EngineFault({
|
|
1481
|
+
code: "MANIFEST_INCOMPATIBLE",
|
|
1482
|
+
message: "Reference vectors have incompatible dimensions",
|
|
1483
|
+
});
|
|
1484
|
+
}
|
|
1485
|
+
vector.forEach((value, index) => {
|
|
1486
|
+
result[index] = result[index] + value;
|
|
1487
|
+
});
|
|
1488
|
+
}
|
|
1489
|
+
for (let index = 0; index < result.length; index += 1) {
|
|
1490
|
+
result[index] = result[index] / vectors.length;
|
|
1491
|
+
}
|
|
1492
|
+
return result;
|
|
1493
|
+
}
|
|
1494
|
+
function encodeCursor(generation, offset) {
|
|
1495
|
+
return Buffer.from(canonicalJson({ generation, offset }), "utf8").toString("base64url");
|
|
1496
|
+
}
|
|
1497
|
+
function decodeCursor(cursor, generation) {
|
|
1498
|
+
if (!cursor)
|
|
1499
|
+
return 0;
|
|
1500
|
+
const parsed = parseJson(Buffer.from(cursor, "base64url").toString("utf8"), {});
|
|
1501
|
+
if (parsed.generation !== generation
|
|
1502
|
+
|| !Number.isSafeInteger(parsed.offset)
|
|
1503
|
+
|| (parsed.offset ?? -1) < 0) {
|
|
1504
|
+
throw new EngineFault({
|
|
1505
|
+
code: "STALE_REVISION",
|
|
1506
|
+
message: "Search cursor belongs to another index generation",
|
|
1507
|
+
});
|
|
1508
|
+
}
|
|
1509
|
+
return parsed.offset;
|
|
1510
|
+
}
|
|
1511
|
+
function hashJson(value) {
|
|
1512
|
+
return `sha256:${createHash("sha256")
|
|
1513
|
+
.update(canonicalJson(value))
|
|
1514
|
+
.digest("hex")}`;
|
|
1515
|
+
}
|
|
1516
|
+
function deterministicUuid(namespace, value) {
|
|
1517
|
+
const bytes = createHash("sha256")
|
|
1518
|
+
.update(namespace)
|
|
1519
|
+
.update("\0")
|
|
1520
|
+
.update(value)
|
|
1521
|
+
.digest()
|
|
1522
|
+
.subarray(0, 16);
|
|
1523
|
+
bytes[6] = (bytes[6] & 0x0f) | 0x70;
|
|
1524
|
+
bytes[8] = (bytes[8] & 0x3f) | 0x80;
|
|
1525
|
+
const hex = bytes.toString("hex");
|
|
1526
|
+
return [
|
|
1527
|
+
hex.slice(0, 8),
|
|
1528
|
+
hex.slice(8, 12),
|
|
1529
|
+
hex.slice(12, 16),
|
|
1530
|
+
hex.slice(16, 20),
|
|
1531
|
+
hex.slice(20),
|
|
1532
|
+
].join("-");
|
|
1533
|
+
}
|
|
1534
|
+
function requiredText(value, label) {
|
|
1535
|
+
const normalized = value.trim();
|
|
1536
|
+
if (!normalized)
|
|
1537
|
+
throw new Error(`${label} is required`);
|
|
1538
|
+
return normalized;
|
|
1539
|
+
}
|
|
1540
|
+
function safeIntegerAtLeast(value, minimum, label) {
|
|
1541
|
+
if (!Number.isSafeInteger(value) || value < minimum) {
|
|
1542
|
+
throw new Error(`${label} must be a safe integer of at least ${minimum}`);
|
|
1543
|
+
}
|
|
1544
|
+
}
|
|
1545
|
+
function rangesOverlap(leftStart, leftEnd, rightStart, rightEnd) {
|
|
1546
|
+
return leftStart < rightEnd && rightStart < leftEnd;
|
|
1547
|
+
}
|
|
1548
|
+
function unique(values) {
|
|
1549
|
+
return [...new Set(values)];
|
|
1550
|
+
}
|
|
1551
|
+
//# sourceMappingURL=temporal-search.js.map
|