remdb 0.2.6__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of remdb might be problematic. Click here for more details.
- rem/__init__.py +2 -0
- rem/agentic/README.md +650 -0
- rem/agentic/__init__.py +39 -0
- rem/agentic/agents/README.md +155 -0
- rem/agentic/agents/__init__.py +8 -0
- rem/agentic/context.py +148 -0
- rem/agentic/context_builder.py +329 -0
- rem/agentic/mcp/__init__.py +0 -0
- rem/agentic/mcp/tool_wrapper.py +107 -0
- rem/agentic/otel/__init__.py +5 -0
- rem/agentic/otel/setup.py +151 -0
- rem/agentic/providers/phoenix.py +674 -0
- rem/agentic/providers/pydantic_ai.py +572 -0
- rem/agentic/query.py +117 -0
- rem/agentic/query_helper.py +89 -0
- rem/agentic/schema.py +396 -0
- rem/agentic/serialization.py +245 -0
- rem/agentic/tools/__init__.py +5 -0
- rem/agentic/tools/rem_tools.py +231 -0
- rem/api/README.md +420 -0
- rem/api/main.py +324 -0
- rem/api/mcp_router/prompts.py +182 -0
- rem/api/mcp_router/resources.py +536 -0
- rem/api/mcp_router/server.py +213 -0
- rem/api/mcp_router/tools.py +584 -0
- rem/api/routers/auth.py +229 -0
- rem/api/routers/chat/__init__.py +5 -0
- rem/api/routers/chat/completions.py +281 -0
- rem/api/routers/chat/json_utils.py +76 -0
- rem/api/routers/chat/models.py +124 -0
- rem/api/routers/chat/streaming.py +185 -0
- rem/auth/README.md +258 -0
- rem/auth/__init__.py +26 -0
- rem/auth/middleware.py +100 -0
- rem/auth/providers/__init__.py +13 -0
- rem/auth/providers/base.py +376 -0
- rem/auth/providers/google.py +163 -0
- rem/auth/providers/microsoft.py +237 -0
- rem/cli/README.md +455 -0
- rem/cli/__init__.py +8 -0
- rem/cli/commands/README.md +126 -0
- rem/cli/commands/__init__.py +3 -0
- rem/cli/commands/ask.py +565 -0
- rem/cli/commands/configure.py +423 -0
- rem/cli/commands/db.py +493 -0
- rem/cli/commands/dreaming.py +324 -0
- rem/cli/commands/experiments.py +1124 -0
- rem/cli/commands/mcp.py +66 -0
- rem/cli/commands/process.py +245 -0
- rem/cli/commands/schema.py +183 -0
- rem/cli/commands/serve.py +106 -0
- rem/cli/dreaming.py +363 -0
- rem/cli/main.py +88 -0
- rem/config.py +237 -0
- rem/mcp_server.py +41 -0
- rem/models/core/__init__.py +49 -0
- rem/models/core/core_model.py +64 -0
- rem/models/core/engram.py +333 -0
- rem/models/core/experiment.py +628 -0
- rem/models/core/inline_edge.py +132 -0
- rem/models/core/rem_query.py +243 -0
- rem/models/entities/__init__.py +43 -0
- rem/models/entities/file.py +57 -0
- rem/models/entities/image_resource.py +88 -0
- rem/models/entities/message.py +35 -0
- rem/models/entities/moment.py +123 -0
- rem/models/entities/ontology.py +191 -0
- rem/models/entities/ontology_config.py +131 -0
- rem/models/entities/resource.py +95 -0
- rem/models/entities/schema.py +87 -0
- rem/models/entities/user.py +85 -0
- rem/py.typed +0 -0
- rem/schemas/README.md +507 -0
- rem/schemas/__init__.py +6 -0
- rem/schemas/agents/README.md +92 -0
- rem/schemas/agents/core/moment-builder.yaml +178 -0
- rem/schemas/agents/core/rem-query-agent.yaml +226 -0
- rem/schemas/agents/core/resource-affinity-assessor.yaml +99 -0
- rem/schemas/agents/core/simple-assistant.yaml +19 -0
- rem/schemas/agents/core/user-profile-builder.yaml +163 -0
- rem/schemas/agents/examples/contract-analyzer.yaml +317 -0
- rem/schemas/agents/examples/contract-extractor.yaml +134 -0
- rem/schemas/agents/examples/cv-parser.yaml +263 -0
- rem/schemas/agents/examples/hello-world.yaml +37 -0
- rem/schemas/agents/examples/query.yaml +54 -0
- rem/schemas/agents/examples/simple.yaml +21 -0
- rem/schemas/agents/examples/test.yaml +29 -0
- rem/schemas/agents/rem.yaml +128 -0
- rem/schemas/evaluators/hello-world/default.yaml +77 -0
- rem/schemas/evaluators/rem/faithfulness.yaml +219 -0
- rem/schemas/evaluators/rem/lookup-correctness.yaml +182 -0
- rem/schemas/evaluators/rem/retrieval-precision.yaml +199 -0
- rem/schemas/evaluators/rem/retrieval-recall.yaml +211 -0
- rem/schemas/evaluators/rem/search-correctness.yaml +192 -0
- rem/services/__init__.py +16 -0
- rem/services/audio/INTEGRATION.md +308 -0
- rem/services/audio/README.md +376 -0
- rem/services/audio/__init__.py +15 -0
- rem/services/audio/chunker.py +354 -0
- rem/services/audio/transcriber.py +259 -0
- rem/services/content/README.md +1269 -0
- rem/services/content/__init__.py +5 -0
- rem/services/content/providers.py +806 -0
- rem/services/content/service.py +657 -0
- rem/services/dreaming/README.md +230 -0
- rem/services/dreaming/__init__.py +53 -0
- rem/services/dreaming/affinity_service.py +336 -0
- rem/services/dreaming/moment_service.py +264 -0
- rem/services/dreaming/ontology_service.py +54 -0
- rem/services/dreaming/user_model_service.py +297 -0
- rem/services/dreaming/utils.py +39 -0
- rem/services/embeddings/__init__.py +11 -0
- rem/services/embeddings/api.py +120 -0
- rem/services/embeddings/worker.py +421 -0
- rem/services/fs/README.md +662 -0
- rem/services/fs/__init__.py +62 -0
- rem/services/fs/examples.py +206 -0
- rem/services/fs/examples_paths.py +204 -0
- rem/services/fs/git_provider.py +935 -0
- rem/services/fs/local_provider.py +760 -0
- rem/services/fs/parsing-hooks-examples.md +172 -0
- rem/services/fs/paths.py +276 -0
- rem/services/fs/provider.py +460 -0
- rem/services/fs/s3_provider.py +1042 -0
- rem/services/fs/service.py +186 -0
- rem/services/git/README.md +1075 -0
- rem/services/git/__init__.py +17 -0
- rem/services/git/service.py +469 -0
- rem/services/phoenix/EXPERIMENT_DESIGN.md +1146 -0
- rem/services/phoenix/README.md +453 -0
- rem/services/phoenix/__init__.py +46 -0
- rem/services/phoenix/client.py +686 -0
- rem/services/phoenix/config.py +88 -0
- rem/services/phoenix/prompt_labels.py +477 -0
- rem/services/postgres/README.md +575 -0
- rem/services/postgres/__init__.py +23 -0
- rem/services/postgres/migration_service.py +427 -0
- rem/services/postgres/pydantic_to_sqlalchemy.py +232 -0
- rem/services/postgres/register_type.py +352 -0
- rem/services/postgres/repository.py +337 -0
- rem/services/postgres/schema_generator.py +379 -0
- rem/services/postgres/service.py +802 -0
- rem/services/postgres/sql_builder.py +354 -0
- rem/services/rem/README.md +304 -0
- rem/services/rem/__init__.py +23 -0
- rem/services/rem/exceptions.py +71 -0
- rem/services/rem/executor.py +293 -0
- rem/services/rem/parser.py +145 -0
- rem/services/rem/queries.py +196 -0
- rem/services/rem/query.py +371 -0
- rem/services/rem/service.py +527 -0
- rem/services/session/README.md +374 -0
- rem/services/session/__init__.py +6 -0
- rem/services/session/compression.py +360 -0
- rem/services/session/reload.py +77 -0
- rem/settings.py +1235 -0
- rem/sql/002_install_models.sql +1068 -0
- rem/sql/background_indexes.sql +42 -0
- rem/sql/install_models.sql +1038 -0
- rem/sql/migrations/001_install.sql +503 -0
- rem/sql/migrations/002_install_models.sql +1202 -0
- rem/utils/AGENTIC_CHUNKING.md +597 -0
- rem/utils/README.md +583 -0
- rem/utils/__init__.py +43 -0
- rem/utils/agentic_chunking.py +622 -0
- rem/utils/batch_ops.py +343 -0
- rem/utils/chunking.py +108 -0
- rem/utils/clip_embeddings.py +276 -0
- rem/utils/dict_utils.py +98 -0
- rem/utils/embeddings.py +423 -0
- rem/utils/examples/embeddings_example.py +305 -0
- rem/utils/examples/sql_types_example.py +202 -0
- rem/utils/markdown.py +16 -0
- rem/utils/model_helpers.py +236 -0
- rem/utils/schema_loader.py +229 -0
- rem/utils/sql_types.py +348 -0
- rem/utils/user_id.py +81 -0
- rem/utils/vision.py +330 -0
- rem/workers/README.md +506 -0
- rem/workers/__init__.py +5 -0
- rem/workers/dreaming.py +502 -0
- rem/workers/engram_processor.py +312 -0
- rem/workers/sqs_file_processor.py +193 -0
- remdb-0.2.6.dist-info/METADATA +1191 -0
- remdb-0.2.6.dist-info/RECORD +187 -0
- remdb-0.2.6.dist-info/WHEEL +4 -0
- remdb-0.2.6.dist-info/entry_points.txt +2 -0
|
@@ -0,0 +1,1202 @@
|
|
|
1
|
+
-- REM Model Schema (install_models.sql)
|
|
2
|
+
-- Generated from Pydantic models
|
|
3
|
+
-- Source directory: src/rem/models/entities
|
|
4
|
+
-- Generated at: 2025-11-21T22:20:29.773072
|
|
5
|
+
--
|
|
6
|
+
-- DO NOT EDIT MANUALLY - Regenerate with: rem schema generate
|
|
7
|
+
--
|
|
8
|
+
-- This script creates:
|
|
9
|
+
-- 1. Primary entity tables
|
|
10
|
+
-- 2. Embeddings tables (embeddings_<table>)
|
|
11
|
+
-- 3. KV_STORE triggers for cache maintenance
|
|
12
|
+
-- 4. Indexes (foreground only, background indexes separate)
|
|
13
|
+
|
|
14
|
+
-- ============================================================================
|
|
15
|
+
-- PREREQUISITES CHECK
|
|
16
|
+
-- ============================================================================
|
|
17
|
+
|
|
18
|
+
DO $$
|
|
19
|
+
BEGIN
|
|
20
|
+
-- Check that install.sql has been run
|
|
21
|
+
IF NOT EXISTS (SELECT 1 FROM pg_tables WHERE tablename = 'kv_store') THEN
|
|
22
|
+
RAISE EXCEPTION 'KV_STORE table not found. Run sql/install.sql first.';
|
|
23
|
+
END IF;
|
|
24
|
+
|
|
25
|
+
IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname = 'vector') THEN
|
|
26
|
+
RAISE EXCEPTION 'pgvector extension not found. Run sql/install.sql first.';
|
|
27
|
+
END IF;
|
|
28
|
+
|
|
29
|
+
RAISE NOTICE 'Prerequisites check passed';
|
|
30
|
+
END $$;
|
|
31
|
+
|
|
32
|
+
-- ======================================================================
|
|
33
|
+
-- USERS (Model: User)
|
|
34
|
+
-- ======================================================================
|
|
35
|
+
|
|
36
|
+
CREATE TABLE IF NOT EXISTS users (
|
|
37
|
+
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
|
38
|
+
tenant_id VARCHAR(100) NOT NULL,
|
|
39
|
+
user_id VARCHAR(256),
|
|
40
|
+
name VARCHAR(256) NOT NULL,
|
|
41
|
+
email VARCHAR(256),
|
|
42
|
+
role VARCHAR(256),
|
|
43
|
+
tier TEXT,
|
|
44
|
+
sec_policy JSONB DEFAULT '{}'::jsonb,
|
|
45
|
+
summary TEXT,
|
|
46
|
+
interests TEXT[] DEFAULT ARRAY[]::TEXT[],
|
|
47
|
+
preferred_topics TEXT[] DEFAULT ARRAY[]::TEXT[],
|
|
48
|
+
activity_level VARCHAR(256),
|
|
49
|
+
last_active_at TIMESTAMP,
|
|
50
|
+
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
51
|
+
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
52
|
+
deleted_at TIMESTAMP,
|
|
53
|
+
graph_edges JSONB DEFAULT '[]'::jsonb,
|
|
54
|
+
metadata JSONB DEFAULT '{}'::jsonb,
|
|
55
|
+
tags TEXT[] DEFAULT ARRAY[]::TEXT[]
|
|
56
|
+
);
|
|
57
|
+
|
|
58
|
+
CREATE INDEX IF NOT EXISTS idx_users_tenant ON users (tenant_id);
|
|
59
|
+
CREATE INDEX IF NOT EXISTS idx_users_user ON users (user_id);
|
|
60
|
+
CREATE INDEX IF NOT EXISTS idx_users_graph_edges ON users USING GIN (graph_edges);
|
|
61
|
+
CREATE INDEX IF NOT EXISTS idx_users_metadata ON users USING GIN (metadata);
|
|
62
|
+
CREATE INDEX IF NOT EXISTS idx_users_tags ON users USING GIN (tags);
|
|
63
|
+
|
|
64
|
+
-- Embeddings for users
|
|
65
|
+
CREATE TABLE IF NOT EXISTS embeddings_users (
|
|
66
|
+
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
|
67
|
+
entity_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
|
68
|
+
field_name VARCHAR(100) NOT NULL,
|
|
69
|
+
provider VARCHAR(50) NOT NULL DEFAULT 'openai',
|
|
70
|
+
model VARCHAR(100) NOT NULL DEFAULT 'text-embedding-3-small',
|
|
71
|
+
embedding vector(1536) NOT NULL,
|
|
72
|
+
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
73
|
+
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
74
|
+
|
|
75
|
+
-- Unique: one embedding per entity per field per provider
|
|
76
|
+
UNIQUE (entity_id, field_name, provider)
|
|
77
|
+
);
|
|
78
|
+
|
|
79
|
+
-- Index for entity lookup (get all embeddings for entity)
|
|
80
|
+
CREATE INDEX IF NOT EXISTS idx_embeddings_users_entity ON embeddings_users (entity_id);
|
|
81
|
+
|
|
82
|
+
-- Index for field + provider lookup
|
|
83
|
+
CREATE INDEX IF NOT EXISTS idx_embeddings_users_field_provider ON embeddings_users (field_name, provider);
|
|
84
|
+
|
|
85
|
+
-- HNSW index for vector similarity search (created in background)
|
|
86
|
+
-- Note: This will be created by background thread after data load
|
|
87
|
+
-- CREATE INDEX IF NOT EXISTS idx_embeddings_users_vector_hnsw ON embeddings_users
|
|
88
|
+
-- USING hnsw (embedding vector_cosine_ops);
|
|
89
|
+
|
|
90
|
+
-- KV_STORE trigger for users
|
|
91
|
+
-- Trigger function to maintain KV_STORE for users
|
|
92
|
+
CREATE OR REPLACE FUNCTION fn_users_kv_store_upsert()
|
|
93
|
+
RETURNS TRIGGER AS $$
|
|
94
|
+
BEGIN
|
|
95
|
+
IF (TG_OP = 'DELETE') THEN
|
|
96
|
+
-- Remove from KV_STORE on delete
|
|
97
|
+
DELETE FROM kv_store
|
|
98
|
+
WHERE entity_id = OLD.id;
|
|
99
|
+
RETURN OLD;
|
|
100
|
+
ELSIF (TG_OP = 'INSERT' OR TG_OP = 'UPDATE') THEN
|
|
101
|
+
-- Upsert to KV_STORE (O(1) lookup by entity_key)
|
|
102
|
+
INSERT INTO kv_store (
|
|
103
|
+
entity_key,
|
|
104
|
+
entity_type,
|
|
105
|
+
entity_id,
|
|
106
|
+
tenant_id,
|
|
107
|
+
user_id,
|
|
108
|
+
metadata,
|
|
109
|
+
graph_edges,
|
|
110
|
+
updated_at
|
|
111
|
+
) VALUES (
|
|
112
|
+
NEW.name, -- Use name as entity_key (json_schema_extra)
|
|
113
|
+
'users',
|
|
114
|
+
NEW.id,
|
|
115
|
+
NEW.tenant_id,
|
|
116
|
+
NEW.user_id,
|
|
117
|
+
NEW.metadata,
|
|
118
|
+
COALESCE(NEW.graph_edges, '[]'::jsonb),
|
|
119
|
+
CURRENT_TIMESTAMP
|
|
120
|
+
)
|
|
121
|
+
ON CONFLICT (tenant_id, entity_key)
|
|
122
|
+
DO UPDATE SET
|
|
123
|
+
entity_id = EXCLUDED.entity_id,
|
|
124
|
+
user_id = EXCLUDED.user_id,
|
|
125
|
+
metadata = EXCLUDED.metadata,
|
|
126
|
+
graph_edges = EXCLUDED.graph_edges,
|
|
127
|
+
updated_at = CURRENT_TIMESTAMP;
|
|
128
|
+
|
|
129
|
+
RETURN NEW;
|
|
130
|
+
END IF;
|
|
131
|
+
END;
|
|
132
|
+
$$ LANGUAGE plpgsql;
|
|
133
|
+
|
|
134
|
+
-- Create trigger
|
|
135
|
+
DROP TRIGGER IF EXISTS trg_users_kv_store ON users;
|
|
136
|
+
CREATE TRIGGER trg_users_kv_store
|
|
137
|
+
AFTER INSERT OR UPDATE OR DELETE ON users
|
|
138
|
+
FOR EACH ROW EXECUTE FUNCTION fn_users_kv_store_upsert();
|
|
139
|
+
|
|
140
|
+
-- ======================================================================
|
|
141
|
+
-- IMAGE_RESOURCES (Model: ImageResource)
|
|
142
|
+
-- ======================================================================
|
|
143
|
+
|
|
144
|
+
CREATE TABLE IF NOT EXISTS image_resources (
|
|
145
|
+
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
|
146
|
+
tenant_id VARCHAR(100) NOT NULL,
|
|
147
|
+
user_id VARCHAR(256),
|
|
148
|
+
name VARCHAR(256) NOT NULL,
|
|
149
|
+
uri VARCHAR(256),
|
|
150
|
+
ordinal INTEGER,
|
|
151
|
+
content TEXT,
|
|
152
|
+
timestamp TIMESTAMP,
|
|
153
|
+
category VARCHAR(256),
|
|
154
|
+
related_entities JSONB DEFAULT '{}'::jsonb,
|
|
155
|
+
image_width INTEGER,
|
|
156
|
+
image_height INTEGER,
|
|
157
|
+
image_format VARCHAR(256),
|
|
158
|
+
vision_description TEXT,
|
|
159
|
+
vision_provider VARCHAR(256),
|
|
160
|
+
vision_model VARCHAR(256),
|
|
161
|
+
clip_embedding JSONB,
|
|
162
|
+
clip_dimensions INTEGER,
|
|
163
|
+
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
164
|
+
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
165
|
+
deleted_at TIMESTAMP,
|
|
166
|
+
graph_edges JSONB DEFAULT '[]'::jsonb,
|
|
167
|
+
metadata JSONB DEFAULT '{}'::jsonb,
|
|
168
|
+
tags TEXT[] DEFAULT ARRAY[]::TEXT[]
|
|
169
|
+
);
|
|
170
|
+
|
|
171
|
+
CREATE INDEX IF NOT EXISTS idx_image_resources_tenant ON image_resources (tenant_id);
|
|
172
|
+
CREATE INDEX IF NOT EXISTS idx_image_resources_user ON image_resources (user_id);
|
|
173
|
+
CREATE INDEX IF NOT EXISTS idx_image_resources_graph_edges ON image_resources USING GIN (graph_edges);
|
|
174
|
+
CREATE INDEX IF NOT EXISTS idx_image_resources_metadata ON image_resources USING GIN (metadata);
|
|
175
|
+
CREATE INDEX IF NOT EXISTS idx_image_resources_tags ON image_resources USING GIN (tags);
|
|
176
|
+
|
|
177
|
+
-- Embeddings for image_resources
|
|
178
|
+
CREATE TABLE IF NOT EXISTS embeddings_image_resources (
|
|
179
|
+
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
|
180
|
+
entity_id UUID NOT NULL REFERENCES image_resources(id) ON DELETE CASCADE,
|
|
181
|
+
field_name VARCHAR(100) NOT NULL,
|
|
182
|
+
provider VARCHAR(50) NOT NULL DEFAULT 'openai',
|
|
183
|
+
model VARCHAR(100) NOT NULL DEFAULT 'text-embedding-3-small',
|
|
184
|
+
embedding vector(1536) NOT NULL,
|
|
185
|
+
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
186
|
+
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
187
|
+
|
|
188
|
+
-- Unique: one embedding per entity per field per provider
|
|
189
|
+
UNIQUE (entity_id, field_name, provider)
|
|
190
|
+
);
|
|
191
|
+
|
|
192
|
+
-- Index for entity lookup (get all embeddings for entity)
|
|
193
|
+
CREATE INDEX IF NOT EXISTS idx_embeddings_image_resources_entity ON embeddings_image_resources (entity_id);
|
|
194
|
+
|
|
195
|
+
-- Index for field + provider lookup
|
|
196
|
+
CREATE INDEX IF NOT EXISTS idx_embeddings_image_resources_field_provider ON embeddings_image_resources (field_name, provider);
|
|
197
|
+
|
|
198
|
+
-- HNSW index for vector similarity search (created in background)
|
|
199
|
+
-- Note: This will be created by background thread after data load
|
|
200
|
+
-- CREATE INDEX IF NOT EXISTS idx_embeddings_image_resources_vector_hnsw ON embeddings_image_resources
|
|
201
|
+
-- USING hnsw (embedding vector_cosine_ops);
|
|
202
|
+
|
|
203
|
+
-- KV_STORE trigger for image_resources
|
|
204
|
+
-- Trigger function to maintain KV_STORE for image_resources
|
|
205
|
+
CREATE OR REPLACE FUNCTION fn_image_resources_kv_store_upsert()
|
|
206
|
+
RETURNS TRIGGER AS $$
|
|
207
|
+
BEGIN
|
|
208
|
+
IF (TG_OP = 'DELETE') THEN
|
|
209
|
+
-- Remove from KV_STORE on delete
|
|
210
|
+
DELETE FROM kv_store
|
|
211
|
+
WHERE entity_id = OLD.id;
|
|
212
|
+
RETURN OLD;
|
|
213
|
+
ELSIF (TG_OP = 'INSERT' OR TG_OP = 'UPDATE') THEN
|
|
214
|
+
-- Upsert to KV_STORE (O(1) lookup by entity_key)
|
|
215
|
+
INSERT INTO kv_store (
|
|
216
|
+
entity_key,
|
|
217
|
+
entity_type,
|
|
218
|
+
entity_id,
|
|
219
|
+
tenant_id,
|
|
220
|
+
user_id,
|
|
221
|
+
metadata,
|
|
222
|
+
graph_edges,
|
|
223
|
+
updated_at
|
|
224
|
+
) VALUES (
|
|
225
|
+
NEW.name, -- Use name as entity_key (json_schema_extra)
|
|
226
|
+
'image_resources',
|
|
227
|
+
NEW.id,
|
|
228
|
+
NEW.tenant_id,
|
|
229
|
+
NEW.user_id,
|
|
230
|
+
NEW.metadata,
|
|
231
|
+
COALESCE(NEW.graph_edges, '[]'::jsonb),
|
|
232
|
+
CURRENT_TIMESTAMP
|
|
233
|
+
)
|
|
234
|
+
ON CONFLICT (tenant_id, entity_key)
|
|
235
|
+
DO UPDATE SET
|
|
236
|
+
entity_id = EXCLUDED.entity_id,
|
|
237
|
+
user_id = EXCLUDED.user_id,
|
|
238
|
+
metadata = EXCLUDED.metadata,
|
|
239
|
+
graph_edges = EXCLUDED.graph_edges,
|
|
240
|
+
updated_at = CURRENT_TIMESTAMP;
|
|
241
|
+
|
|
242
|
+
RETURN NEW;
|
|
243
|
+
END IF;
|
|
244
|
+
END;
|
|
245
|
+
$$ LANGUAGE plpgsql;
|
|
246
|
+
|
|
247
|
+
-- Create trigger
|
|
248
|
+
DROP TRIGGER IF EXISTS trg_image_resources_kv_store ON image_resources;
|
|
249
|
+
CREATE TRIGGER trg_image_resources_kv_store
|
|
250
|
+
AFTER INSERT OR UPDATE OR DELETE ON image_resources
|
|
251
|
+
FOR EACH ROW EXECUTE FUNCTION fn_image_resources_kv_store_upsert();
|
|
252
|
+
|
|
253
|
+
-- ======================================================================
|
|
254
|
+
-- MOMENTS (Model: Moment)
|
|
255
|
+
-- ======================================================================
|
|
256
|
+
|
|
257
|
+
CREATE TABLE IF NOT EXISTS moments (
|
|
258
|
+
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
|
259
|
+
tenant_id VARCHAR(100) NOT NULL,
|
|
260
|
+
user_id VARCHAR(256),
|
|
261
|
+
name VARCHAR(256) NOT NULL,
|
|
262
|
+
moment_type VARCHAR(256),
|
|
263
|
+
category VARCHAR(256),
|
|
264
|
+
starts_timestamp TIMESTAMP NOT NULL,
|
|
265
|
+
ends_timestamp TIMESTAMP,
|
|
266
|
+
present_persons JSONB DEFAULT '{}'::jsonb,
|
|
267
|
+
emotion_tags TEXT[] DEFAULT ARRAY[]::TEXT[],
|
|
268
|
+
topic_tags TEXT[] DEFAULT ARRAY[]::TEXT[],
|
|
269
|
+
summary TEXT,
|
|
270
|
+
source_resource_ids TEXT[] DEFAULT ARRAY[]::TEXT[],
|
|
271
|
+
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
272
|
+
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
273
|
+
deleted_at TIMESTAMP,
|
|
274
|
+
graph_edges JSONB DEFAULT '[]'::jsonb,
|
|
275
|
+
metadata JSONB DEFAULT '{}'::jsonb,
|
|
276
|
+
tags TEXT[] DEFAULT ARRAY[]::TEXT[]
|
|
277
|
+
);
|
|
278
|
+
|
|
279
|
+
CREATE INDEX IF NOT EXISTS idx_moments_tenant ON moments (tenant_id);
|
|
280
|
+
CREATE INDEX IF NOT EXISTS idx_moments_user ON moments (user_id);
|
|
281
|
+
CREATE INDEX IF NOT EXISTS idx_moments_graph_edges ON moments USING GIN (graph_edges);
|
|
282
|
+
CREATE INDEX IF NOT EXISTS idx_moments_metadata ON moments USING GIN (metadata);
|
|
283
|
+
CREATE INDEX IF NOT EXISTS idx_moments_tags ON moments USING GIN (tags);
|
|
284
|
+
|
|
285
|
+
-- Embeddings for moments
|
|
286
|
+
CREATE TABLE IF NOT EXISTS embeddings_moments (
|
|
287
|
+
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
|
288
|
+
entity_id UUID NOT NULL REFERENCES moments(id) ON DELETE CASCADE,
|
|
289
|
+
field_name VARCHAR(100) NOT NULL,
|
|
290
|
+
provider VARCHAR(50) NOT NULL DEFAULT 'openai',
|
|
291
|
+
model VARCHAR(100) NOT NULL DEFAULT 'text-embedding-3-small',
|
|
292
|
+
embedding vector(1536) NOT NULL,
|
|
293
|
+
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
294
|
+
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
295
|
+
|
|
296
|
+
-- Unique: one embedding per entity per field per provider
|
|
297
|
+
UNIQUE (entity_id, field_name, provider)
|
|
298
|
+
);
|
|
299
|
+
|
|
300
|
+
-- Index for entity lookup (get all embeddings for entity)
|
|
301
|
+
CREATE INDEX IF NOT EXISTS idx_embeddings_moments_entity ON embeddings_moments (entity_id);
|
|
302
|
+
|
|
303
|
+
-- Index for field + provider lookup
|
|
304
|
+
CREATE INDEX IF NOT EXISTS idx_embeddings_moments_field_provider ON embeddings_moments (field_name, provider);
|
|
305
|
+
|
|
306
|
+
-- HNSW index for vector similarity search (created in background)
|
|
307
|
+
-- Note: This will be created by background thread after data load
|
|
308
|
+
-- CREATE INDEX IF NOT EXISTS idx_embeddings_moments_vector_hnsw ON embeddings_moments
|
|
309
|
+
-- USING hnsw (embedding vector_cosine_ops);
|
|
310
|
+
|
|
311
|
+
-- KV_STORE trigger for moments
|
|
312
|
+
-- Trigger function to maintain KV_STORE for moments
|
|
313
|
+
CREATE OR REPLACE FUNCTION fn_moments_kv_store_upsert()
|
|
314
|
+
RETURNS TRIGGER AS $$
|
|
315
|
+
BEGIN
|
|
316
|
+
IF (TG_OP = 'DELETE') THEN
|
|
317
|
+
-- Remove from KV_STORE on delete
|
|
318
|
+
DELETE FROM kv_store
|
|
319
|
+
WHERE entity_id = OLD.id;
|
|
320
|
+
RETURN OLD;
|
|
321
|
+
ELSIF (TG_OP = 'INSERT' OR TG_OP = 'UPDATE') THEN
|
|
322
|
+
-- Upsert to KV_STORE (O(1) lookup by entity_key)
|
|
323
|
+
INSERT INTO kv_store (
|
|
324
|
+
entity_key,
|
|
325
|
+
entity_type,
|
|
326
|
+
entity_id,
|
|
327
|
+
tenant_id,
|
|
328
|
+
user_id,
|
|
329
|
+
metadata,
|
|
330
|
+
graph_edges,
|
|
331
|
+
updated_at
|
|
332
|
+
) VALUES (
|
|
333
|
+
NEW.name, -- Use name as entity_key (json_schema_extra)
|
|
334
|
+
'moments',
|
|
335
|
+
NEW.id,
|
|
336
|
+
NEW.tenant_id,
|
|
337
|
+
NEW.user_id,
|
|
338
|
+
NEW.metadata,
|
|
339
|
+
COALESCE(NEW.graph_edges, '[]'::jsonb),
|
|
340
|
+
CURRENT_TIMESTAMP
|
|
341
|
+
)
|
|
342
|
+
ON CONFLICT (tenant_id, entity_key)
|
|
343
|
+
DO UPDATE SET
|
|
344
|
+
entity_id = EXCLUDED.entity_id,
|
|
345
|
+
user_id = EXCLUDED.user_id,
|
|
346
|
+
metadata = EXCLUDED.metadata,
|
|
347
|
+
graph_edges = EXCLUDED.graph_edges,
|
|
348
|
+
updated_at = CURRENT_TIMESTAMP;
|
|
349
|
+
|
|
350
|
+
RETURN NEW;
|
|
351
|
+
END IF;
|
|
352
|
+
END;
|
|
353
|
+
$$ LANGUAGE plpgsql;
|
|
354
|
+
|
|
355
|
+
-- Create trigger
|
|
356
|
+
DROP TRIGGER IF EXISTS trg_moments_kv_store ON moments;
|
|
357
|
+
CREATE TRIGGER trg_moments_kv_store
|
|
358
|
+
AFTER INSERT OR UPDATE OR DELETE ON moments
|
|
359
|
+
FOR EACH ROW EXECUTE FUNCTION fn_moments_kv_store_upsert();
|
|
360
|
+
|
|
361
|
+
-- ======================================================================
|
|
362
|
+
-- PERSONS (Model: Person)
|
|
363
|
+
-- ======================================================================
|
|
364
|
+
|
|
365
|
+
CREATE TABLE IF NOT EXISTS persons (
|
|
366
|
+
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
|
367
|
+
tenant_id VARCHAR(100) NOT NULL,
|
|
368
|
+
user_id VARCHAR(256),
|
|
369
|
+
name VARCHAR(256) NOT NULL,
|
|
370
|
+
role VARCHAR(256),
|
|
371
|
+
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
372
|
+
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
373
|
+
deleted_at TIMESTAMP,
|
|
374
|
+
graph_edges JSONB DEFAULT '[]'::jsonb,
|
|
375
|
+
metadata JSONB DEFAULT '{}'::jsonb,
|
|
376
|
+
tags TEXT[] DEFAULT ARRAY[]::TEXT[]
|
|
377
|
+
);
|
|
378
|
+
|
|
379
|
+
CREATE INDEX IF NOT EXISTS idx_persons_tenant ON persons (tenant_id);
|
|
380
|
+
CREATE INDEX IF NOT EXISTS idx_persons_user ON persons (user_id);
|
|
381
|
+
CREATE INDEX IF NOT EXISTS idx_persons_graph_edges ON persons USING GIN (graph_edges);
|
|
382
|
+
CREATE INDEX IF NOT EXISTS idx_persons_metadata ON persons USING GIN (metadata);
|
|
383
|
+
CREATE INDEX IF NOT EXISTS idx_persons_tags ON persons USING GIN (tags);
|
|
384
|
+
|
|
385
|
+
-- KV_STORE trigger for persons
|
|
386
|
+
-- Trigger function to maintain KV_STORE for persons
|
|
387
|
+
CREATE OR REPLACE FUNCTION fn_persons_kv_store_upsert()
|
|
388
|
+
RETURNS TRIGGER AS $$
|
|
389
|
+
BEGIN
|
|
390
|
+
IF (TG_OP = 'DELETE') THEN
|
|
391
|
+
-- Remove from KV_STORE on delete
|
|
392
|
+
DELETE FROM kv_store
|
|
393
|
+
WHERE entity_id = OLD.id;
|
|
394
|
+
RETURN OLD;
|
|
395
|
+
ELSIF (TG_OP = 'INSERT' OR TG_OP = 'UPDATE') THEN
|
|
396
|
+
-- Upsert to KV_STORE (O(1) lookup by entity_key)
|
|
397
|
+
INSERT INTO kv_store (
|
|
398
|
+
entity_key,
|
|
399
|
+
entity_type,
|
|
400
|
+
entity_id,
|
|
401
|
+
tenant_id,
|
|
402
|
+
user_id,
|
|
403
|
+
metadata,
|
|
404
|
+
graph_edges,
|
|
405
|
+
updated_at
|
|
406
|
+
) VALUES (
|
|
407
|
+
NEW.name, -- Use name as entity_key (json_schema_extra)
|
|
408
|
+
'persons',
|
|
409
|
+
NEW.id,
|
|
410
|
+
NEW.tenant_id,
|
|
411
|
+
NEW.user_id,
|
|
412
|
+
NEW.metadata,
|
|
413
|
+
COALESCE(NEW.graph_edges, '[]'::jsonb),
|
|
414
|
+
CURRENT_TIMESTAMP
|
|
415
|
+
)
|
|
416
|
+
ON CONFLICT (tenant_id, entity_key)
|
|
417
|
+
DO UPDATE SET
|
|
418
|
+
entity_id = EXCLUDED.entity_id,
|
|
419
|
+
user_id = EXCLUDED.user_id,
|
|
420
|
+
metadata = EXCLUDED.metadata,
|
|
421
|
+
graph_edges = EXCLUDED.graph_edges,
|
|
422
|
+
updated_at = CURRENT_TIMESTAMP;
|
|
423
|
+
|
|
424
|
+
RETURN NEW;
|
|
425
|
+
END IF;
|
|
426
|
+
END;
|
|
427
|
+
$$ LANGUAGE plpgsql;
|
|
428
|
+
|
|
429
|
+
-- Create trigger
|
|
430
|
+
DROP TRIGGER IF EXISTS trg_persons_kv_store ON persons;
|
|
431
|
+
CREATE TRIGGER trg_persons_kv_store
|
|
432
|
+
AFTER INSERT OR UPDATE OR DELETE ON persons
|
|
433
|
+
FOR EACH ROW EXECUTE FUNCTION fn_persons_kv_store_upsert();
|
|
434
|
+
|
|
435
|
+
-- ======================================================================
|
|
436
|
+
-- RESOURCES (Model: Resource)
|
|
437
|
+
-- ======================================================================
|
|
438
|
+
|
|
439
|
+
CREATE TABLE IF NOT EXISTS resources (
|
|
440
|
+
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
|
441
|
+
tenant_id VARCHAR(100) NOT NULL,
|
|
442
|
+
user_id VARCHAR(256),
|
|
443
|
+
name VARCHAR(256) NOT NULL,
|
|
444
|
+
uri VARCHAR(256),
|
|
445
|
+
ordinal INTEGER,
|
|
446
|
+
content TEXT,
|
|
447
|
+
timestamp TIMESTAMP,
|
|
448
|
+
category VARCHAR(256),
|
|
449
|
+
related_entities JSONB DEFAULT '{}'::jsonb,
|
|
450
|
+
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
451
|
+
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
452
|
+
deleted_at TIMESTAMP,
|
|
453
|
+
graph_edges JSONB DEFAULT '[]'::jsonb,
|
|
454
|
+
metadata JSONB DEFAULT '{}'::jsonb,
|
|
455
|
+
tags TEXT[] DEFAULT ARRAY[]::TEXT[]
|
|
456
|
+
);
|
|
457
|
+
|
|
458
|
+
CREATE INDEX IF NOT EXISTS idx_resources_tenant ON resources (tenant_id);
|
|
459
|
+
CREATE INDEX IF NOT EXISTS idx_resources_user ON resources (user_id);
|
|
460
|
+
CREATE INDEX IF NOT EXISTS idx_resources_graph_edges ON resources USING GIN (graph_edges);
|
|
461
|
+
CREATE INDEX IF NOT EXISTS idx_resources_metadata ON resources USING GIN (metadata);
|
|
462
|
+
CREATE INDEX IF NOT EXISTS idx_resources_tags ON resources USING GIN (tags);
|
|
463
|
+
|
|
464
|
+
-- Embeddings for resources
|
|
465
|
+
CREATE TABLE IF NOT EXISTS embeddings_resources (
|
|
466
|
+
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
|
467
|
+
entity_id UUID NOT NULL REFERENCES resources(id) ON DELETE CASCADE,
|
|
468
|
+
field_name VARCHAR(100) NOT NULL,
|
|
469
|
+
provider VARCHAR(50) NOT NULL DEFAULT 'openai',
|
|
470
|
+
model VARCHAR(100) NOT NULL DEFAULT 'text-embedding-3-small',
|
|
471
|
+
embedding vector(1536) NOT NULL,
|
|
472
|
+
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
473
|
+
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
474
|
+
|
|
475
|
+
-- Unique: one embedding per entity per field per provider
|
|
476
|
+
UNIQUE (entity_id, field_name, provider)
|
|
477
|
+
);
|
|
478
|
+
|
|
479
|
+
-- Index for entity lookup (get all embeddings for entity)
|
|
480
|
+
CREATE INDEX IF NOT EXISTS idx_embeddings_resources_entity ON embeddings_resources (entity_id);
|
|
481
|
+
|
|
482
|
+
-- Index for field + provider lookup
|
|
483
|
+
CREATE INDEX IF NOT EXISTS idx_embeddings_resources_field_provider ON embeddings_resources (field_name, provider);
|
|
484
|
+
|
|
485
|
+
-- HNSW index for vector similarity search (created in background)
|
|
486
|
+
-- Note: This will be created by background thread after data load
|
|
487
|
+
-- CREATE INDEX IF NOT EXISTS idx_embeddings_resources_vector_hnsw ON embeddings_resources
|
|
488
|
+
-- USING hnsw (embedding vector_cosine_ops);
|
|
489
|
+
|
|
490
|
+
-- KV_STORE trigger for resources
|
|
491
|
+
-- Trigger function to maintain KV_STORE for resources
|
|
492
|
+
CREATE OR REPLACE FUNCTION fn_resources_kv_store_upsert()
|
|
493
|
+
RETURNS TRIGGER AS $$
|
|
494
|
+
BEGIN
|
|
495
|
+
IF (TG_OP = 'DELETE') THEN
|
|
496
|
+
-- Remove from KV_STORE on delete
|
|
497
|
+
DELETE FROM kv_store
|
|
498
|
+
WHERE entity_id = OLD.id;
|
|
499
|
+
RETURN OLD;
|
|
500
|
+
ELSIF (TG_OP = 'INSERT' OR TG_OP = 'UPDATE') THEN
|
|
501
|
+
-- Upsert to KV_STORE (O(1) lookup by entity_key)
|
|
502
|
+
INSERT INTO kv_store (
|
|
503
|
+
entity_key,
|
|
504
|
+
entity_type,
|
|
505
|
+
entity_id,
|
|
506
|
+
tenant_id,
|
|
507
|
+
user_id,
|
|
508
|
+
metadata,
|
|
509
|
+
graph_edges,
|
|
510
|
+
updated_at
|
|
511
|
+
) VALUES (
|
|
512
|
+
NEW.name, -- Use name as entity_key (json_schema_extra)
|
|
513
|
+
'resources',
|
|
514
|
+
NEW.id,
|
|
515
|
+
NEW.tenant_id,
|
|
516
|
+
NEW.user_id,
|
|
517
|
+
NEW.metadata,
|
|
518
|
+
COALESCE(NEW.graph_edges, '[]'::jsonb),
|
|
519
|
+
CURRENT_TIMESTAMP
|
|
520
|
+
)
|
|
521
|
+
ON CONFLICT (tenant_id, entity_key)
|
|
522
|
+
DO UPDATE SET
|
|
523
|
+
entity_id = EXCLUDED.entity_id,
|
|
524
|
+
user_id = EXCLUDED.user_id,
|
|
525
|
+
metadata = EXCLUDED.metadata,
|
|
526
|
+
graph_edges = EXCLUDED.graph_edges,
|
|
527
|
+
updated_at = CURRENT_TIMESTAMP;
|
|
528
|
+
|
|
529
|
+
RETURN NEW;
|
|
530
|
+
END IF;
|
|
531
|
+
END;
|
|
532
|
+
$$ LANGUAGE plpgsql;
|
|
533
|
+
|
|
534
|
+
-- Create trigger
|
|
535
|
+
DROP TRIGGER IF EXISTS trg_resources_kv_store ON resources;
|
|
536
|
+
CREATE TRIGGER trg_resources_kv_store
|
|
537
|
+
AFTER INSERT OR UPDATE OR DELETE ON resources
|
|
538
|
+
FOR EACH ROW EXECUTE FUNCTION fn_resources_kv_store_upsert();
|
|
539
|
+
|
|
540
|
+
-- ======================================================================
|
|
541
|
+
-- MESSAGES (Model: Message)
|
|
542
|
+
-- ======================================================================
|
|
543
|
+
|
|
544
|
+
CREATE TABLE IF NOT EXISTS messages (
|
|
545
|
+
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
|
546
|
+
tenant_id VARCHAR(100) NOT NULL,
|
|
547
|
+
user_id VARCHAR(256),
|
|
548
|
+
content TEXT NOT NULL,
|
|
549
|
+
message_type TEXT,
|
|
550
|
+
session_id TEXT,
|
|
551
|
+
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
552
|
+
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
553
|
+
deleted_at TIMESTAMP,
|
|
554
|
+
graph_edges JSONB DEFAULT '[]'::jsonb,
|
|
555
|
+
metadata JSONB DEFAULT '{}'::jsonb,
|
|
556
|
+
tags TEXT[] DEFAULT ARRAY[]::TEXT[]
|
|
557
|
+
);
|
|
558
|
+
|
|
559
|
+
CREATE INDEX IF NOT EXISTS idx_messages_tenant ON messages (tenant_id);
|
|
560
|
+
CREATE INDEX IF NOT EXISTS idx_messages_user ON messages (user_id);
|
|
561
|
+
CREATE INDEX IF NOT EXISTS idx_messages_graph_edges ON messages USING GIN (graph_edges);
|
|
562
|
+
CREATE INDEX IF NOT EXISTS idx_messages_metadata ON messages USING GIN (metadata);
|
|
563
|
+
CREATE INDEX IF NOT EXISTS idx_messages_tags ON messages USING GIN (tags);
|
|
564
|
+
|
|
565
|
+
-- Embeddings for messages
|
|
566
|
+
CREATE TABLE IF NOT EXISTS embeddings_messages (
|
|
567
|
+
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
|
568
|
+
entity_id UUID NOT NULL REFERENCES messages(id) ON DELETE CASCADE,
|
|
569
|
+
field_name VARCHAR(100) NOT NULL,
|
|
570
|
+
provider VARCHAR(50) NOT NULL DEFAULT 'openai',
|
|
571
|
+
model VARCHAR(100) NOT NULL DEFAULT 'text-embedding-3-small',
|
|
572
|
+
embedding vector(1536) NOT NULL,
|
|
573
|
+
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
574
|
+
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
575
|
+
|
|
576
|
+
-- Unique: one embedding per entity per field per provider
|
|
577
|
+
UNIQUE (entity_id, field_name, provider)
|
|
578
|
+
);
|
|
579
|
+
|
|
580
|
+
-- Index for entity lookup (get all embeddings for entity)
|
|
581
|
+
CREATE INDEX IF NOT EXISTS idx_embeddings_messages_entity ON embeddings_messages (entity_id);
|
|
582
|
+
|
|
583
|
+
-- Index for field + provider lookup
|
|
584
|
+
CREATE INDEX IF NOT EXISTS idx_embeddings_messages_field_provider ON embeddings_messages (field_name, provider);
|
|
585
|
+
|
|
586
|
+
-- HNSW index for vector similarity search (created in background)
|
|
587
|
+
-- Note: This will be created by background thread after data load
|
|
588
|
+
-- CREATE INDEX IF NOT EXISTS idx_embeddings_messages_vector_hnsw ON embeddings_messages
|
|
589
|
+
-- USING hnsw (embedding vector_cosine_ops);
|
|
590
|
+
|
|
591
|
+
-- KV_STORE trigger for messages
|
|
592
|
+
-- Trigger function to maintain KV_STORE for messages
|
|
593
|
+
CREATE OR REPLACE FUNCTION fn_messages_kv_store_upsert()
|
|
594
|
+
RETURNS TRIGGER AS $$
|
|
595
|
+
BEGIN
|
|
596
|
+
IF (TG_OP = 'DELETE') THEN
|
|
597
|
+
-- Remove from KV_STORE on delete
|
|
598
|
+
DELETE FROM kv_store
|
|
599
|
+
WHERE entity_id = OLD.id;
|
|
600
|
+
RETURN OLD;
|
|
601
|
+
ELSIF (TG_OP = 'INSERT' OR TG_OP = 'UPDATE') THEN
|
|
602
|
+
-- Upsert to KV_STORE (O(1) lookup by entity_key)
|
|
603
|
+
INSERT INTO kv_store (
|
|
604
|
+
entity_key,
|
|
605
|
+
entity_type,
|
|
606
|
+
entity_id,
|
|
607
|
+
tenant_id,
|
|
608
|
+
user_id,
|
|
609
|
+
metadata,
|
|
610
|
+
graph_edges,
|
|
611
|
+
updated_at
|
|
612
|
+
) VALUES (
|
|
613
|
+
NEW.id::VARCHAR, -- Messages use id as entity_key (no name field)
|
|
614
|
+
'messages',
|
|
615
|
+
NEW.id,
|
|
616
|
+
NEW.tenant_id,
|
|
617
|
+
NEW.user_id,
|
|
618
|
+
NEW.metadata,
|
|
619
|
+
COALESCE(NEW.graph_edges, '[]'::jsonb),
|
|
620
|
+
CURRENT_TIMESTAMP
|
|
621
|
+
)
|
|
622
|
+
ON CONFLICT (tenant_id, entity_key)
|
|
623
|
+
DO UPDATE SET
|
|
624
|
+
entity_id = EXCLUDED.entity_id,
|
|
625
|
+
user_id = EXCLUDED.user_id,
|
|
626
|
+
metadata = EXCLUDED.metadata,
|
|
627
|
+
graph_edges = EXCLUDED.graph_edges,
|
|
628
|
+
updated_at = CURRENT_TIMESTAMP;
|
|
629
|
+
|
|
630
|
+
RETURN NEW;
|
|
631
|
+
END IF;
|
|
632
|
+
END;
|
|
633
|
+
$$ LANGUAGE plpgsql;
|
|
634
|
+
|
|
635
|
+
-- Create trigger
|
|
636
|
+
DROP TRIGGER IF EXISTS trg_messages_kv_store ON messages;
|
|
637
|
+
CREATE TRIGGER trg_messages_kv_store
|
|
638
|
+
AFTER INSERT OR UPDATE OR DELETE ON messages
|
|
639
|
+
FOR EACH ROW EXECUTE FUNCTION fn_messages_kv_store_upsert();
|
|
640
|
+
|
|
641
|
+
-- ======================================================================
|
|
642
|
+
-- FILES (Model: File)
|
|
643
|
+
-- ======================================================================
|
|
644
|
+
|
|
645
|
+
CREATE TABLE IF NOT EXISTS files (
|
|
646
|
+
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
|
647
|
+
tenant_id VARCHAR(100) NOT NULL,
|
|
648
|
+
user_id VARCHAR(256),
|
|
649
|
+
name VARCHAR(256) NOT NULL,
|
|
650
|
+
uri VARCHAR(256) NOT NULL,
|
|
651
|
+
content TEXT,
|
|
652
|
+
timestamp VARCHAR(256),
|
|
653
|
+
size_bytes INTEGER,
|
|
654
|
+
mime_type VARCHAR(256),
|
|
655
|
+
processing_status VARCHAR(256),
|
|
656
|
+
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
657
|
+
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
658
|
+
deleted_at TIMESTAMP,
|
|
659
|
+
graph_edges JSONB DEFAULT '[]'::jsonb,
|
|
660
|
+
metadata JSONB DEFAULT '{}'::jsonb,
|
|
661
|
+
tags TEXT[] DEFAULT ARRAY[]::TEXT[]
|
|
662
|
+
);
|
|
663
|
+
|
|
664
|
+
CREATE INDEX IF NOT EXISTS idx_files_tenant ON files (tenant_id);
|
|
665
|
+
CREATE INDEX IF NOT EXISTS idx_files_user ON files (user_id);
|
|
666
|
+
CREATE INDEX IF NOT EXISTS idx_files_graph_edges ON files USING GIN (graph_edges);
|
|
667
|
+
CREATE INDEX IF NOT EXISTS idx_files_metadata ON files USING GIN (metadata);
|
|
668
|
+
CREATE INDEX IF NOT EXISTS idx_files_tags ON files USING GIN (tags);
|
|
669
|
+
|
|
670
|
+
-- Embeddings for files
|
|
671
|
+
CREATE TABLE IF NOT EXISTS embeddings_files (
|
|
672
|
+
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
|
673
|
+
entity_id UUID NOT NULL REFERENCES files(id) ON DELETE CASCADE,
|
|
674
|
+
field_name VARCHAR(100) NOT NULL,
|
|
675
|
+
provider VARCHAR(50) NOT NULL DEFAULT 'openai',
|
|
676
|
+
model VARCHAR(100) NOT NULL DEFAULT 'text-embedding-3-small',
|
|
677
|
+
embedding vector(1536) NOT NULL,
|
|
678
|
+
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
679
|
+
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
680
|
+
|
|
681
|
+
-- Unique: one embedding per entity per field per provider
|
|
682
|
+
UNIQUE (entity_id, field_name, provider)
|
|
683
|
+
);
|
|
684
|
+
|
|
685
|
+
-- Index for entity lookup (get all embeddings for entity)
|
|
686
|
+
CREATE INDEX IF NOT EXISTS idx_embeddings_files_entity ON embeddings_files (entity_id);
|
|
687
|
+
|
|
688
|
+
-- Index for field + provider lookup
|
|
689
|
+
CREATE INDEX IF NOT EXISTS idx_embeddings_files_field_provider ON embeddings_files (field_name, provider);
|
|
690
|
+
|
|
691
|
+
-- HNSW index for vector similarity search (created in background)
|
|
692
|
+
-- Note: This will be created by background thread after data load
|
|
693
|
+
-- CREATE INDEX IF NOT EXISTS idx_embeddings_files_vector_hnsw ON embeddings_files
|
|
694
|
+
-- USING hnsw (embedding vector_cosine_ops);
|
|
695
|
+
|
|
696
|
+
-- KV_STORE trigger for files
|
|
697
|
+
-- Trigger function to maintain KV_STORE for files
|
|
698
|
+
CREATE OR REPLACE FUNCTION fn_files_kv_store_upsert()
|
|
699
|
+
RETURNS TRIGGER AS $$
|
|
700
|
+
BEGIN
|
|
701
|
+
IF (TG_OP = 'DELETE') THEN
|
|
702
|
+
-- Remove from KV_STORE on delete
|
|
703
|
+
DELETE FROM kv_store
|
|
704
|
+
WHERE entity_id = OLD.id;
|
|
705
|
+
RETURN OLD;
|
|
706
|
+
ELSIF (TG_OP = 'INSERT' OR TG_OP = 'UPDATE') THEN
|
|
707
|
+
-- Upsert to KV_STORE (O(1) lookup by entity_key)
|
|
708
|
+
INSERT INTO kv_store (
|
|
709
|
+
entity_key,
|
|
710
|
+
entity_type,
|
|
711
|
+
entity_id,
|
|
712
|
+
tenant_id,
|
|
713
|
+
user_id,
|
|
714
|
+
metadata,
|
|
715
|
+
graph_edges,
|
|
716
|
+
updated_at
|
|
717
|
+
) VALUES (
|
|
718
|
+
NEW.name, -- Use name as entity_key (json_schema_extra)
|
|
719
|
+
'files',
|
|
720
|
+
NEW.id,
|
|
721
|
+
NEW.tenant_id,
|
|
722
|
+
NEW.user_id,
|
|
723
|
+
NEW.metadata,
|
|
724
|
+
COALESCE(NEW.graph_edges, '[]'::jsonb),
|
|
725
|
+
CURRENT_TIMESTAMP
|
|
726
|
+
)
|
|
727
|
+
ON CONFLICT (tenant_id, entity_key)
|
|
728
|
+
DO UPDATE SET
|
|
729
|
+
entity_id = EXCLUDED.entity_id,
|
|
730
|
+
user_id = EXCLUDED.user_id,
|
|
731
|
+
metadata = EXCLUDED.metadata,
|
|
732
|
+
graph_edges = EXCLUDED.graph_edges,
|
|
733
|
+
updated_at = CURRENT_TIMESTAMP;
|
|
734
|
+
|
|
735
|
+
RETURN NEW;
|
|
736
|
+
END IF;
|
|
737
|
+
END;
|
|
738
|
+
$$ LANGUAGE plpgsql;
|
|
739
|
+
|
|
740
|
+
-- Create trigger
|
|
741
|
+
DROP TRIGGER IF EXISTS trg_files_kv_store ON files;
|
|
742
|
+
CREATE TRIGGER trg_files_kv_store
|
|
743
|
+
AFTER INSERT OR UPDATE OR DELETE ON files
|
|
744
|
+
FOR EACH ROW EXECUTE FUNCTION fn_files_kv_store_upsert();
|
|
745
|
+
|
|
746
|
+
-- ======================================================================
|
|
747
|
+
-- ONTOLOGIES (Model: Ontology)
|
|
748
|
+
-- ======================================================================
|
|
749
|
+
|
|
750
|
+
CREATE TABLE IF NOT EXISTS ontologies (
|
|
751
|
+
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
|
752
|
+
tenant_id VARCHAR(100) NOT NULL,
|
|
753
|
+
user_id VARCHAR(256),
|
|
754
|
+
name VARCHAR(256) NOT NULL,
|
|
755
|
+
file_id TEXT NOT NULL,
|
|
756
|
+
agent_schema_id VARCHAR(256) NOT NULL,
|
|
757
|
+
provider_name VARCHAR(256) NOT NULL,
|
|
758
|
+
model_name VARCHAR(256) NOT NULL,
|
|
759
|
+
extracted_data JSONB NOT NULL,
|
|
760
|
+
confidence_score DOUBLE PRECISION,
|
|
761
|
+
extraction_timestamp VARCHAR(256),
|
|
762
|
+
embedding_text TEXT,
|
|
763
|
+
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
764
|
+
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
765
|
+
deleted_at TIMESTAMP,
|
|
766
|
+
graph_edges JSONB DEFAULT '[]'::jsonb,
|
|
767
|
+
metadata JSONB DEFAULT '{}'::jsonb,
|
|
768
|
+
tags TEXT[] DEFAULT ARRAY[]::TEXT[]
|
|
769
|
+
);
|
|
770
|
+
|
|
771
|
+
CREATE INDEX IF NOT EXISTS idx_ontologies_tenant ON ontologies (tenant_id);
|
|
772
|
+
CREATE INDEX IF NOT EXISTS idx_ontologies_user ON ontologies (user_id);
|
|
773
|
+
CREATE INDEX IF NOT EXISTS idx_ontologies_graph_edges ON ontologies USING GIN (graph_edges);
|
|
774
|
+
CREATE INDEX IF NOT EXISTS idx_ontologies_metadata ON ontologies USING GIN (metadata);
|
|
775
|
+
CREATE INDEX IF NOT EXISTS idx_ontologies_tags ON ontologies USING GIN (tags);
|
|
776
|
+
|
|
777
|
+
-- KV_STORE trigger for ontologies
|
|
778
|
+
-- Trigger function to maintain KV_STORE for ontologies
|
|
779
|
+
CREATE OR REPLACE FUNCTION fn_ontologies_kv_store_upsert()
|
|
780
|
+
RETURNS TRIGGER AS $$
|
|
781
|
+
BEGIN
|
|
782
|
+
IF (TG_OP = 'DELETE') THEN
|
|
783
|
+
-- Remove from KV_STORE on delete
|
|
784
|
+
DELETE FROM kv_store
|
|
785
|
+
WHERE entity_id = OLD.id;
|
|
786
|
+
RETURN OLD;
|
|
787
|
+
ELSIF (TG_OP = 'INSERT' OR TG_OP = 'UPDATE') THEN
|
|
788
|
+
-- Upsert to KV_STORE (O(1) lookup by entity_key)
|
|
789
|
+
INSERT INTO kv_store (
|
|
790
|
+
entity_key,
|
|
791
|
+
entity_type,
|
|
792
|
+
entity_id,
|
|
793
|
+
tenant_id,
|
|
794
|
+
user_id,
|
|
795
|
+
metadata,
|
|
796
|
+
graph_edges,
|
|
797
|
+
updated_at
|
|
798
|
+
) VALUES (
|
|
799
|
+
NEW.name, -- Use name as entity_key (json_schema_extra)
|
|
800
|
+
'ontologies',
|
|
801
|
+
NEW.id,
|
|
802
|
+
NEW.tenant_id,
|
|
803
|
+
NEW.user_id,
|
|
804
|
+
NEW.metadata,
|
|
805
|
+
COALESCE(NEW.graph_edges, '[]'::jsonb),
|
|
806
|
+
CURRENT_TIMESTAMP
|
|
807
|
+
)
|
|
808
|
+
ON CONFLICT (tenant_id, entity_key)
|
|
809
|
+
DO UPDATE SET
|
|
810
|
+
entity_id = EXCLUDED.entity_id,
|
|
811
|
+
user_id = EXCLUDED.user_id,
|
|
812
|
+
metadata = EXCLUDED.metadata,
|
|
813
|
+
graph_edges = EXCLUDED.graph_edges,
|
|
814
|
+
updated_at = CURRENT_TIMESTAMP;
|
|
815
|
+
|
|
816
|
+
RETURN NEW;
|
|
817
|
+
END IF;
|
|
818
|
+
END;
|
|
819
|
+
$$ LANGUAGE plpgsql;
|
|
820
|
+
|
|
821
|
+
-- Create trigger
|
|
822
|
+
DROP TRIGGER IF EXISTS trg_ontologies_kv_store ON ontologies;
|
|
823
|
+
CREATE TRIGGER trg_ontologies_kv_store
|
|
824
|
+
AFTER INSERT OR UPDATE OR DELETE ON ontologies
|
|
825
|
+
FOR EACH ROW EXECUTE FUNCTION fn_ontologies_kv_store_upsert();
|
|
826
|
+
|
|
827
|
+
-- ======================================================================
|
|
828
|
+
-- ONTOLOGY_CONFIGS (Model: OntologyConfig)
|
|
829
|
+
-- ======================================================================
|
|
830
|
+
|
|
831
|
+
CREATE TABLE IF NOT EXISTS ontology_configs (
|
|
832
|
+
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
|
833
|
+
tenant_id VARCHAR(100) NOT NULL,
|
|
834
|
+
user_id VARCHAR(256),
|
|
835
|
+
name VARCHAR(256) NOT NULL,
|
|
836
|
+
agent_schema_id VARCHAR(256) NOT NULL,
|
|
837
|
+
description TEXT,
|
|
838
|
+
mime_type_pattern VARCHAR(256),
|
|
839
|
+
uri_pattern VARCHAR(256),
|
|
840
|
+
tag_filter TEXT[],
|
|
841
|
+
priority INTEGER,
|
|
842
|
+
enabled BOOLEAN,
|
|
843
|
+
provider_name VARCHAR(256),
|
|
844
|
+
model_name VARCHAR(256),
|
|
845
|
+
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
846
|
+
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
847
|
+
deleted_at TIMESTAMP,
|
|
848
|
+
graph_edges JSONB DEFAULT '[]'::jsonb,
|
|
849
|
+
metadata JSONB DEFAULT '{}'::jsonb,
|
|
850
|
+
tags TEXT[] DEFAULT ARRAY[]::TEXT[]
|
|
851
|
+
);
|
|
852
|
+
|
|
853
|
+
CREATE INDEX IF NOT EXISTS idx_ontology_configs_tenant ON ontology_configs (tenant_id);
|
|
854
|
+
CREATE INDEX IF NOT EXISTS idx_ontology_configs_user ON ontology_configs (user_id);
|
|
855
|
+
CREATE INDEX IF NOT EXISTS idx_ontology_configs_graph_edges ON ontology_configs USING GIN (graph_edges);
|
|
856
|
+
CREATE INDEX IF NOT EXISTS idx_ontology_configs_metadata ON ontology_configs USING GIN (metadata);
|
|
857
|
+
CREATE INDEX IF NOT EXISTS idx_ontology_configs_tags ON ontology_configs USING GIN (tags);
|
|
858
|
+
|
|
859
|
+
-- Embeddings for ontology_configs
|
|
860
|
+
CREATE TABLE IF NOT EXISTS embeddings_ontology_configs (
|
|
861
|
+
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
|
862
|
+
entity_id UUID NOT NULL REFERENCES ontology_configs(id) ON DELETE CASCADE,
|
|
863
|
+
field_name VARCHAR(100) NOT NULL,
|
|
864
|
+
provider VARCHAR(50) NOT NULL DEFAULT 'openai',
|
|
865
|
+
model VARCHAR(100) NOT NULL DEFAULT 'text-embedding-3-small',
|
|
866
|
+
embedding vector(1536) NOT NULL,
|
|
867
|
+
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
868
|
+
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
869
|
+
|
|
870
|
+
-- Unique: one embedding per entity per field per provider
|
|
871
|
+
UNIQUE (entity_id, field_name, provider)
|
|
872
|
+
);
|
|
873
|
+
|
|
874
|
+
-- Index for entity lookup (get all embeddings for entity)
|
|
875
|
+
CREATE INDEX IF NOT EXISTS idx_embeddings_ontology_configs_entity ON embeddings_ontology_configs (entity_id);
|
|
876
|
+
|
|
877
|
+
-- Index for field + provider lookup
|
|
878
|
+
CREATE INDEX IF NOT EXISTS idx_embeddings_ontology_configs_field_provider ON embeddings_ontology_configs (field_name, provider);
|
|
879
|
+
|
|
880
|
+
-- HNSW index for vector similarity search (created in background)
|
|
881
|
+
-- Note: This will be created by background thread after data load
|
|
882
|
+
-- CREATE INDEX IF NOT EXISTS idx_embeddings_ontology_configs_vector_hnsw ON embeddings_ontology_configs
|
|
883
|
+
-- USING hnsw (embedding vector_cosine_ops);
|
|
884
|
+
|
|
885
|
+
-- KV_STORE trigger for ontology_configs
|
|
886
|
+
-- Trigger function to maintain KV_STORE for ontology_configs
|
|
887
|
+
CREATE OR REPLACE FUNCTION fn_ontology_configs_kv_store_upsert()
|
|
888
|
+
RETURNS TRIGGER AS $$
|
|
889
|
+
BEGIN
|
|
890
|
+
IF (TG_OP = 'DELETE') THEN
|
|
891
|
+
-- Remove from KV_STORE on delete
|
|
892
|
+
DELETE FROM kv_store
|
|
893
|
+
WHERE entity_id = OLD.id;
|
|
894
|
+
RETURN OLD;
|
|
895
|
+
ELSIF (TG_OP = 'INSERT' OR TG_OP = 'UPDATE') THEN
|
|
896
|
+
-- Upsert to KV_STORE (O(1) lookup by entity_key)
|
|
897
|
+
INSERT INTO kv_store (
|
|
898
|
+
entity_key,
|
|
899
|
+
entity_type,
|
|
900
|
+
entity_id,
|
|
901
|
+
tenant_id,
|
|
902
|
+
user_id,
|
|
903
|
+
metadata,
|
|
904
|
+
graph_edges,
|
|
905
|
+
updated_at
|
|
906
|
+
) VALUES (
|
|
907
|
+
NEW.name, -- Use name as entity_key (json_schema_extra)
|
|
908
|
+
'ontology_configs',
|
|
909
|
+
NEW.id,
|
|
910
|
+
NEW.tenant_id,
|
|
911
|
+
NEW.user_id,
|
|
912
|
+
NEW.metadata,
|
|
913
|
+
COALESCE(NEW.graph_edges, '[]'::jsonb),
|
|
914
|
+
CURRENT_TIMESTAMP
|
|
915
|
+
)
|
|
916
|
+
ON CONFLICT (tenant_id, entity_key)
|
|
917
|
+
DO UPDATE SET
|
|
918
|
+
entity_id = EXCLUDED.entity_id,
|
|
919
|
+
user_id = EXCLUDED.user_id,
|
|
920
|
+
metadata = EXCLUDED.metadata,
|
|
921
|
+
graph_edges = EXCLUDED.graph_edges,
|
|
922
|
+
updated_at = CURRENT_TIMESTAMP;
|
|
923
|
+
|
|
924
|
+
RETURN NEW;
|
|
925
|
+
END IF;
|
|
926
|
+
END;
|
|
927
|
+
$$ LANGUAGE plpgsql;
|
|
928
|
+
|
|
929
|
+
-- Create trigger
|
|
930
|
+
DROP TRIGGER IF EXISTS trg_ontology_configs_kv_store ON ontology_configs;
|
|
931
|
+
CREATE TRIGGER trg_ontology_configs_kv_store
|
|
932
|
+
AFTER INSERT OR UPDATE OR DELETE ON ontology_configs
|
|
933
|
+
FOR EACH ROW EXECUTE FUNCTION fn_ontology_configs_kv_store_upsert();
|
|
934
|
+
|
|
935
|
+
-- ======================================================================
|
|
936
|
+
-- SCHEMAS (Model: Schema)
|
|
937
|
+
-- ======================================================================
|
|
938
|
+
|
|
939
|
+
CREATE TABLE IF NOT EXISTS schemas (
|
|
940
|
+
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
|
941
|
+
tenant_id VARCHAR(100) NOT NULL,
|
|
942
|
+
user_id VARCHAR(256),
|
|
943
|
+
name VARCHAR(256) NOT NULL,
|
|
944
|
+
content TEXT,
|
|
945
|
+
spec JSONB NOT NULL,
|
|
946
|
+
category VARCHAR(256),
|
|
947
|
+
provider_configs JSONB DEFAULT '{}'::jsonb,
|
|
948
|
+
embedding_fields TEXT[] DEFAULT ARRAY[]::TEXT[],
|
|
949
|
+
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
950
|
+
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
951
|
+
deleted_at TIMESTAMP,
|
|
952
|
+
graph_edges JSONB DEFAULT '[]'::jsonb,
|
|
953
|
+
metadata JSONB DEFAULT '{}'::jsonb,
|
|
954
|
+
tags TEXT[] DEFAULT ARRAY[]::TEXT[]
|
|
955
|
+
);
|
|
956
|
+
|
|
957
|
+
CREATE INDEX IF NOT EXISTS idx_schemas_tenant ON schemas (tenant_id);
|
|
958
|
+
CREATE INDEX IF NOT EXISTS idx_schemas_user ON schemas (user_id);
|
|
959
|
+
CREATE INDEX IF NOT EXISTS idx_schemas_graph_edges ON schemas USING GIN (graph_edges);
|
|
960
|
+
CREATE INDEX IF NOT EXISTS idx_schemas_metadata ON schemas USING GIN (metadata);
|
|
961
|
+
CREATE INDEX IF NOT EXISTS idx_schemas_tags ON schemas USING GIN (tags);
|
|
962
|
+
|
|
963
|
+
-- Embeddings for schemas
|
|
964
|
+
CREATE TABLE IF NOT EXISTS embeddings_schemas (
|
|
965
|
+
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
|
966
|
+
entity_id UUID NOT NULL REFERENCES schemas(id) ON DELETE CASCADE,
|
|
967
|
+
field_name VARCHAR(100) NOT NULL,
|
|
968
|
+
provider VARCHAR(50) NOT NULL DEFAULT 'openai',
|
|
969
|
+
model VARCHAR(100) NOT NULL DEFAULT 'text-embedding-3-small',
|
|
970
|
+
embedding vector(1536) NOT NULL,
|
|
971
|
+
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
972
|
+
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
973
|
+
|
|
974
|
+
-- Unique: one embedding per entity per field per provider
|
|
975
|
+
UNIQUE (entity_id, field_name, provider)
|
|
976
|
+
);
|
|
977
|
+
|
|
978
|
+
-- Index for entity lookup (get all embeddings for entity)
|
|
979
|
+
CREATE INDEX IF NOT EXISTS idx_embeddings_schemas_entity ON embeddings_schemas (entity_id);
|
|
980
|
+
|
|
981
|
+
-- Index for field + provider lookup
|
|
982
|
+
CREATE INDEX IF NOT EXISTS idx_embeddings_schemas_field_provider ON embeddings_schemas (field_name, provider);
|
|
983
|
+
|
|
984
|
+
-- HNSW index for vector similarity search (created in background)
|
|
985
|
+
-- Note: This will be created by background thread after data load
|
|
986
|
+
-- CREATE INDEX IF NOT EXISTS idx_embeddings_schemas_vector_hnsw ON embeddings_schemas
|
|
987
|
+
-- USING hnsw (embedding vector_cosine_ops);
|
|
988
|
+
|
|
989
|
+
-- KV_STORE trigger for schemas
|
|
990
|
+
-- Trigger function to maintain KV_STORE for schemas
|
|
991
|
+
CREATE OR REPLACE FUNCTION fn_schemas_kv_store_upsert()
|
|
992
|
+
RETURNS TRIGGER AS $$
|
|
993
|
+
BEGIN
|
|
994
|
+
IF (TG_OP = 'DELETE') THEN
|
|
995
|
+
-- Remove from KV_STORE on delete
|
|
996
|
+
DELETE FROM kv_store
|
|
997
|
+
WHERE entity_id = OLD.id;
|
|
998
|
+
RETURN OLD;
|
|
999
|
+
ELSIF (TG_OP = 'INSERT' OR TG_OP = 'UPDATE') THEN
|
|
1000
|
+
-- Upsert to KV_STORE (O(1) lookup by entity_key)
|
|
1001
|
+
INSERT INTO kv_store (
|
|
1002
|
+
entity_key,
|
|
1003
|
+
entity_type,
|
|
1004
|
+
entity_id,
|
|
1005
|
+
tenant_id,
|
|
1006
|
+
user_id,
|
|
1007
|
+
metadata,
|
|
1008
|
+
graph_edges,
|
|
1009
|
+
updated_at
|
|
1010
|
+
) VALUES (
|
|
1011
|
+
NEW.name, -- Use name as entity_key (json_schema_extra)
|
|
1012
|
+
'schemas',
|
|
1013
|
+
NEW.id,
|
|
1014
|
+
NEW.tenant_id,
|
|
1015
|
+
NEW.user_id,
|
|
1016
|
+
NEW.metadata,
|
|
1017
|
+
COALESCE(NEW.graph_edges, '[]'::jsonb),
|
|
1018
|
+
CURRENT_TIMESTAMP
|
|
1019
|
+
)
|
|
1020
|
+
ON CONFLICT (tenant_id, entity_key)
|
|
1021
|
+
DO UPDATE SET
|
|
1022
|
+
entity_id = EXCLUDED.entity_id,
|
|
1023
|
+
user_id = EXCLUDED.user_id,
|
|
1024
|
+
metadata = EXCLUDED.metadata,
|
|
1025
|
+
graph_edges = EXCLUDED.graph_edges,
|
|
1026
|
+
updated_at = CURRENT_TIMESTAMP;
|
|
1027
|
+
|
|
1028
|
+
RETURN NEW;
|
|
1029
|
+
END IF;
|
|
1030
|
+
END;
|
|
1031
|
+
$$ LANGUAGE plpgsql;
|
|
1032
|
+
|
|
1033
|
+
-- Create trigger
|
|
1034
|
+
DROP TRIGGER IF EXISTS trg_schemas_kv_store ON schemas;
|
|
1035
|
+
CREATE TRIGGER trg_schemas_kv_store
|
|
1036
|
+
AFTER INSERT OR UPDATE OR DELETE ON schemas
|
|
1037
|
+
FOR EACH ROW EXECUTE FUNCTION fn_schemas_kv_store_upsert();
|
|
1038
|
+
|
|
1039
|
+
-- ============================================================================
|
|
1040
|
+
-- RECORD MIGRATION
|
|
1041
|
+
-- ============================================================================
|
|
1042
|
+
|
|
1043
|
+
INSERT INTO rem_migrations (name, type, version)
|
|
1044
|
+
VALUES ('install_models.sql', 'models', '1.0.0')
|
|
1045
|
+
ON CONFLICT (name) DO UPDATE
|
|
1046
|
+
SET applied_at = CURRENT_TIMESTAMP,
|
|
1047
|
+
applied_by = CURRENT_USER;
|
|
1048
|
+
|
|
1049
|
+
DO $$
|
|
1050
|
+
BEGIN
|
|
1051
|
+
RAISE NOTICE '============================================================';
|
|
1052
|
+
RAISE NOTICE 'REM Model Schema Applied: 10 tables';
|
|
1053
|
+
RAISE NOTICE '============================================================';
|
|
1054
|
+
RAISE NOTICE ' ✓ files (1 embeddable fields)';
|
|
1055
|
+
RAISE NOTICE ' ✓ image_resources (1 embeddable fields)';
|
|
1056
|
+
RAISE NOTICE ' ✓ messages (1 embeddable fields)';
|
|
1057
|
+
RAISE NOTICE ' ✓ moments (1 embeddable fields)';
|
|
1058
|
+
RAISE NOTICE ' ✓ ontologies';
|
|
1059
|
+
RAISE NOTICE ' ✓ ontology_configs (1 embeddable fields)';
|
|
1060
|
+
RAISE NOTICE ' ✓ persons';
|
|
1061
|
+
RAISE NOTICE ' ✓ resources (1 embeddable fields)';
|
|
1062
|
+
RAISE NOTICE ' ✓ schemas (1 embeddable fields)';
|
|
1063
|
+
RAISE NOTICE ' ✓ users (1 embeddable fields)';
|
|
1064
|
+
RAISE NOTICE '';
|
|
1065
|
+
RAISE NOTICE 'Next: Run background indexes if needed';
|
|
1066
|
+
RAISE NOTICE ' rem db migrate --background-indexes';
|
|
1067
|
+
RAISE NOTICE '============================================================';
|
|
1068
|
+
END $$;
|
|
1069
|
+
-- ============================================================================
|
|
1070
|
+
-- REM TRAVERSE (Graph Traversal)
|
|
1071
|
+
-- ============================================================================
|
|
1072
|
+
|
|
1073
|
+
-- REM TRAVERSE: Recursive graph traversal following edges
|
|
1074
|
+
-- Explores graph_edges starting from entity_key up to max_depth
|
|
1075
|
+
-- Uses cached kv_store.graph_edges for fast traversal (no polymorphic view!)
|
|
1076
|
+
-- When keys_only=false, automatically fetches full entity records
|
|
1077
|
+
CREATE OR REPLACE FUNCTION rem_traverse(
|
|
1078
|
+
p_entity_key VARCHAR(255),
|
|
1079
|
+
p_tenant_id VARCHAR(100), -- Backward compat parameter (not used for filtering)
|
|
1080
|
+
p_user_id VARCHAR(100),
|
|
1081
|
+
p_max_depth INTEGER DEFAULT 1,
|
|
1082
|
+
p_rel_type VARCHAR(100) DEFAULT NULL,
|
|
1083
|
+
p_keys_only BOOLEAN DEFAULT FALSE
|
|
1084
|
+
)
|
|
1085
|
+
RETURNS TABLE(
|
|
1086
|
+
depth INTEGER,
|
|
1087
|
+
entity_key VARCHAR(255),
|
|
1088
|
+
entity_type VARCHAR(100),
|
|
1089
|
+
entity_id UUID,
|
|
1090
|
+
rel_type VARCHAR(100),
|
|
1091
|
+
rel_weight REAL,
|
|
1092
|
+
path TEXT[],
|
|
1093
|
+
entity_record JSONB
|
|
1094
|
+
) AS $$
|
|
1095
|
+
DECLARE
|
|
1096
|
+
graph_keys RECORD;
|
|
1097
|
+
entities_by_table JSONB := '{}'::jsonb;
|
|
1098
|
+
table_keys JSONB;
|
|
1099
|
+
BEGIN
|
|
1100
|
+
-- First, build graph structure from KV store
|
|
1101
|
+
FOR graph_keys IN
|
|
1102
|
+
WITH RECURSIVE graph_traversal AS (
|
|
1103
|
+
-- Base case: Find starting entity
|
|
1104
|
+
SELECT
|
|
1105
|
+
0 AS depth,
|
|
1106
|
+
kv.entity_key,
|
|
1107
|
+
kv.entity_type,
|
|
1108
|
+
kv.entity_id,
|
|
1109
|
+
NULL::VARCHAR(100) AS rel_type,
|
|
1110
|
+
NULL::REAL AS rel_weight,
|
|
1111
|
+
ARRAY[kv.entity_key]::TEXT[] AS path
|
|
1112
|
+
FROM kv_store kv
|
|
1113
|
+
WHERE kv.user_id = p_user_id
|
|
1114
|
+
AND kv.entity_key = p_entity_key
|
|
1115
|
+
|
|
1116
|
+
UNION ALL
|
|
1117
|
+
|
|
1118
|
+
-- Recursive case: Follow outbound edges from discovered entities
|
|
1119
|
+
SELECT
|
|
1120
|
+
gt.depth + 1,
|
|
1121
|
+
target_kv.entity_key,
|
|
1122
|
+
target_kv.entity_type,
|
|
1123
|
+
target_kv.entity_id,
|
|
1124
|
+
(edge->>'rel_type')::VARCHAR(100) AS rel_type,
|
|
1125
|
+
COALESCE((edge->>'weight')::REAL, 1.0) AS rel_weight,
|
|
1126
|
+
gt.path || target_kv.entity_key AS path
|
|
1127
|
+
FROM graph_traversal gt
|
|
1128
|
+
-- Join to KV store to get source entity (with cached graph_edges!)
|
|
1129
|
+
JOIN kv_store source_kv ON source_kv.entity_key = gt.entity_key
|
|
1130
|
+
AND source_kv.user_id = p_user_id
|
|
1131
|
+
-- Extract edges directly from cached kv_store.graph_edges (NO polymorphic view!)
|
|
1132
|
+
CROSS JOIN LATERAL jsonb_array_elements(COALESCE(source_kv.graph_edges, '[]'::jsonb)) AS edge
|
|
1133
|
+
-- Lookup target entity in KV store
|
|
1134
|
+
JOIN kv_store target_kv ON target_kv.entity_key = (edge->>'dst')::VARCHAR(255)
|
|
1135
|
+
AND target_kv.user_id = p_user_id
|
|
1136
|
+
WHERE gt.depth < p_max_depth
|
|
1137
|
+
-- Filter by relationship type if specified
|
|
1138
|
+
AND (p_rel_type IS NULL OR (edge->>'rel_type')::VARCHAR(100) = p_rel_type)
|
|
1139
|
+
-- Prevent cycles by checking path
|
|
1140
|
+
AND NOT (target_kv.entity_key = ANY(gt.path))
|
|
1141
|
+
)
|
|
1142
|
+
SELECT DISTINCT ON (entity_key)
|
|
1143
|
+
gt.depth,
|
|
1144
|
+
gt.entity_key,
|
|
1145
|
+
gt.entity_type,
|
|
1146
|
+
gt.entity_id,
|
|
1147
|
+
gt.rel_type,
|
|
1148
|
+
gt.rel_weight,
|
|
1149
|
+
gt.path
|
|
1150
|
+
FROM graph_traversal gt
|
|
1151
|
+
WHERE gt.depth > 0 -- Exclude starting entity
|
|
1152
|
+
ORDER BY gt.entity_key, gt.depth
|
|
1153
|
+
LOOP
|
|
1154
|
+
IF p_keys_only THEN
|
|
1155
|
+
-- Return just graph structure (no entity_record)
|
|
1156
|
+
depth := graph_keys.depth;
|
|
1157
|
+
entity_key := graph_keys.entity_key;
|
|
1158
|
+
entity_type := graph_keys.entity_type;
|
|
1159
|
+
entity_id := graph_keys.entity_id;
|
|
1160
|
+
rel_type := graph_keys.rel_type;
|
|
1161
|
+
rel_weight := graph_keys.rel_weight;
|
|
1162
|
+
path := graph_keys.path;
|
|
1163
|
+
entity_record := NULL;
|
|
1164
|
+
RETURN NEXT;
|
|
1165
|
+
ELSE
|
|
1166
|
+
-- Build JSONB mapping {table: [keys]} for batch fetch
|
|
1167
|
+
IF entities_by_table ? graph_keys.entity_type THEN
|
|
1168
|
+
table_keys := entities_by_table->graph_keys.entity_type;
|
|
1169
|
+
entities_by_table := jsonb_set(
|
|
1170
|
+
entities_by_table,
|
|
1171
|
+
ARRAY[graph_keys.entity_type],
|
|
1172
|
+
table_keys || jsonb_build_array(graph_keys.entity_key)
|
|
1173
|
+
);
|
|
1174
|
+
ELSE
|
|
1175
|
+
entities_by_table := jsonb_set(
|
|
1176
|
+
entities_by_table,
|
|
1177
|
+
ARRAY[graph_keys.entity_type],
|
|
1178
|
+
jsonb_build_array(graph_keys.entity_key)
|
|
1179
|
+
);
|
|
1180
|
+
END IF;
|
|
1181
|
+
END IF;
|
|
1182
|
+
END LOOP;
|
|
1183
|
+
|
|
1184
|
+
-- If keys_only=false, fetch full records using rem_fetch
|
|
1185
|
+
IF NOT p_keys_only AND entities_by_table != '{}'::jsonb THEN
|
|
1186
|
+
RETURN QUERY
|
|
1187
|
+
SELECT
|
|
1188
|
+
NULL::INTEGER AS depth,
|
|
1189
|
+
f.entity_key::VARCHAR(255),
|
|
1190
|
+
f.entity_type::VARCHAR(100),
|
|
1191
|
+
NULL::UUID AS entity_id,
|
|
1192
|
+
NULL::VARCHAR(100) AS rel_type,
|
|
1193
|
+
NULL::REAL AS rel_weight,
|
|
1194
|
+
NULL::TEXT[] AS path,
|
|
1195
|
+
f.entity_record
|
|
1196
|
+
FROM rem_fetch(entities_by_table, p_user_id) f;
|
|
1197
|
+
END IF;
|
|
1198
|
+
END;
|
|
1199
|
+
$$ LANGUAGE plpgsql STABLE;
|
|
1200
|
+
|
|
1201
|
+
COMMENT ON FUNCTION rem_traverse IS
|
|
1202
|
+
'REM TRAVERSE query: Recursive graph traversal using cached kv_store.graph_edges. When keys_only=false (default), automatically fetches full entity records via rem_fetch.';
|