hindsight-api 0.2.1__py3-none-any.whl → 0.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- hindsight_api/admin/__init__.py +1 -0
- hindsight_api/admin/cli.py +311 -0
- hindsight_api/alembic/versions/f1a2b3c4d5e6_add_memory_links_composite_index.py +44 -0
- hindsight_api/alembic/versions/g2a3b4c5d6e7_add_tags_column.py +48 -0
- hindsight_api/alembic/versions/h3c4d5e6f7g8_mental_models_v4.py +112 -0
- hindsight_api/alembic/versions/i4d5e6f7g8h9_delete_opinions.py +41 -0
- hindsight_api/alembic/versions/j5e6f7g8h9i0_mental_model_versions.py +95 -0
- hindsight_api/alembic/versions/k6f7g8h9i0j1_add_directive_subtype.py +58 -0
- hindsight_api/alembic/versions/l7g8h9i0j1k2_add_worker_columns.py +109 -0
- hindsight_api/alembic/versions/m8h9i0j1k2l3_mental_model_id_to_text.py +41 -0
- hindsight_api/alembic/versions/n9i0j1k2l3m4_learnings_and_pinned_reflections.py +134 -0
- hindsight_api/alembic/versions/o0j1k2l3m4n5_migrate_mental_models_data.py +113 -0
- hindsight_api/alembic/versions/p1k2l3m4n5o6_new_knowledge_architecture.py +194 -0
- hindsight_api/alembic/versions/q2l3m4n5o6p7_fix_mental_model_fact_type.py +50 -0
- hindsight_api/alembic/versions/r3m4n5o6p7q8_add_reflect_response_to_reflections.py +47 -0
- hindsight_api/alembic/versions/s4n5o6p7q8r9_add_consolidated_at_to_memory_units.py +53 -0
- hindsight_api/alembic/versions/t5o6p7q8r9s0_rename_mental_models_to_observations.py +134 -0
- hindsight_api/alembic/versions/u6p7q8r9s0t1_mental_models_text_id.py +41 -0
- hindsight_api/alembic/versions/v7q8r9s0t1u2_add_max_tokens_to_mental_models.py +50 -0
- hindsight_api/api/http.py +1406 -118
- hindsight_api/api/mcp.py +11 -196
- hindsight_api/config.py +359 -27
- hindsight_api/engine/consolidation/__init__.py +5 -0
- hindsight_api/engine/consolidation/consolidator.py +859 -0
- hindsight_api/engine/consolidation/prompts.py +69 -0
- hindsight_api/engine/cross_encoder.py +706 -88
- hindsight_api/engine/db_budget.py +284 -0
- hindsight_api/engine/db_utils.py +11 -0
- hindsight_api/engine/directives/__init__.py +5 -0
- hindsight_api/engine/directives/models.py +37 -0
- hindsight_api/engine/embeddings.py +553 -29
- hindsight_api/engine/entity_resolver.py +8 -5
- hindsight_api/engine/interface.py +40 -17
- hindsight_api/engine/llm_wrapper.py +744 -68
- hindsight_api/engine/memory_engine.py +2505 -1017
- hindsight_api/engine/mental_models/__init__.py +14 -0
- hindsight_api/engine/mental_models/models.py +53 -0
- hindsight_api/engine/query_analyzer.py +4 -3
- hindsight_api/engine/reflect/__init__.py +18 -0
- hindsight_api/engine/reflect/agent.py +933 -0
- hindsight_api/engine/reflect/models.py +109 -0
- hindsight_api/engine/reflect/observations.py +186 -0
- hindsight_api/engine/reflect/prompts.py +483 -0
- hindsight_api/engine/reflect/tools.py +437 -0
- hindsight_api/engine/reflect/tools_schema.py +250 -0
- hindsight_api/engine/response_models.py +168 -4
- hindsight_api/engine/retain/bank_utils.py +79 -201
- hindsight_api/engine/retain/fact_extraction.py +424 -195
- hindsight_api/engine/retain/fact_storage.py +35 -12
- hindsight_api/engine/retain/link_utils.py +29 -24
- hindsight_api/engine/retain/orchestrator.py +24 -43
- hindsight_api/engine/retain/types.py +11 -2
- hindsight_api/engine/search/graph_retrieval.py +43 -14
- hindsight_api/engine/search/link_expansion_retrieval.py +391 -0
- hindsight_api/engine/search/mpfp_retrieval.py +362 -117
- hindsight_api/engine/search/reranking.py +2 -2
- hindsight_api/engine/search/retrieval.py +848 -201
- hindsight_api/engine/search/tags.py +172 -0
- hindsight_api/engine/search/think_utils.py +42 -141
- hindsight_api/engine/search/trace.py +12 -1
- hindsight_api/engine/search/tracer.py +26 -6
- hindsight_api/engine/search/types.py +21 -3
- hindsight_api/engine/task_backend.py +113 -106
- hindsight_api/engine/utils.py +1 -152
- hindsight_api/extensions/__init__.py +10 -1
- hindsight_api/extensions/builtin/tenant.py +5 -1
- hindsight_api/extensions/context.py +10 -1
- hindsight_api/extensions/operation_validator.py +81 -4
- hindsight_api/extensions/tenant.py +26 -0
- hindsight_api/main.py +69 -6
- hindsight_api/mcp_local.py +12 -53
- hindsight_api/mcp_tools.py +494 -0
- hindsight_api/metrics.py +433 -48
- hindsight_api/migrations.py +141 -1
- hindsight_api/models.py +3 -3
- hindsight_api/pg0.py +53 -0
- hindsight_api/server.py +39 -2
- hindsight_api/worker/__init__.py +11 -0
- hindsight_api/worker/main.py +296 -0
- hindsight_api/worker/poller.py +486 -0
- {hindsight_api-0.2.1.dist-info → hindsight_api-0.4.0.dist-info}/METADATA +16 -6
- hindsight_api-0.4.0.dist-info/RECORD +112 -0
- {hindsight_api-0.2.1.dist-info → hindsight_api-0.4.0.dist-info}/entry_points.txt +2 -0
- hindsight_api/engine/retain/observation_regeneration.py +0 -254
- hindsight_api/engine/search/observation_utils.py +0 -125
- hindsight_api/engine/search/scoring.py +0 -159
- hindsight_api-0.2.1.dist-info/RECORD +0 -75
- {hindsight_api-0.2.1.dist-info → hindsight_api-0.4.0.dist-info}/WHEEL +0 -0
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
"""add_directive_subtype
|
|
2
|
+
|
|
3
|
+
Revision ID: k6f7g8h9i0j1
|
|
4
|
+
Revises: j5e6f7g8h9i0
|
|
5
|
+
Create Date: 2026-01-16 00:00:00.000000
|
|
6
|
+
|
|
7
|
+
This migration adds 'directive' to the mental_models subtype constraint.
|
|
8
|
+
Directives are hard rules with user-provided observations that the reflect agent must follow.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from collections.abc import Sequence
|
|
12
|
+
|
|
13
|
+
from alembic import context, op
|
|
14
|
+
|
|
15
|
+
# revision identifiers, used by Alembic.
|
|
16
|
+
revision: str = "k6f7g8h9i0j1"
|
|
17
|
+
down_revision: str | Sequence[str] | None = "j5e6f7g8h9i0"
|
|
18
|
+
branch_labels: str | Sequence[str] | None = None
|
|
19
|
+
depends_on: str | Sequence[str] | None = None
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _get_schema_prefix() -> str:
|
|
23
|
+
"""Get schema prefix for table names (required for multi-tenant support)."""
|
|
24
|
+
schema = context.config.get_main_option("target_schema")
|
|
25
|
+
return f'"{schema}".' if schema else ""
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def upgrade() -> None:
|
|
29
|
+
"""Add 'directive' to mental_models subtype constraint."""
|
|
30
|
+
schema = _get_schema_prefix()
|
|
31
|
+
|
|
32
|
+
# Drop existing constraint
|
|
33
|
+
op.execute(f"ALTER TABLE {schema}mental_models DROP CONSTRAINT IF EXISTS ck_mental_models_subtype")
|
|
34
|
+
|
|
35
|
+
# Create new constraint with 'directive' added
|
|
36
|
+
op.execute(f"""
|
|
37
|
+
ALTER TABLE {schema}mental_models
|
|
38
|
+
ADD CONSTRAINT ck_mental_models_subtype
|
|
39
|
+
CHECK (subtype IN ('structural', 'emergent', 'pinned', 'learned', 'directive'))
|
|
40
|
+
""")
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def downgrade() -> None:
|
|
44
|
+
"""Remove 'directive' from mental_models subtype constraint."""
|
|
45
|
+
schema = _get_schema_prefix()
|
|
46
|
+
|
|
47
|
+
# First delete any directives (cannot downgrade if they exist)
|
|
48
|
+
op.execute(f"DELETE FROM {schema}mental_models WHERE subtype = 'directive'")
|
|
49
|
+
|
|
50
|
+
# Drop constraint with directive
|
|
51
|
+
op.execute(f"ALTER TABLE {schema}mental_models DROP CONSTRAINT IF EXISTS ck_mental_models_subtype")
|
|
52
|
+
|
|
53
|
+
# Recreate original constraint without directive
|
|
54
|
+
op.execute(f"""
|
|
55
|
+
ALTER TABLE {schema}mental_models
|
|
56
|
+
ADD CONSTRAINT ck_mental_models_subtype
|
|
57
|
+
CHECK (subtype IN ('structural', 'emergent', 'pinned', 'learned'))
|
|
58
|
+
""")
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
"""add_worker_columns
|
|
2
|
+
|
|
3
|
+
Revision ID: l7g8h9i0j1k2
|
|
4
|
+
Revises: k6f7g8h9i0j1
|
|
5
|
+
Create Date: 2026-01-19 00:00:00.000000
|
|
6
|
+
|
|
7
|
+
This migration adds columns to async_operations for distributed worker support:
|
|
8
|
+
- worker_id: ID of the worker that claimed the task
|
|
9
|
+
- claimed_at: When the task was claimed
|
|
10
|
+
- retry_count: Number of retry attempts
|
|
11
|
+
- task_payload: The serialized task dictionary
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from collections.abc import Sequence
|
|
15
|
+
|
|
16
|
+
import sqlalchemy as sa
|
|
17
|
+
from alembic import context, op
|
|
18
|
+
from sqlalchemy.dialects import postgresql
|
|
19
|
+
|
|
20
|
+
# revision identifiers, used by Alembic.
|
|
21
|
+
revision: str = "l7g8h9i0j1k2"
|
|
22
|
+
down_revision: str | Sequence[str] | None = "k6f7g8h9i0j1"
|
|
23
|
+
branch_labels: str | Sequence[str] | None = None
|
|
24
|
+
depends_on: str | Sequence[str] | None = None
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def _get_schema_prefix() -> str:
|
|
28
|
+
"""Get schema prefix for table names (required for multi-tenant support)."""
|
|
29
|
+
schema = context.config.get_main_option("target_schema")
|
|
30
|
+
return f'"{schema}".' if schema else ""
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def upgrade() -> None:
|
|
34
|
+
"""Add worker columns to async_operations."""
|
|
35
|
+
schema = _get_schema_prefix()
|
|
36
|
+
|
|
37
|
+
# Add worker_id column (ID of worker that claimed the task)
|
|
38
|
+
op.add_column(
|
|
39
|
+
"async_operations",
|
|
40
|
+
sa.Column("worker_id", sa.Text(), nullable=True),
|
|
41
|
+
schema=context.config.get_main_option("target_schema") or None,
|
|
42
|
+
)
|
|
43
|
+
|
|
44
|
+
# Add claimed_at column (when task was claimed by worker)
|
|
45
|
+
op.add_column(
|
|
46
|
+
"async_operations",
|
|
47
|
+
sa.Column("claimed_at", postgresql.TIMESTAMP(timezone=True), nullable=True),
|
|
48
|
+
schema=context.config.get_main_option("target_schema") or None,
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
# Add retry_count column (number of retry attempts)
|
|
52
|
+
op.add_column(
|
|
53
|
+
"async_operations",
|
|
54
|
+
sa.Column("retry_count", sa.Integer(), server_default="0", nullable=False),
|
|
55
|
+
schema=context.config.get_main_option("target_schema") or None,
|
|
56
|
+
)
|
|
57
|
+
|
|
58
|
+
# Add task_payload column (serialized task dictionary)
|
|
59
|
+
op.add_column(
|
|
60
|
+
"async_operations",
|
|
61
|
+
sa.Column(
|
|
62
|
+
"task_payload",
|
|
63
|
+
postgresql.JSONB(astext_type=sa.Text()),
|
|
64
|
+
nullable=True,
|
|
65
|
+
),
|
|
66
|
+
schema=context.config.get_main_option("target_schema") or None,
|
|
67
|
+
)
|
|
68
|
+
|
|
69
|
+
# Add index for efficient worker polling (pending tasks ordered by creation time)
|
|
70
|
+
op.execute(
|
|
71
|
+
f"CREATE INDEX idx_async_operations_pending_claim ON {schema}async_operations (status, created_at) "
|
|
72
|
+
f"WHERE status = 'pending' AND task_payload IS NOT NULL"
|
|
73
|
+
)
|
|
74
|
+
|
|
75
|
+
# Add index for finding tasks by worker_id (for decommissioning)
|
|
76
|
+
op.execute(
|
|
77
|
+
f"CREATE INDEX idx_async_operations_worker_id ON {schema}async_operations (worker_id) WHERE worker_id IS NOT NULL"
|
|
78
|
+
)
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def downgrade() -> None:
|
|
82
|
+
"""Remove worker columns from async_operations."""
|
|
83
|
+
schema = _get_schema_prefix()
|
|
84
|
+
|
|
85
|
+
# Drop indexes
|
|
86
|
+
op.execute(f"DROP INDEX IF EXISTS {schema}idx_async_operations_pending_claim")
|
|
87
|
+
op.execute(f"DROP INDEX IF EXISTS {schema}idx_async_operations_worker_id")
|
|
88
|
+
|
|
89
|
+
# Drop columns
|
|
90
|
+
op.drop_column(
|
|
91
|
+
"async_operations",
|
|
92
|
+
"task_payload",
|
|
93
|
+
schema=context.config.get_main_option("target_schema") or None,
|
|
94
|
+
)
|
|
95
|
+
op.drop_column(
|
|
96
|
+
"async_operations",
|
|
97
|
+
"retry_count",
|
|
98
|
+
schema=context.config.get_main_option("target_schema") or None,
|
|
99
|
+
)
|
|
100
|
+
op.drop_column(
|
|
101
|
+
"async_operations",
|
|
102
|
+
"claimed_at",
|
|
103
|
+
schema=context.config.get_main_option("target_schema") or None,
|
|
104
|
+
)
|
|
105
|
+
op.drop_column(
|
|
106
|
+
"async_operations",
|
|
107
|
+
"worker_id",
|
|
108
|
+
schema=context.config.get_main_option("target_schema") or None,
|
|
109
|
+
)
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
"""mental_model_id_to_text
|
|
2
|
+
|
|
3
|
+
Revision ID: m8h9i0j1k2l3
|
|
4
|
+
Revises: l7g8h9i0j1k2
|
|
5
|
+
Create Date: 2026-01-19 00:00:00.000000
|
|
6
|
+
|
|
7
|
+
This migration changes the mental_models.id column from VARCHAR(64) to TEXT
|
|
8
|
+
to support longer model IDs (e.g., entity names that exceed 64 characters).
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from collections.abc import Sequence
|
|
12
|
+
|
|
13
|
+
from alembic import context, op
|
|
14
|
+
|
|
15
|
+
# revision identifiers, used by Alembic.
|
|
16
|
+
revision: str = "m8h9i0j1k2l3"
|
|
17
|
+
down_revision: str | Sequence[str] | None = "l7g8h9i0j1k2"
|
|
18
|
+
branch_labels: str | Sequence[str] | None = None
|
|
19
|
+
depends_on: str | Sequence[str] | None = None
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _get_schema_prefix() -> str:
|
|
23
|
+
"""Get schema prefix for table names (required for multi-tenant support)."""
|
|
24
|
+
schema = context.config.get_main_option("target_schema")
|
|
25
|
+
return f'"{schema}".' if schema else ""
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def upgrade() -> None:
|
|
29
|
+
"""Change mental_models.id from VARCHAR(64) to TEXT."""
|
|
30
|
+
schema = _get_schema_prefix()
|
|
31
|
+
|
|
32
|
+
# Alter the id column type from VARCHAR(64) to TEXT
|
|
33
|
+
op.execute(f"ALTER TABLE {schema}mental_models ALTER COLUMN id TYPE TEXT")
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def downgrade() -> None:
|
|
37
|
+
"""Revert mental_models.id from TEXT to VARCHAR(64)."""
|
|
38
|
+
schema = _get_schema_prefix()
|
|
39
|
+
|
|
40
|
+
# Note: This may fail if any id values exceed 64 characters
|
|
41
|
+
op.execute(f"ALTER TABLE {schema}mental_models ALTER COLUMN id TYPE VARCHAR(64)")
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
"""learnings_and_pinned_reflections
|
|
2
|
+
|
|
3
|
+
Revision ID: n9i0j1k2l3m4
|
|
4
|
+
Revises: m8h9i0j1k2l3
|
|
5
|
+
Create Date: 2026-01-21 00:00:00.000000
|
|
6
|
+
|
|
7
|
+
This migration:
|
|
8
|
+
1. Creates the 'learnings' table for automatic bottom-up consolidation
|
|
9
|
+
2. Creates the 'pinned_reflections' table for user-curated living documents
|
|
10
|
+
3. Adds consolidation tracking columns to the 'banks' table
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from collections.abc import Sequence
|
|
14
|
+
|
|
15
|
+
from alembic import context, op
|
|
16
|
+
|
|
17
|
+
# revision identifiers, used by Alembic.
|
|
18
|
+
revision: str = "n9i0j1k2l3m4"
|
|
19
|
+
down_revision: str | Sequence[str] | None = "m8h9i0j1k2l3"
|
|
20
|
+
branch_labels: str | Sequence[str] | None = None
|
|
21
|
+
depends_on: str | Sequence[str] | None = None
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _get_schema_prefix() -> str:
|
|
25
|
+
"""Get schema prefix for table names (required for multi-tenant support)."""
|
|
26
|
+
schema = context.config.get_main_option("target_schema")
|
|
27
|
+
return f'"{schema}".' if schema else ""
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def upgrade() -> None:
|
|
31
|
+
"""Create learnings and pinned_reflections tables."""
|
|
32
|
+
schema = _get_schema_prefix()
|
|
33
|
+
|
|
34
|
+
# 1. Create learnings table
|
|
35
|
+
op.execute(f"""
|
|
36
|
+
CREATE TABLE {schema}learnings (
|
|
37
|
+
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
|
38
|
+
bank_id VARCHAR(64) NOT NULL,
|
|
39
|
+
text TEXT NOT NULL,
|
|
40
|
+
proof_count INT NOT NULL DEFAULT 1,
|
|
41
|
+
history JSONB DEFAULT '[]'::jsonb,
|
|
42
|
+
mission_context VARCHAR(64),
|
|
43
|
+
pre_mission_change BOOLEAN DEFAULT FALSE,
|
|
44
|
+
embedding vector(384),
|
|
45
|
+
tags VARCHAR[] DEFAULT ARRAY[]::VARCHAR[],
|
|
46
|
+
created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(),
|
|
47
|
+
updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now()
|
|
48
|
+
)
|
|
49
|
+
""")
|
|
50
|
+
|
|
51
|
+
# Add foreign key constraint
|
|
52
|
+
op.execute(f"""
|
|
53
|
+
ALTER TABLE {schema}learnings
|
|
54
|
+
ADD CONSTRAINT fk_learnings_bank_id
|
|
55
|
+
FOREIGN KEY (bank_id) REFERENCES {schema}banks(bank_id) ON DELETE CASCADE
|
|
56
|
+
""")
|
|
57
|
+
|
|
58
|
+
# Indexes for learnings
|
|
59
|
+
op.execute(f"CREATE INDEX idx_learnings_bank_id ON {schema}learnings(bank_id)")
|
|
60
|
+
op.execute(f"""
|
|
61
|
+
CREATE INDEX idx_learnings_embedding ON {schema}learnings
|
|
62
|
+
USING hnsw (embedding vector_cosine_ops)
|
|
63
|
+
""")
|
|
64
|
+
op.execute(f"CREATE INDEX idx_learnings_tags ON {schema}learnings USING GIN(tags)")
|
|
65
|
+
|
|
66
|
+
# Full-text search for learnings
|
|
67
|
+
op.execute(f"""
|
|
68
|
+
ALTER TABLE {schema}learnings ADD COLUMN search_vector tsvector
|
|
69
|
+
GENERATED ALWAYS AS (to_tsvector('english', text)) STORED
|
|
70
|
+
""")
|
|
71
|
+
op.execute(f"CREATE INDEX idx_learnings_text_search ON {schema}learnings USING gin(search_vector)")
|
|
72
|
+
|
|
73
|
+
# 2. Create pinned_reflections table
|
|
74
|
+
op.execute(f"""
|
|
75
|
+
CREATE TABLE {schema}pinned_reflections (
|
|
76
|
+
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
|
77
|
+
bank_id VARCHAR(64) NOT NULL,
|
|
78
|
+
name VARCHAR(256) NOT NULL,
|
|
79
|
+
source_query TEXT NOT NULL,
|
|
80
|
+
content TEXT NOT NULL,
|
|
81
|
+
embedding vector(384),
|
|
82
|
+
tags VARCHAR[] DEFAULT ARRAY[]::VARCHAR[],
|
|
83
|
+
last_refreshed_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(),
|
|
84
|
+
created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now()
|
|
85
|
+
)
|
|
86
|
+
""")
|
|
87
|
+
|
|
88
|
+
# Add foreign key constraint
|
|
89
|
+
op.execute(f"""
|
|
90
|
+
ALTER TABLE {schema}pinned_reflections
|
|
91
|
+
ADD CONSTRAINT fk_pinned_reflections_bank_id
|
|
92
|
+
FOREIGN KEY (bank_id) REFERENCES {schema}banks(bank_id) ON DELETE CASCADE
|
|
93
|
+
""")
|
|
94
|
+
|
|
95
|
+
# Indexes for pinned_reflections
|
|
96
|
+
op.execute(f"CREATE INDEX idx_pinned_reflections_bank_id ON {schema}pinned_reflections(bank_id)")
|
|
97
|
+
op.execute(f"""
|
|
98
|
+
CREATE INDEX idx_pinned_reflections_embedding ON {schema}pinned_reflections
|
|
99
|
+
USING hnsw (embedding vector_cosine_ops)
|
|
100
|
+
""")
|
|
101
|
+
op.execute(f"CREATE INDEX idx_pinned_reflections_tags ON {schema}pinned_reflections USING GIN(tags)")
|
|
102
|
+
|
|
103
|
+
# Full-text search for pinned_reflections
|
|
104
|
+
op.execute(f"""
|
|
105
|
+
ALTER TABLE {schema}pinned_reflections ADD COLUMN search_vector tsvector
|
|
106
|
+
GENERATED ALWAYS AS (to_tsvector('english', COALESCE(name, '') || ' ' || content)) STORED
|
|
107
|
+
""")
|
|
108
|
+
op.execute(f"""
|
|
109
|
+
CREATE INDEX idx_pinned_reflections_text_search ON {schema}pinned_reflections
|
|
110
|
+
USING gin(search_vector)
|
|
111
|
+
""")
|
|
112
|
+
|
|
113
|
+
# 3. Add consolidation tracking columns to banks table
|
|
114
|
+
op.execute(f"""
|
|
115
|
+
ALTER TABLE {schema}banks
|
|
116
|
+
ADD COLUMN IF NOT EXISTS last_consolidated_at TIMESTAMP WITH TIME ZONE
|
|
117
|
+
""")
|
|
118
|
+
op.execute(f"""
|
|
119
|
+
ALTER TABLE {schema}banks
|
|
120
|
+
ADD COLUMN IF NOT EXISTS mission_changed_at TIMESTAMP WITH TIME ZONE
|
|
121
|
+
""")
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def downgrade() -> None:
|
|
125
|
+
"""Drop learnings and pinned_reflections tables."""
|
|
126
|
+
schema = _get_schema_prefix()
|
|
127
|
+
|
|
128
|
+
# Drop tables
|
|
129
|
+
op.execute(f"DROP TABLE IF EXISTS {schema}learnings CASCADE")
|
|
130
|
+
op.execute(f"DROP TABLE IF EXISTS {schema}pinned_reflections CASCADE")
|
|
131
|
+
|
|
132
|
+
# Remove columns from banks
|
|
133
|
+
op.execute(f"ALTER TABLE {schema}banks DROP COLUMN IF EXISTS last_consolidated_at")
|
|
134
|
+
op.execute(f"ALTER TABLE {schema}banks DROP COLUMN IF EXISTS mission_changed_at")
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
"""migrate_mental_models_data
|
|
2
|
+
|
|
3
|
+
Revision ID: o0j1k2l3m4n5
|
|
4
|
+
Revises: n9i0j1k2l3m4
|
|
5
|
+
Create Date: 2026-01-21 00:00:00.000000
|
|
6
|
+
|
|
7
|
+
This migration:
|
|
8
|
+
1. Migrates existing 'pinned' mental models to the new 'pinned_reflections' table
|
|
9
|
+
2. Migrates existing 'learned' mental models to the new 'learnings' table
|
|
10
|
+
3. Deletes non-directive mental models (structural, emergent, pinned, learned)
|
|
11
|
+
4. Drops the mental_model_versions table (no longer used)
|
|
12
|
+
5. Adds a CHECK constraint that only 'directive' subtype is allowed
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from collections.abc import Sequence
|
|
16
|
+
|
|
17
|
+
from alembic import context, op
|
|
18
|
+
|
|
19
|
+
# revision identifiers, used by Alembic.
|
|
20
|
+
revision: str = "o0j1k2l3m4n5"
|
|
21
|
+
down_revision: str | Sequence[str] | None = "n9i0j1k2l3m4"
|
|
22
|
+
branch_labels: str | Sequence[str] | None = None
|
|
23
|
+
depends_on: str | Sequence[str] | None = None
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _get_schema_prefix() -> str:
|
|
27
|
+
"""Get schema prefix for table names (required for multi-tenant support)."""
|
|
28
|
+
schema = context.config.get_main_option("target_schema")
|
|
29
|
+
return f'"{schema}".' if schema else ""
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def upgrade() -> None:
|
|
33
|
+
"""Migrate data and clean up old mental models."""
|
|
34
|
+
schema = _get_schema_prefix()
|
|
35
|
+
|
|
36
|
+
# 1. Migrate 'pinned' mental models to pinned_reflections
|
|
37
|
+
# For pinned models, the first observation's content becomes the pinned reflection content
|
|
38
|
+
op.execute(f"""
|
|
39
|
+
INSERT INTO {schema}pinned_reflections (bank_id, name, source_query, content, tags, created_at)
|
|
40
|
+
SELECT
|
|
41
|
+
bank_id,
|
|
42
|
+
name,
|
|
43
|
+
description AS source_query,
|
|
44
|
+
COALESCE(
|
|
45
|
+
observations->'observations'->0->>'content',
|
|
46
|
+
description,
|
|
47
|
+
''
|
|
48
|
+
) AS content,
|
|
49
|
+
tags,
|
|
50
|
+
created_at
|
|
51
|
+
FROM {schema}mental_models
|
|
52
|
+
WHERE subtype = 'pinned'
|
|
53
|
+
ON CONFLICT DO NOTHING
|
|
54
|
+
""")
|
|
55
|
+
|
|
56
|
+
# 2. Migrate 'learned' mental models to learnings
|
|
57
|
+
# Each observation in a learned model becomes a separate learning
|
|
58
|
+
op.execute(f"""
|
|
59
|
+
INSERT INTO {schema}learnings (bank_id, text, proof_count, tags, created_at)
|
|
60
|
+
SELECT
|
|
61
|
+
mm.bank_id,
|
|
62
|
+
obs->>'content' AS text,
|
|
63
|
+
GREATEST(1, COALESCE(jsonb_array_length(obs->'evidence'), 1)) AS proof_count,
|
|
64
|
+
mm.tags,
|
|
65
|
+
mm.created_at
|
|
66
|
+
FROM {schema}mental_models mm,
|
|
67
|
+
LATERAL jsonb_array_elements(mm.observations->'observations') AS obs
|
|
68
|
+
WHERE mm.subtype = 'learned'
|
|
69
|
+
AND obs->>'content' IS NOT NULL
|
|
70
|
+
AND obs->>'content' != ''
|
|
71
|
+
ON CONFLICT DO NOTHING
|
|
72
|
+
""")
|
|
73
|
+
|
|
74
|
+
# 3. Delete all non-directive mental models (they've been migrated or are obsolete)
|
|
75
|
+
op.execute(f"""
|
|
76
|
+
DELETE FROM {schema}mental_models
|
|
77
|
+
WHERE subtype != 'directive'
|
|
78
|
+
""")
|
|
79
|
+
|
|
80
|
+
# 4. Drop the mental_model_versions table (no longer used)
|
|
81
|
+
op.execute(f"DROP TABLE IF EXISTS {schema}mental_model_versions CASCADE")
|
|
82
|
+
|
|
83
|
+
# 5. Drop old constraints and add new one that only allows 'directive'
|
|
84
|
+
op.execute(f"ALTER TABLE {schema}mental_models DROP CONSTRAINT IF EXISTS ck_mental_models_subtype")
|
|
85
|
+
op.execute(f"""
|
|
86
|
+
ALTER TABLE {schema}mental_models
|
|
87
|
+
ADD CONSTRAINT ck_mental_models_subtype CHECK (subtype = 'directive')
|
|
88
|
+
""")
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def downgrade() -> None:
|
|
92
|
+
"""Reverse the migration (data migration is one-way, so this just removes constraints)."""
|
|
93
|
+
schema = _get_schema_prefix()
|
|
94
|
+
|
|
95
|
+
# Remove the directive-only constraint
|
|
96
|
+
op.execute(f"ALTER TABLE {schema}mental_models DROP CONSTRAINT IF EXISTS ck_mental_models_subtype")
|
|
97
|
+
|
|
98
|
+
# Re-create mental_model_versions table
|
|
99
|
+
op.execute(f"""
|
|
100
|
+
CREATE TABLE IF NOT EXISTS {schema}mental_model_versions (
|
|
101
|
+
id SERIAL PRIMARY KEY,
|
|
102
|
+
bank_id VARCHAR(64) NOT NULL,
|
|
103
|
+
model_id VARCHAR(128) NOT NULL,
|
|
104
|
+
version INT NOT NULL,
|
|
105
|
+
observations JSONB NOT NULL,
|
|
106
|
+
created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now()
|
|
107
|
+
)
|
|
108
|
+
""")
|
|
109
|
+
op.execute(
|
|
110
|
+
f"CREATE INDEX IF NOT EXISTS idx_mm_versions_lookup ON {schema}mental_model_versions(bank_id, model_id, version DESC)"
|
|
111
|
+
)
|
|
112
|
+
|
|
113
|
+
# Note: Data migration cannot be reversed - pinned_reflections and learnings data remains
|
|
@@ -0,0 +1,194 @@
|
|
|
1
|
+
"""new_knowledge_architecture
|
|
2
|
+
|
|
3
|
+
Revision ID: p1k2l3m4n5o6
|
|
4
|
+
Revises: o0j1k2l3m4n5
|
|
5
|
+
Create Date: 2026-01-21 00:00:00.000000
|
|
6
|
+
|
|
7
|
+
This migration implements the new knowledge architecture:
|
|
8
|
+
1. Drops the 'learnings' table (mental models are now in memory_units)
|
|
9
|
+
2. Renames 'pinned_reflections' to 'reflections'
|
|
10
|
+
3. Drops the 'mental_models' table completely
|
|
11
|
+
4. Creates 'directives' table for hard rules
|
|
12
|
+
5. Adds mental model support columns to 'memory_units' (proof_count, source_memory_ids, history)
|
|
13
|
+
|
|
14
|
+
The new architecture:
|
|
15
|
+
- Directives: Hard rules in their own table
|
|
16
|
+
- Mental Models: Stored in memory_units with fact_type='mental_model'
|
|
17
|
+
- Reflections: User-curated documents (renamed from pinned_reflections)
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
from collections.abc import Sequence
|
|
21
|
+
|
|
22
|
+
from alembic import context, op
|
|
23
|
+
|
|
24
|
+
# revision identifiers, used by Alembic.
|
|
25
|
+
revision: str = "p1k2l3m4n5o6"
|
|
26
|
+
down_revision: str | Sequence[str] | None = "o0j1k2l3m4n5"
|
|
27
|
+
branch_labels: str | Sequence[str] | None = None
|
|
28
|
+
depends_on: str | Sequence[str] | None = None
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _get_schema_prefix() -> str:
|
|
32
|
+
"""Get schema prefix for table names (required for multi-tenant support)."""
|
|
33
|
+
schema = context.config.get_main_option("target_schema")
|
|
34
|
+
return f'"{schema}".' if schema else ""
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def upgrade() -> None:
|
|
38
|
+
"""Implement new knowledge architecture."""
|
|
39
|
+
schema = _get_schema_prefix()
|
|
40
|
+
|
|
41
|
+
# 1. Drop the learnings table (mental models will be in memory_units)
|
|
42
|
+
op.execute(f"DROP TABLE IF EXISTS {schema}learnings CASCADE")
|
|
43
|
+
|
|
44
|
+
# 2. Rename pinned_reflections to reflections
|
|
45
|
+
op.execute(f"ALTER TABLE IF EXISTS {schema}pinned_reflections RENAME TO reflections")
|
|
46
|
+
|
|
47
|
+
# Rename indexes for reflections
|
|
48
|
+
op.execute(f"ALTER INDEX IF EXISTS {schema}idx_pinned_reflections_bank_id RENAME TO idx_reflections_bank_id")
|
|
49
|
+
op.execute(f"ALTER INDEX IF EXISTS {schema}idx_pinned_reflections_embedding RENAME TO idx_reflections_embedding")
|
|
50
|
+
op.execute(f"ALTER INDEX IF EXISTS {schema}idx_pinned_reflections_tags RENAME TO idx_reflections_tags")
|
|
51
|
+
op.execute(
|
|
52
|
+
f"ALTER INDEX IF EXISTS {schema}idx_pinned_reflections_text_search RENAME TO idx_reflections_text_search"
|
|
53
|
+
)
|
|
54
|
+
|
|
55
|
+
# Rename foreign key constraint
|
|
56
|
+
op.execute(f"""
|
|
57
|
+
ALTER TABLE {schema}reflections
|
|
58
|
+
DROP CONSTRAINT IF EXISTS fk_pinned_reflections_bank_id
|
|
59
|
+
""")
|
|
60
|
+
op.execute(f"""
|
|
61
|
+
ALTER TABLE {schema}reflections
|
|
62
|
+
ADD CONSTRAINT fk_reflections_bank_id
|
|
63
|
+
FOREIGN KEY (bank_id) REFERENCES {schema}banks(bank_id) ON DELETE CASCADE
|
|
64
|
+
""")
|
|
65
|
+
|
|
66
|
+
# 3. Drop the mental_models table completely
|
|
67
|
+
op.execute(f"DROP TABLE IF EXISTS {schema}mental_models CASCADE")
|
|
68
|
+
|
|
69
|
+
# 4. Create directives table
|
|
70
|
+
op.execute(f"""
|
|
71
|
+
CREATE TABLE {schema}directives (
|
|
72
|
+
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
|
73
|
+
bank_id VARCHAR(64) NOT NULL,
|
|
74
|
+
name VARCHAR(256) NOT NULL,
|
|
75
|
+
content TEXT NOT NULL,
|
|
76
|
+
priority INT NOT NULL DEFAULT 0,
|
|
77
|
+
is_active BOOLEAN NOT NULL DEFAULT TRUE,
|
|
78
|
+
tags VARCHAR[] DEFAULT ARRAY[]::VARCHAR[],
|
|
79
|
+
created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(),
|
|
80
|
+
updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now()
|
|
81
|
+
)
|
|
82
|
+
""")
|
|
83
|
+
|
|
84
|
+
# Add foreign key and indexes for directives
|
|
85
|
+
op.execute(f"""
|
|
86
|
+
ALTER TABLE {schema}directives
|
|
87
|
+
ADD CONSTRAINT fk_directives_bank_id
|
|
88
|
+
FOREIGN KEY (bank_id) REFERENCES {schema}banks(bank_id) ON DELETE CASCADE
|
|
89
|
+
""")
|
|
90
|
+
op.execute(f"CREATE INDEX idx_directives_bank_id ON {schema}directives(bank_id)")
|
|
91
|
+
op.execute(f"CREATE INDEX idx_directives_bank_active ON {schema}directives(bank_id, is_active)")
|
|
92
|
+
op.execute(f"CREATE INDEX idx_directives_tags ON {schema}directives USING GIN(tags)")
|
|
93
|
+
|
|
94
|
+
# 5. Add mental model support columns to memory_units
|
|
95
|
+
# proof_count: Number of memories that support this mental model
|
|
96
|
+
op.execute(f"""
|
|
97
|
+
ALTER TABLE {schema}memory_units
|
|
98
|
+
ADD COLUMN IF NOT EXISTS proof_count INT DEFAULT 1
|
|
99
|
+
""")
|
|
100
|
+
|
|
101
|
+
# source_memory_ids: Array of memory IDs that consolidated into this mental model
|
|
102
|
+
op.execute(f"""
|
|
103
|
+
ALTER TABLE {schema}memory_units
|
|
104
|
+
ADD COLUMN IF NOT EXISTS source_memory_ids UUID[] DEFAULT ARRAY[]::UUID[]
|
|
105
|
+
""")
|
|
106
|
+
|
|
107
|
+
# history: JSONB array tracking changes to mental models
|
|
108
|
+
op.execute(f"""
|
|
109
|
+
ALTER TABLE {schema}memory_units
|
|
110
|
+
ADD COLUMN IF NOT EXISTS history JSONB DEFAULT '[]'::jsonb
|
|
111
|
+
""")
|
|
112
|
+
|
|
113
|
+
# Add index for finding mental models
|
|
114
|
+
op.execute(f"""
|
|
115
|
+
CREATE INDEX IF NOT EXISTS idx_memory_units_mental_models
|
|
116
|
+
ON {schema}memory_units(bank_id, fact_type)
|
|
117
|
+
WHERE fact_type = 'mental_model'
|
|
118
|
+
""")
|
|
119
|
+
|
|
120
|
+
# 6. Update fact_type check constraint to include 'mental_model'
|
|
121
|
+
op.execute(f"ALTER TABLE {schema}memory_units DROP CONSTRAINT IF EXISTS memory_units_fact_type_check")
|
|
122
|
+
op.execute(f"""
|
|
123
|
+
ALTER TABLE {schema}memory_units
|
|
124
|
+
ADD CONSTRAINT memory_units_fact_type_check
|
|
125
|
+
CHECK (fact_type IN ('world', 'experience', 'opinion', 'observation', 'mental_model'))
|
|
126
|
+
""")
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def downgrade() -> None:
|
|
130
|
+
"""Reverse the migration."""
|
|
131
|
+
schema = _get_schema_prefix()
|
|
132
|
+
|
|
133
|
+
# Restore original fact_type check constraint (without 'mental_model')
|
|
134
|
+
op.execute(f"ALTER TABLE {schema}memory_units DROP CONSTRAINT IF EXISTS memory_units_fact_type_check")
|
|
135
|
+
op.execute(f"""
|
|
136
|
+
ALTER TABLE {schema}memory_units
|
|
137
|
+
ADD CONSTRAINT memory_units_fact_type_check
|
|
138
|
+
CHECK (fact_type IN ('world', 'experience', 'opinion', 'observation'))
|
|
139
|
+
""")
|
|
140
|
+
|
|
141
|
+
# Drop mental model columns from memory_units
|
|
142
|
+
op.execute(f"ALTER TABLE {schema}memory_units DROP COLUMN IF EXISTS proof_count")
|
|
143
|
+
op.execute(f"ALTER TABLE {schema}memory_units DROP COLUMN IF EXISTS source_memory_ids")
|
|
144
|
+
op.execute(f"ALTER TABLE {schema}memory_units DROP COLUMN IF EXISTS history")
|
|
145
|
+
op.execute(f"DROP INDEX IF EXISTS {schema}idx_memory_units_mental_models")
|
|
146
|
+
|
|
147
|
+
# Drop directives table
|
|
148
|
+
op.execute(f"DROP TABLE IF EXISTS {schema}directives CASCADE")
|
|
149
|
+
|
|
150
|
+
# Rename reflections back to pinned_reflections
|
|
151
|
+
op.execute(f"ALTER TABLE IF EXISTS {schema}reflections RENAME TO pinned_reflections")
|
|
152
|
+
|
|
153
|
+
# Restore indexes
|
|
154
|
+
op.execute(f"ALTER INDEX IF EXISTS {schema}idx_reflections_bank_id RENAME TO idx_pinned_reflections_bank_id")
|
|
155
|
+
op.execute(f"ALTER INDEX IF EXISTS {schema}idx_reflections_embedding RENAME TO idx_pinned_reflections_embedding")
|
|
156
|
+
op.execute(f"ALTER INDEX IF EXISTS {schema}idx_reflections_tags RENAME TO idx_pinned_reflections_tags")
|
|
157
|
+
op.execute(
|
|
158
|
+
f"ALTER INDEX IF EXISTS {schema}idx_reflections_text_search RENAME TO idx_pinned_reflections_text_search"
|
|
159
|
+
)
|
|
160
|
+
|
|
161
|
+
# Restore foreign key
|
|
162
|
+
op.execute(f"""
|
|
163
|
+
ALTER TABLE {schema}pinned_reflections
|
|
164
|
+
DROP CONSTRAINT IF EXISTS fk_reflections_bank_id
|
|
165
|
+
""")
|
|
166
|
+
op.execute(f"""
|
|
167
|
+
ALTER TABLE {schema}pinned_reflections
|
|
168
|
+
ADD CONSTRAINT fk_pinned_reflections_bank_id
|
|
169
|
+
FOREIGN KEY (bank_id) REFERENCES {schema}banks(bank_id) ON DELETE CASCADE
|
|
170
|
+
""")
|
|
171
|
+
|
|
172
|
+
# Re-create learnings table
|
|
173
|
+
op.execute(f"""
|
|
174
|
+
CREATE TABLE IF NOT EXISTS {schema}learnings (
|
|
175
|
+
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
|
176
|
+
bank_id VARCHAR(64) NOT NULL,
|
|
177
|
+
text TEXT NOT NULL,
|
|
178
|
+
proof_count INT NOT NULL DEFAULT 1,
|
|
179
|
+
history JSONB DEFAULT '[]'::jsonb,
|
|
180
|
+
mission_context VARCHAR(64),
|
|
181
|
+
pre_mission_change BOOLEAN DEFAULT FALSE,
|
|
182
|
+
embedding vector(384),
|
|
183
|
+
tags VARCHAR[] DEFAULT ARRAY[]::VARCHAR[],
|
|
184
|
+
created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(),
|
|
185
|
+
updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now()
|
|
186
|
+
)
|
|
187
|
+
""")
|
|
188
|
+
op.execute(f"""
|
|
189
|
+
ALTER TABLE {schema}learnings
|
|
190
|
+
ADD CONSTRAINT fk_learnings_bank_id
|
|
191
|
+
FOREIGN KEY (bank_id) REFERENCES {schema}banks(bank_id) ON DELETE CASCADE
|
|
192
|
+
""")
|
|
193
|
+
|
|
194
|
+
# Note: mental_models table recreation is complex and would need separate handling
|