hindsight-api 0.2.1__py3-none-any.whl → 0.4.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (88) hide show
  1. hindsight_api/admin/__init__.py +1 -0
  2. hindsight_api/admin/cli.py +311 -0
  3. hindsight_api/alembic/versions/f1a2b3c4d5e6_add_memory_links_composite_index.py +44 -0
  4. hindsight_api/alembic/versions/g2a3b4c5d6e7_add_tags_column.py +48 -0
  5. hindsight_api/alembic/versions/h3c4d5e6f7g8_mental_models_v4.py +112 -0
  6. hindsight_api/alembic/versions/i4d5e6f7g8h9_delete_opinions.py +41 -0
  7. hindsight_api/alembic/versions/j5e6f7g8h9i0_mental_model_versions.py +95 -0
  8. hindsight_api/alembic/versions/k6f7g8h9i0j1_add_directive_subtype.py +58 -0
  9. hindsight_api/alembic/versions/l7g8h9i0j1k2_add_worker_columns.py +109 -0
  10. hindsight_api/alembic/versions/m8h9i0j1k2l3_mental_model_id_to_text.py +41 -0
  11. hindsight_api/alembic/versions/n9i0j1k2l3m4_learnings_and_pinned_reflections.py +134 -0
  12. hindsight_api/alembic/versions/o0j1k2l3m4n5_migrate_mental_models_data.py +113 -0
  13. hindsight_api/alembic/versions/p1k2l3m4n5o6_new_knowledge_architecture.py +194 -0
  14. hindsight_api/alembic/versions/q2l3m4n5o6p7_fix_mental_model_fact_type.py +50 -0
  15. hindsight_api/alembic/versions/r3m4n5o6p7q8_add_reflect_response_to_reflections.py +47 -0
  16. hindsight_api/alembic/versions/s4n5o6p7q8r9_add_consolidated_at_to_memory_units.py +53 -0
  17. hindsight_api/alembic/versions/t5o6p7q8r9s0_rename_mental_models_to_observations.py +134 -0
  18. hindsight_api/alembic/versions/u6p7q8r9s0t1_mental_models_text_id.py +41 -0
  19. hindsight_api/alembic/versions/v7q8r9s0t1u2_add_max_tokens_to_mental_models.py +50 -0
  20. hindsight_api/api/http.py +1406 -118
  21. hindsight_api/api/mcp.py +11 -196
  22. hindsight_api/config.py +359 -27
  23. hindsight_api/engine/consolidation/__init__.py +5 -0
  24. hindsight_api/engine/consolidation/consolidator.py +859 -0
  25. hindsight_api/engine/consolidation/prompts.py +69 -0
  26. hindsight_api/engine/cross_encoder.py +706 -88
  27. hindsight_api/engine/db_budget.py +284 -0
  28. hindsight_api/engine/db_utils.py +11 -0
  29. hindsight_api/engine/directives/__init__.py +5 -0
  30. hindsight_api/engine/directives/models.py +37 -0
  31. hindsight_api/engine/embeddings.py +553 -29
  32. hindsight_api/engine/entity_resolver.py +8 -5
  33. hindsight_api/engine/interface.py +40 -17
  34. hindsight_api/engine/llm_wrapper.py +744 -68
  35. hindsight_api/engine/memory_engine.py +2505 -1017
  36. hindsight_api/engine/mental_models/__init__.py +14 -0
  37. hindsight_api/engine/mental_models/models.py +53 -0
  38. hindsight_api/engine/query_analyzer.py +4 -3
  39. hindsight_api/engine/reflect/__init__.py +18 -0
  40. hindsight_api/engine/reflect/agent.py +933 -0
  41. hindsight_api/engine/reflect/models.py +109 -0
  42. hindsight_api/engine/reflect/observations.py +186 -0
  43. hindsight_api/engine/reflect/prompts.py +483 -0
  44. hindsight_api/engine/reflect/tools.py +437 -0
  45. hindsight_api/engine/reflect/tools_schema.py +250 -0
  46. hindsight_api/engine/response_models.py +168 -4
  47. hindsight_api/engine/retain/bank_utils.py +79 -201
  48. hindsight_api/engine/retain/fact_extraction.py +424 -195
  49. hindsight_api/engine/retain/fact_storage.py +35 -12
  50. hindsight_api/engine/retain/link_utils.py +29 -24
  51. hindsight_api/engine/retain/orchestrator.py +24 -43
  52. hindsight_api/engine/retain/types.py +11 -2
  53. hindsight_api/engine/search/graph_retrieval.py +43 -14
  54. hindsight_api/engine/search/link_expansion_retrieval.py +391 -0
  55. hindsight_api/engine/search/mpfp_retrieval.py +362 -117
  56. hindsight_api/engine/search/reranking.py +2 -2
  57. hindsight_api/engine/search/retrieval.py +848 -201
  58. hindsight_api/engine/search/tags.py +172 -0
  59. hindsight_api/engine/search/think_utils.py +42 -141
  60. hindsight_api/engine/search/trace.py +12 -1
  61. hindsight_api/engine/search/tracer.py +26 -6
  62. hindsight_api/engine/search/types.py +21 -3
  63. hindsight_api/engine/task_backend.py +113 -106
  64. hindsight_api/engine/utils.py +1 -152
  65. hindsight_api/extensions/__init__.py +10 -1
  66. hindsight_api/extensions/builtin/tenant.py +5 -1
  67. hindsight_api/extensions/context.py +10 -1
  68. hindsight_api/extensions/operation_validator.py +81 -4
  69. hindsight_api/extensions/tenant.py +26 -0
  70. hindsight_api/main.py +69 -6
  71. hindsight_api/mcp_local.py +12 -53
  72. hindsight_api/mcp_tools.py +494 -0
  73. hindsight_api/metrics.py +433 -48
  74. hindsight_api/migrations.py +141 -1
  75. hindsight_api/models.py +3 -3
  76. hindsight_api/pg0.py +53 -0
  77. hindsight_api/server.py +39 -2
  78. hindsight_api/worker/__init__.py +11 -0
  79. hindsight_api/worker/main.py +296 -0
  80. hindsight_api/worker/poller.py +486 -0
  81. {hindsight_api-0.2.1.dist-info → hindsight_api-0.4.0.dist-info}/METADATA +16 -6
  82. hindsight_api-0.4.0.dist-info/RECORD +112 -0
  83. {hindsight_api-0.2.1.dist-info → hindsight_api-0.4.0.dist-info}/entry_points.txt +2 -0
  84. hindsight_api/engine/retain/observation_regeneration.py +0 -254
  85. hindsight_api/engine/search/observation_utils.py +0 -125
  86. hindsight_api/engine/search/scoring.py +0 -159
  87. hindsight_api-0.2.1.dist-info/RECORD +0 -75
  88. {hindsight_api-0.2.1.dist-info → hindsight_api-0.4.0.dist-info}/WHEEL +0 -0
@@ -0,0 +1,58 @@
1
+ """add_directive_subtype
2
+
3
+ Revision ID: k6f7g8h9i0j1
4
+ Revises: j5e6f7g8h9i0
5
+ Create Date: 2026-01-16 00:00:00.000000
6
+
7
+ This migration adds 'directive' to the mental_models subtype constraint.
8
+ Directives are hard rules with user-provided observations that the reflect agent must follow.
9
+ """
10
+
11
+ from collections.abc import Sequence
12
+
13
+ from alembic import context, op
14
+
15
+ # revision identifiers, used by Alembic.
16
+ revision: str = "k6f7g8h9i0j1"
17
+ down_revision: str | Sequence[str] | None = "j5e6f7g8h9i0"
18
+ branch_labels: str | Sequence[str] | None = None
19
+ depends_on: str | Sequence[str] | None = None
20
+
21
+
22
+ def _get_schema_prefix() -> str:
23
+ """Get schema prefix for table names (required for multi-tenant support)."""
24
+ schema = context.config.get_main_option("target_schema")
25
+ return f'"{schema}".' if schema else ""
26
+
27
+
28
+ def upgrade() -> None:
29
+ """Add 'directive' to mental_models subtype constraint."""
30
+ schema = _get_schema_prefix()
31
+
32
+ # Drop existing constraint
33
+ op.execute(f"ALTER TABLE {schema}mental_models DROP CONSTRAINT IF EXISTS ck_mental_models_subtype")
34
+
35
+ # Create new constraint with 'directive' added
36
+ op.execute(f"""
37
+ ALTER TABLE {schema}mental_models
38
+ ADD CONSTRAINT ck_mental_models_subtype
39
+ CHECK (subtype IN ('structural', 'emergent', 'pinned', 'learned', 'directive'))
40
+ """)
41
+
42
+
43
+ def downgrade() -> None:
44
+ """Remove 'directive' from mental_models subtype constraint."""
45
+ schema = _get_schema_prefix()
46
+
47
+ # First delete any directives (cannot downgrade if they exist)
48
+ op.execute(f"DELETE FROM {schema}mental_models WHERE subtype = 'directive'")
49
+
50
+ # Drop constraint with directive
51
+ op.execute(f"ALTER TABLE {schema}mental_models DROP CONSTRAINT IF EXISTS ck_mental_models_subtype")
52
+
53
+ # Recreate original constraint without directive
54
+ op.execute(f"""
55
+ ALTER TABLE {schema}mental_models
56
+ ADD CONSTRAINT ck_mental_models_subtype
57
+ CHECK (subtype IN ('structural', 'emergent', 'pinned', 'learned'))
58
+ """)
@@ -0,0 +1,109 @@
1
+ """add_worker_columns
2
+
3
+ Revision ID: l7g8h9i0j1k2
4
+ Revises: k6f7g8h9i0j1
5
+ Create Date: 2026-01-19 00:00:00.000000
6
+
7
+ This migration adds columns to async_operations for distributed worker support:
8
+ - worker_id: ID of the worker that claimed the task
9
+ - claimed_at: When the task was claimed
10
+ - retry_count: Number of retry attempts
11
+ - task_payload: The serialized task dictionary
12
+ """
13
+
14
+ from collections.abc import Sequence
15
+
16
+ import sqlalchemy as sa
17
+ from alembic import context, op
18
+ from sqlalchemy.dialects import postgresql
19
+
20
+ # revision identifiers, used by Alembic.
21
+ revision: str = "l7g8h9i0j1k2"
22
+ down_revision: str | Sequence[str] | None = "k6f7g8h9i0j1"
23
+ branch_labels: str | Sequence[str] | None = None
24
+ depends_on: str | Sequence[str] | None = None
25
+
26
+
27
+ def _get_schema_prefix() -> str:
28
+ """Get schema prefix for table names (required for multi-tenant support)."""
29
+ schema = context.config.get_main_option("target_schema")
30
+ return f'"{schema}".' if schema else ""
31
+
32
+
33
+ def upgrade() -> None:
34
+ """Add worker columns to async_operations."""
35
+ schema = _get_schema_prefix()
36
+
37
+ # Add worker_id column (ID of worker that claimed the task)
38
+ op.add_column(
39
+ "async_operations",
40
+ sa.Column("worker_id", sa.Text(), nullable=True),
41
+ schema=context.config.get_main_option("target_schema") or None,
42
+ )
43
+
44
+ # Add claimed_at column (when task was claimed by worker)
45
+ op.add_column(
46
+ "async_operations",
47
+ sa.Column("claimed_at", postgresql.TIMESTAMP(timezone=True), nullable=True),
48
+ schema=context.config.get_main_option("target_schema") or None,
49
+ )
50
+
51
+ # Add retry_count column (number of retry attempts)
52
+ op.add_column(
53
+ "async_operations",
54
+ sa.Column("retry_count", sa.Integer(), server_default="0", nullable=False),
55
+ schema=context.config.get_main_option("target_schema") or None,
56
+ )
57
+
58
+ # Add task_payload column (serialized task dictionary)
59
+ op.add_column(
60
+ "async_operations",
61
+ sa.Column(
62
+ "task_payload",
63
+ postgresql.JSONB(astext_type=sa.Text()),
64
+ nullable=True,
65
+ ),
66
+ schema=context.config.get_main_option("target_schema") or None,
67
+ )
68
+
69
+ # Add index for efficient worker polling (pending tasks ordered by creation time)
70
+ op.execute(
71
+ f"CREATE INDEX idx_async_operations_pending_claim ON {schema}async_operations (status, created_at) "
72
+ f"WHERE status = 'pending' AND task_payload IS NOT NULL"
73
+ )
74
+
75
+ # Add index for finding tasks by worker_id (for decommissioning)
76
+ op.execute(
77
+ f"CREATE INDEX idx_async_operations_worker_id ON {schema}async_operations (worker_id) WHERE worker_id IS NOT NULL"
78
+ )
79
+
80
+
81
+ def downgrade() -> None:
82
+ """Remove worker columns from async_operations."""
83
+ schema = _get_schema_prefix()
84
+
85
+ # Drop indexes
86
+ op.execute(f"DROP INDEX IF EXISTS {schema}idx_async_operations_pending_claim")
87
+ op.execute(f"DROP INDEX IF EXISTS {schema}idx_async_operations_worker_id")
88
+
89
+ # Drop columns
90
+ op.drop_column(
91
+ "async_operations",
92
+ "task_payload",
93
+ schema=context.config.get_main_option("target_schema") or None,
94
+ )
95
+ op.drop_column(
96
+ "async_operations",
97
+ "retry_count",
98
+ schema=context.config.get_main_option("target_schema") or None,
99
+ )
100
+ op.drop_column(
101
+ "async_operations",
102
+ "claimed_at",
103
+ schema=context.config.get_main_option("target_schema") or None,
104
+ )
105
+ op.drop_column(
106
+ "async_operations",
107
+ "worker_id",
108
+ schema=context.config.get_main_option("target_schema") or None,
109
+ )
@@ -0,0 +1,41 @@
1
+ """mental_model_id_to_text
2
+
3
+ Revision ID: m8h9i0j1k2l3
4
+ Revises: l7g8h9i0j1k2
5
+ Create Date: 2026-01-19 00:00:00.000000
6
+
7
+ This migration changes the mental_models.id column from VARCHAR(64) to TEXT
8
+ to support longer model IDs (e.g., entity names that exceed 64 characters).
9
+ """
10
+
11
+ from collections.abc import Sequence
12
+
13
+ from alembic import context, op
14
+
15
+ # revision identifiers, used by Alembic.
16
+ revision: str = "m8h9i0j1k2l3"
17
+ down_revision: str | Sequence[str] | None = "l7g8h9i0j1k2"
18
+ branch_labels: str | Sequence[str] | None = None
19
+ depends_on: str | Sequence[str] | None = None
20
+
21
+
22
+ def _get_schema_prefix() -> str:
23
+ """Get schema prefix for table names (required for multi-tenant support)."""
24
+ schema = context.config.get_main_option("target_schema")
25
+ return f'"{schema}".' if schema else ""
26
+
27
+
28
+ def upgrade() -> None:
29
+ """Change mental_models.id from VARCHAR(64) to TEXT."""
30
+ schema = _get_schema_prefix()
31
+
32
+ # Alter the id column type from VARCHAR(64) to TEXT
33
+ op.execute(f"ALTER TABLE {schema}mental_models ALTER COLUMN id TYPE TEXT")
34
+
35
+
36
+ def downgrade() -> None:
37
+ """Revert mental_models.id from TEXT to VARCHAR(64)."""
38
+ schema = _get_schema_prefix()
39
+
40
+ # Note: This may fail if any id values exceed 64 characters
41
+ op.execute(f"ALTER TABLE {schema}mental_models ALTER COLUMN id TYPE VARCHAR(64)")
@@ -0,0 +1,134 @@
1
+ """learnings_and_pinned_reflections
2
+
3
+ Revision ID: n9i0j1k2l3m4
4
+ Revises: m8h9i0j1k2l3
5
+ Create Date: 2026-01-21 00:00:00.000000
6
+
7
+ This migration:
8
+ 1. Creates the 'learnings' table for automatic bottom-up consolidation
9
+ 2. Creates the 'pinned_reflections' table for user-curated living documents
10
+ 3. Adds consolidation tracking columns to the 'banks' table
11
+ """
12
+
13
+ from collections.abc import Sequence
14
+
15
+ from alembic import context, op
16
+
17
+ # revision identifiers, used by Alembic.
18
+ revision: str = "n9i0j1k2l3m4"
19
+ down_revision: str | Sequence[str] | None = "m8h9i0j1k2l3"
20
+ branch_labels: str | Sequence[str] | None = None
21
+ depends_on: str | Sequence[str] | None = None
22
+
23
+
24
+ def _get_schema_prefix() -> str:
25
+ """Get schema prefix for table names (required for multi-tenant support)."""
26
+ schema = context.config.get_main_option("target_schema")
27
+ return f'"{schema}".' if schema else ""
28
+
29
+
30
+ def upgrade() -> None:
31
+ """Create learnings and pinned_reflections tables."""
32
+ schema = _get_schema_prefix()
33
+
34
+ # 1. Create learnings table
35
+ op.execute(f"""
36
+ CREATE TABLE {schema}learnings (
37
+ id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
38
+ bank_id VARCHAR(64) NOT NULL,
39
+ text TEXT NOT NULL,
40
+ proof_count INT NOT NULL DEFAULT 1,
41
+ history JSONB DEFAULT '[]'::jsonb,
42
+ mission_context VARCHAR(64),
43
+ pre_mission_change BOOLEAN DEFAULT FALSE,
44
+ embedding vector(384),
45
+ tags VARCHAR[] DEFAULT ARRAY[]::VARCHAR[],
46
+ created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(),
47
+ updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now()
48
+ )
49
+ """)
50
+
51
+ # Add foreign key constraint
52
+ op.execute(f"""
53
+ ALTER TABLE {schema}learnings
54
+ ADD CONSTRAINT fk_learnings_bank_id
55
+ FOREIGN KEY (bank_id) REFERENCES {schema}banks(bank_id) ON DELETE CASCADE
56
+ """)
57
+
58
+ # Indexes for learnings
59
+ op.execute(f"CREATE INDEX idx_learnings_bank_id ON {schema}learnings(bank_id)")
60
+ op.execute(f"""
61
+ CREATE INDEX idx_learnings_embedding ON {schema}learnings
62
+ USING hnsw (embedding vector_cosine_ops)
63
+ """)
64
+ op.execute(f"CREATE INDEX idx_learnings_tags ON {schema}learnings USING GIN(tags)")
65
+
66
+ # Full-text search for learnings
67
+ op.execute(f"""
68
+ ALTER TABLE {schema}learnings ADD COLUMN search_vector tsvector
69
+ GENERATED ALWAYS AS (to_tsvector('english', text)) STORED
70
+ """)
71
+ op.execute(f"CREATE INDEX idx_learnings_text_search ON {schema}learnings USING gin(search_vector)")
72
+
73
+ # 2. Create pinned_reflections table
74
+ op.execute(f"""
75
+ CREATE TABLE {schema}pinned_reflections (
76
+ id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
77
+ bank_id VARCHAR(64) NOT NULL,
78
+ name VARCHAR(256) NOT NULL,
79
+ source_query TEXT NOT NULL,
80
+ content TEXT NOT NULL,
81
+ embedding vector(384),
82
+ tags VARCHAR[] DEFAULT ARRAY[]::VARCHAR[],
83
+ last_refreshed_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(),
84
+ created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now()
85
+ )
86
+ """)
87
+
88
+ # Add foreign key constraint
89
+ op.execute(f"""
90
+ ALTER TABLE {schema}pinned_reflections
91
+ ADD CONSTRAINT fk_pinned_reflections_bank_id
92
+ FOREIGN KEY (bank_id) REFERENCES {schema}banks(bank_id) ON DELETE CASCADE
93
+ """)
94
+
95
+ # Indexes for pinned_reflections
96
+ op.execute(f"CREATE INDEX idx_pinned_reflections_bank_id ON {schema}pinned_reflections(bank_id)")
97
+ op.execute(f"""
98
+ CREATE INDEX idx_pinned_reflections_embedding ON {schema}pinned_reflections
99
+ USING hnsw (embedding vector_cosine_ops)
100
+ """)
101
+ op.execute(f"CREATE INDEX idx_pinned_reflections_tags ON {schema}pinned_reflections USING GIN(tags)")
102
+
103
+ # Full-text search for pinned_reflections
104
+ op.execute(f"""
105
+ ALTER TABLE {schema}pinned_reflections ADD COLUMN search_vector tsvector
106
+ GENERATED ALWAYS AS (to_tsvector('english', COALESCE(name, '') || ' ' || content)) STORED
107
+ """)
108
+ op.execute(f"""
109
+ CREATE INDEX idx_pinned_reflections_text_search ON {schema}pinned_reflections
110
+ USING gin(search_vector)
111
+ """)
112
+
113
+ # 3. Add consolidation tracking columns to banks table
114
+ op.execute(f"""
115
+ ALTER TABLE {schema}banks
116
+ ADD COLUMN IF NOT EXISTS last_consolidated_at TIMESTAMP WITH TIME ZONE
117
+ """)
118
+ op.execute(f"""
119
+ ALTER TABLE {schema}banks
120
+ ADD COLUMN IF NOT EXISTS mission_changed_at TIMESTAMP WITH TIME ZONE
121
+ """)
122
+
123
+
124
+ def downgrade() -> None:
125
+ """Drop learnings and pinned_reflections tables."""
126
+ schema = _get_schema_prefix()
127
+
128
+ # Drop tables
129
+ op.execute(f"DROP TABLE IF EXISTS {schema}learnings CASCADE")
130
+ op.execute(f"DROP TABLE IF EXISTS {schema}pinned_reflections CASCADE")
131
+
132
+ # Remove columns from banks
133
+ op.execute(f"ALTER TABLE {schema}banks DROP COLUMN IF EXISTS last_consolidated_at")
134
+ op.execute(f"ALTER TABLE {schema}banks DROP COLUMN IF EXISTS mission_changed_at")
@@ -0,0 +1,113 @@
1
+ """migrate_mental_models_data
2
+
3
+ Revision ID: o0j1k2l3m4n5
4
+ Revises: n9i0j1k2l3m4
5
+ Create Date: 2026-01-21 00:00:00.000000
6
+
7
+ This migration:
8
+ 1. Migrates existing 'pinned' mental models to the new 'pinned_reflections' table
9
+ 2. Migrates existing 'learned' mental models to the new 'learnings' table
10
+ 3. Deletes non-directive mental models (structural, emergent, pinned, learned)
11
+ 4. Drops the mental_model_versions table (no longer used)
12
+ 5. Adds a CHECK constraint that only 'directive' subtype is allowed
13
+ """
14
+
15
+ from collections.abc import Sequence
16
+
17
+ from alembic import context, op
18
+
19
+ # revision identifiers, used by Alembic.
20
+ revision: str = "o0j1k2l3m4n5"
21
+ down_revision: str | Sequence[str] | None = "n9i0j1k2l3m4"
22
+ branch_labels: str | Sequence[str] | None = None
23
+ depends_on: str | Sequence[str] | None = None
24
+
25
+
26
+ def _get_schema_prefix() -> str:
27
+ """Get schema prefix for table names (required for multi-tenant support)."""
28
+ schema = context.config.get_main_option("target_schema")
29
+ return f'"{schema}".' if schema else ""
30
+
31
+
32
+ def upgrade() -> None:
33
+ """Migrate data and clean up old mental models."""
34
+ schema = _get_schema_prefix()
35
+
36
+ # 1. Migrate 'pinned' mental models to pinned_reflections
37
+ # For pinned models, the first observation's content becomes the pinned reflection content
38
+ op.execute(f"""
39
+ INSERT INTO {schema}pinned_reflections (bank_id, name, source_query, content, tags, created_at)
40
+ SELECT
41
+ bank_id,
42
+ name,
43
+ description AS source_query,
44
+ COALESCE(
45
+ observations->'observations'->0->>'content',
46
+ description,
47
+ ''
48
+ ) AS content,
49
+ tags,
50
+ created_at
51
+ FROM {schema}mental_models
52
+ WHERE subtype = 'pinned'
53
+ ON CONFLICT DO NOTHING
54
+ """)
55
+
56
+ # 2. Migrate 'learned' mental models to learnings
57
+ # Each observation in a learned model becomes a separate learning
58
+ op.execute(f"""
59
+ INSERT INTO {schema}learnings (bank_id, text, proof_count, tags, created_at)
60
+ SELECT
61
+ mm.bank_id,
62
+ obs->>'content' AS text,
63
+ GREATEST(1, COALESCE(jsonb_array_length(obs->'evidence'), 1)) AS proof_count,
64
+ mm.tags,
65
+ mm.created_at
66
+ FROM {schema}mental_models mm,
67
+ LATERAL jsonb_array_elements(mm.observations->'observations') AS obs
68
+ WHERE mm.subtype = 'learned'
69
+ AND obs->>'content' IS NOT NULL
70
+ AND obs->>'content' != ''
71
+ ON CONFLICT DO NOTHING
72
+ """)
73
+
74
+ # 3. Delete all non-directive mental models (they've been migrated or are obsolete)
75
+ op.execute(f"""
76
+ DELETE FROM {schema}mental_models
77
+ WHERE subtype != 'directive'
78
+ """)
79
+
80
+ # 4. Drop the mental_model_versions table (no longer used)
81
+ op.execute(f"DROP TABLE IF EXISTS {schema}mental_model_versions CASCADE")
82
+
83
+ # 5. Drop old constraints and add new one that only allows 'directive'
84
+ op.execute(f"ALTER TABLE {schema}mental_models DROP CONSTRAINT IF EXISTS ck_mental_models_subtype")
85
+ op.execute(f"""
86
+ ALTER TABLE {schema}mental_models
87
+ ADD CONSTRAINT ck_mental_models_subtype CHECK (subtype = 'directive')
88
+ """)
89
+
90
+
91
+ def downgrade() -> None:
92
+ """Reverse the migration (data migration is one-way, so this just removes constraints)."""
93
+ schema = _get_schema_prefix()
94
+
95
+ # Remove the directive-only constraint
96
+ op.execute(f"ALTER TABLE {schema}mental_models DROP CONSTRAINT IF EXISTS ck_mental_models_subtype")
97
+
98
+ # Re-create mental_model_versions table
99
+ op.execute(f"""
100
+ CREATE TABLE IF NOT EXISTS {schema}mental_model_versions (
101
+ id SERIAL PRIMARY KEY,
102
+ bank_id VARCHAR(64) NOT NULL,
103
+ model_id VARCHAR(128) NOT NULL,
104
+ version INT NOT NULL,
105
+ observations JSONB NOT NULL,
106
+ created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now()
107
+ )
108
+ """)
109
+ op.execute(
110
+ f"CREATE INDEX IF NOT EXISTS idx_mm_versions_lookup ON {schema}mental_model_versions(bank_id, model_id, version DESC)"
111
+ )
112
+
113
+ # Note: Data migration cannot be reversed - pinned_reflections and learnings data remains
@@ -0,0 +1,194 @@
1
+ """new_knowledge_architecture
2
+
3
+ Revision ID: p1k2l3m4n5o6
4
+ Revises: o0j1k2l3m4n5
5
+ Create Date: 2026-01-21 00:00:00.000000
6
+
7
+ This migration implements the new knowledge architecture:
8
+ 1. Drops the 'learnings' table (mental models are now in memory_units)
9
+ 2. Renames 'pinned_reflections' to 'reflections'
10
+ 3. Drops the 'mental_models' table completely
11
+ 4. Creates 'directives' table for hard rules
12
+ 5. Adds mental model support columns to 'memory_units' (proof_count, source_memory_ids, history)
13
+
14
+ The new architecture:
15
+ - Directives: Hard rules in their own table
16
+ - Mental Models: Stored in memory_units with fact_type='mental_model'
17
+ - Reflections: User-curated documents (renamed from pinned_reflections)
18
+ """
19
+
20
+ from collections.abc import Sequence
21
+
22
+ from alembic import context, op
23
+
24
+ # revision identifiers, used by Alembic.
25
+ revision: str = "p1k2l3m4n5o6"
26
+ down_revision: str | Sequence[str] | None = "o0j1k2l3m4n5"
27
+ branch_labels: str | Sequence[str] | None = None
28
+ depends_on: str | Sequence[str] | None = None
29
+
30
+
31
+ def _get_schema_prefix() -> str:
32
+ """Get schema prefix for table names (required for multi-tenant support)."""
33
+ schema = context.config.get_main_option("target_schema")
34
+ return f'"{schema}".' if schema else ""
35
+
36
+
37
+ def upgrade() -> None:
38
+ """Implement new knowledge architecture."""
39
+ schema = _get_schema_prefix()
40
+
41
+ # 1. Drop the learnings table (mental models will be in memory_units)
42
+ op.execute(f"DROP TABLE IF EXISTS {schema}learnings CASCADE")
43
+
44
+ # 2. Rename pinned_reflections to reflections
45
+ op.execute(f"ALTER TABLE IF EXISTS {schema}pinned_reflections RENAME TO reflections")
46
+
47
+ # Rename indexes for reflections
48
+ op.execute(f"ALTER INDEX IF EXISTS {schema}idx_pinned_reflections_bank_id RENAME TO idx_reflections_bank_id")
49
+ op.execute(f"ALTER INDEX IF EXISTS {schema}idx_pinned_reflections_embedding RENAME TO idx_reflections_embedding")
50
+ op.execute(f"ALTER INDEX IF EXISTS {schema}idx_pinned_reflections_tags RENAME TO idx_reflections_tags")
51
+ op.execute(
52
+ f"ALTER INDEX IF EXISTS {schema}idx_pinned_reflections_text_search RENAME TO idx_reflections_text_search"
53
+ )
54
+
55
+ # Rename foreign key constraint
56
+ op.execute(f"""
57
+ ALTER TABLE {schema}reflections
58
+ DROP CONSTRAINT IF EXISTS fk_pinned_reflections_bank_id
59
+ """)
60
+ op.execute(f"""
61
+ ALTER TABLE {schema}reflections
62
+ ADD CONSTRAINT fk_reflections_bank_id
63
+ FOREIGN KEY (bank_id) REFERENCES {schema}banks(bank_id) ON DELETE CASCADE
64
+ """)
65
+
66
+ # 3. Drop the mental_models table completely
67
+ op.execute(f"DROP TABLE IF EXISTS {schema}mental_models CASCADE")
68
+
69
+ # 4. Create directives table
70
+ op.execute(f"""
71
+ CREATE TABLE {schema}directives (
72
+ id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
73
+ bank_id VARCHAR(64) NOT NULL,
74
+ name VARCHAR(256) NOT NULL,
75
+ content TEXT NOT NULL,
76
+ priority INT NOT NULL DEFAULT 0,
77
+ is_active BOOLEAN NOT NULL DEFAULT TRUE,
78
+ tags VARCHAR[] DEFAULT ARRAY[]::VARCHAR[],
79
+ created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(),
80
+ updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now()
81
+ )
82
+ """)
83
+
84
+ # Add foreign key and indexes for directives
85
+ op.execute(f"""
86
+ ALTER TABLE {schema}directives
87
+ ADD CONSTRAINT fk_directives_bank_id
88
+ FOREIGN KEY (bank_id) REFERENCES {schema}banks(bank_id) ON DELETE CASCADE
89
+ """)
90
+ op.execute(f"CREATE INDEX idx_directives_bank_id ON {schema}directives(bank_id)")
91
+ op.execute(f"CREATE INDEX idx_directives_bank_active ON {schema}directives(bank_id, is_active)")
92
+ op.execute(f"CREATE INDEX idx_directives_tags ON {schema}directives USING GIN(tags)")
93
+
94
+ # 5. Add mental model support columns to memory_units
95
+ # proof_count: Number of memories that support this mental model
96
+ op.execute(f"""
97
+ ALTER TABLE {schema}memory_units
98
+ ADD COLUMN IF NOT EXISTS proof_count INT DEFAULT 1
99
+ """)
100
+
101
+ # source_memory_ids: Array of memory IDs that consolidated into this mental model
102
+ op.execute(f"""
103
+ ALTER TABLE {schema}memory_units
104
+ ADD COLUMN IF NOT EXISTS source_memory_ids UUID[] DEFAULT ARRAY[]::UUID[]
105
+ """)
106
+
107
+ # history: JSONB array tracking changes to mental models
108
+ op.execute(f"""
109
+ ALTER TABLE {schema}memory_units
110
+ ADD COLUMN IF NOT EXISTS history JSONB DEFAULT '[]'::jsonb
111
+ """)
112
+
113
+ # Add index for finding mental models
114
+ op.execute(f"""
115
+ CREATE INDEX IF NOT EXISTS idx_memory_units_mental_models
116
+ ON {schema}memory_units(bank_id, fact_type)
117
+ WHERE fact_type = 'mental_model'
118
+ """)
119
+
120
+ # 6. Update fact_type check constraint to include 'mental_model'
121
+ op.execute(f"ALTER TABLE {schema}memory_units DROP CONSTRAINT IF EXISTS memory_units_fact_type_check")
122
+ op.execute(f"""
123
+ ALTER TABLE {schema}memory_units
124
+ ADD CONSTRAINT memory_units_fact_type_check
125
+ CHECK (fact_type IN ('world', 'experience', 'opinion', 'observation', 'mental_model'))
126
+ """)
127
+
128
+
129
+ def downgrade() -> None:
130
+ """Reverse the migration."""
131
+ schema = _get_schema_prefix()
132
+
133
+ # Restore original fact_type check constraint (without 'mental_model')
134
+ op.execute(f"ALTER TABLE {schema}memory_units DROP CONSTRAINT IF EXISTS memory_units_fact_type_check")
135
+ op.execute(f"""
136
+ ALTER TABLE {schema}memory_units
137
+ ADD CONSTRAINT memory_units_fact_type_check
138
+ CHECK (fact_type IN ('world', 'experience', 'opinion', 'observation'))
139
+ """)
140
+
141
+ # Drop mental model columns from memory_units
142
+ op.execute(f"ALTER TABLE {schema}memory_units DROP COLUMN IF EXISTS proof_count")
143
+ op.execute(f"ALTER TABLE {schema}memory_units DROP COLUMN IF EXISTS source_memory_ids")
144
+ op.execute(f"ALTER TABLE {schema}memory_units DROP COLUMN IF EXISTS history")
145
+ op.execute(f"DROP INDEX IF EXISTS {schema}idx_memory_units_mental_models")
146
+
147
+ # Drop directives table
148
+ op.execute(f"DROP TABLE IF EXISTS {schema}directives CASCADE")
149
+
150
+ # Rename reflections back to pinned_reflections
151
+ op.execute(f"ALTER TABLE IF EXISTS {schema}reflections RENAME TO pinned_reflections")
152
+
153
+ # Restore indexes
154
+ op.execute(f"ALTER INDEX IF EXISTS {schema}idx_reflections_bank_id RENAME TO idx_pinned_reflections_bank_id")
155
+ op.execute(f"ALTER INDEX IF EXISTS {schema}idx_reflections_embedding RENAME TO idx_pinned_reflections_embedding")
156
+ op.execute(f"ALTER INDEX IF EXISTS {schema}idx_reflections_tags RENAME TO idx_pinned_reflections_tags")
157
+ op.execute(
158
+ f"ALTER INDEX IF EXISTS {schema}idx_reflections_text_search RENAME TO idx_pinned_reflections_text_search"
159
+ )
160
+
161
+ # Restore foreign key
162
+ op.execute(f"""
163
+ ALTER TABLE {schema}pinned_reflections
164
+ DROP CONSTRAINT IF EXISTS fk_reflections_bank_id
165
+ """)
166
+ op.execute(f"""
167
+ ALTER TABLE {schema}pinned_reflections
168
+ ADD CONSTRAINT fk_pinned_reflections_bank_id
169
+ FOREIGN KEY (bank_id) REFERENCES {schema}banks(bank_id) ON DELETE CASCADE
170
+ """)
171
+
172
+ # Re-create learnings table
173
+ op.execute(f"""
174
+ CREATE TABLE IF NOT EXISTS {schema}learnings (
175
+ id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
176
+ bank_id VARCHAR(64) NOT NULL,
177
+ text TEXT NOT NULL,
178
+ proof_count INT NOT NULL DEFAULT 1,
179
+ history JSONB DEFAULT '[]'::jsonb,
180
+ mission_context VARCHAR(64),
181
+ pre_mission_change BOOLEAN DEFAULT FALSE,
182
+ embedding vector(384),
183
+ tags VARCHAR[] DEFAULT ARRAY[]::VARCHAR[],
184
+ created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(),
185
+ updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now()
186
+ )
187
+ """)
188
+ op.execute(f"""
189
+ ALTER TABLE {schema}learnings
190
+ ADD CONSTRAINT fk_learnings_bank_id
191
+ FOREIGN KEY (bank_id) REFERENCES {schema}banks(bank_id) ON DELETE CASCADE
192
+ """)
193
+
194
+ # Note: mental_models table recreation is complex and would need separate handling