hindsight-api 0.3.0__tar.gz → 0.4.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (120) hide show
  1. {hindsight_api-0.3.0 → hindsight_api-0.4.1}/.gitignore +5 -2
  2. {hindsight_api-0.3.0 → hindsight_api-0.4.1}/PKG-INFO +12 -6
  3. {hindsight_api-0.3.0 → hindsight_api-0.4.1}/hindsight_api/__init__.py +1 -1
  4. {hindsight_api-0.3.0 → hindsight_api-0.4.1}/hindsight_api/admin/cli.py +59 -0
  5. hindsight_api-0.4.1/hindsight_api/alembic/versions/h3c4d5e6f7g8_mental_models_v4.py +112 -0
  6. hindsight_api-0.4.1/hindsight_api/alembic/versions/i4d5e6f7g8h9_delete_opinions.py +41 -0
  7. hindsight_api-0.4.1/hindsight_api/alembic/versions/j5e6f7g8h9i0_mental_model_versions.py +95 -0
  8. hindsight_api-0.4.1/hindsight_api/alembic/versions/k6f7g8h9i0j1_add_directive_subtype.py +58 -0
  9. hindsight_api-0.4.1/hindsight_api/alembic/versions/l7g8h9i0j1k2_add_worker_columns.py +109 -0
  10. hindsight_api-0.4.1/hindsight_api/alembic/versions/m8h9i0j1k2l3_mental_model_id_to_text.py +41 -0
  11. hindsight_api-0.4.1/hindsight_api/alembic/versions/n9i0j1k2l3m4_learnings_and_pinned_reflections.py +134 -0
  12. hindsight_api-0.4.1/hindsight_api/alembic/versions/o0j1k2l3m4n5_migrate_mental_models_data.py +113 -0
  13. hindsight_api-0.4.1/hindsight_api/alembic/versions/p1k2l3m4n5o6_new_knowledge_architecture.py +194 -0
  14. hindsight_api-0.4.1/hindsight_api/alembic/versions/q2l3m4n5o6p7_fix_mental_model_fact_type.py +50 -0
  15. hindsight_api-0.4.1/hindsight_api/alembic/versions/r3m4n5o6p7q8_add_reflect_response_to_reflections.py +47 -0
  16. hindsight_api-0.4.1/hindsight_api/alembic/versions/s4n5o6p7q8r9_add_consolidated_at_to_memory_units.py +53 -0
  17. hindsight_api-0.4.1/hindsight_api/alembic/versions/t5o6p7q8r9s0_rename_mental_models_to_observations.py +134 -0
  18. hindsight_api-0.4.1/hindsight_api/alembic/versions/u6p7q8r9s0t1_mental_models_text_id.py +41 -0
  19. hindsight_api-0.4.1/hindsight_api/alembic/versions/v7q8r9s0t1u2_add_max_tokens_to_mental_models.py +50 -0
  20. {hindsight_api-0.3.0 → hindsight_api-0.4.1}/hindsight_api/api/http.py +1120 -93
  21. hindsight_api-0.4.1/hindsight_api/api/mcp.py +190 -0
  22. {hindsight_api-0.3.0 → hindsight_api-0.4.1}/hindsight_api/config.py +174 -46
  23. hindsight_api-0.4.1/hindsight_api/engine/consolidation/__init__.py +5 -0
  24. hindsight_api-0.4.1/hindsight_api/engine/consolidation/consolidator.py +926 -0
  25. hindsight_api-0.4.1/hindsight_api/engine/consolidation/prompts.py +77 -0
  26. {hindsight_api-0.3.0 → hindsight_api-0.4.1}/hindsight_api/engine/cross_encoder.py +153 -22
  27. hindsight_api-0.4.1/hindsight_api/engine/directives/__init__.py +5 -0
  28. hindsight_api-0.4.1/hindsight_api/engine/directives/models.py +37 -0
  29. {hindsight_api-0.3.0 → hindsight_api-0.4.1}/hindsight_api/engine/embeddings.py +136 -13
  30. {hindsight_api-0.3.0 → hindsight_api-0.4.1}/hindsight_api/engine/interface.py +32 -13
  31. {hindsight_api-0.3.0 → hindsight_api-0.4.1}/hindsight_api/engine/llm_wrapper.py +505 -43
  32. {hindsight_api-0.3.0 → hindsight_api-0.4.1}/hindsight_api/engine/memory_engine.py +2101 -1094
  33. hindsight_api-0.4.1/hindsight_api/engine/mental_models/__init__.py +14 -0
  34. hindsight_api-0.4.1/hindsight_api/engine/mental_models/models.py +53 -0
  35. hindsight_api-0.4.1/hindsight_api/engine/reflect/__init__.py +18 -0
  36. hindsight_api-0.4.1/hindsight_api/engine/reflect/agent.py +933 -0
  37. hindsight_api-0.4.1/hindsight_api/engine/reflect/models.py +109 -0
  38. hindsight_api-0.4.1/hindsight_api/engine/reflect/observations.py +186 -0
  39. hindsight_api-0.4.1/hindsight_api/engine/reflect/prompts.py +483 -0
  40. hindsight_api-0.4.1/hindsight_api/engine/reflect/tools.py +437 -0
  41. hindsight_api-0.4.1/hindsight_api/engine/reflect/tools_schema.py +250 -0
  42. {hindsight_api-0.3.0 → hindsight_api-0.4.1}/hindsight_api/engine/response_models.py +130 -4
  43. hindsight_api-0.4.1/hindsight_api/engine/retain/bank_utils.py +268 -0
  44. {hindsight_api-0.3.0 → hindsight_api-0.4.1}/hindsight_api/engine/retain/fact_extraction.py +81 -48
  45. {hindsight_api-0.3.0 → hindsight_api-0.4.1}/hindsight_api/engine/retain/fact_storage.py +5 -8
  46. {hindsight_api-0.3.0 → hindsight_api-0.4.1}/hindsight_api/engine/retain/link_utils.py +5 -8
  47. {hindsight_api-0.3.0 → hindsight_api-0.4.1}/hindsight_api/engine/retain/orchestrator.py +1 -55
  48. {hindsight_api-0.3.0 → hindsight_api-0.4.1}/hindsight_api/engine/retain/types.py +2 -2
  49. {hindsight_api-0.3.0 → hindsight_api-0.4.1}/hindsight_api/engine/search/graph_retrieval.py +2 -2
  50. hindsight_api-0.4.1/hindsight_api/engine/search/link_expansion_retrieval.py +391 -0
  51. {hindsight_api-0.3.0 → hindsight_api-0.4.1}/hindsight_api/engine/search/mpfp_retrieval.py +1 -1
  52. {hindsight_api-0.3.0 → hindsight_api-0.4.1}/hindsight_api/engine/search/retrieval.py +14 -14
  53. {hindsight_api-0.3.0 → hindsight_api-0.4.1}/hindsight_api/engine/search/think_utils.py +41 -140
  54. {hindsight_api-0.3.0 → hindsight_api-0.4.1}/hindsight_api/engine/search/trace.py +0 -1
  55. {hindsight_api-0.3.0 → hindsight_api-0.4.1}/hindsight_api/engine/search/tracer.py +2 -5
  56. {hindsight_api-0.3.0 → hindsight_api-0.4.1}/hindsight_api/engine/search/types.py +0 -3
  57. hindsight_api-0.4.1/hindsight_api/engine/task_backend.py +257 -0
  58. hindsight_api-0.4.1/hindsight_api/engine/utils.py +67 -0
  59. {hindsight_api-0.3.0 → hindsight_api-0.4.1}/hindsight_api/extensions/__init__.py +10 -1
  60. {hindsight_api-0.3.0 → hindsight_api-0.4.1}/hindsight_api/extensions/builtin/tenant.py +11 -4
  61. {hindsight_api-0.3.0 → hindsight_api-0.4.1}/hindsight_api/extensions/operation_validator.py +81 -4
  62. {hindsight_api-0.3.0 → hindsight_api-0.4.1}/hindsight_api/extensions/tenant.py +26 -0
  63. {hindsight_api-0.3.0 → hindsight_api-0.4.1}/hindsight_api/main.py +28 -5
  64. {hindsight_api-0.3.0 → hindsight_api-0.4.1}/hindsight_api/mcp_local.py +12 -53
  65. hindsight_api-0.4.1/hindsight_api/mcp_tools.py +494 -0
  66. {hindsight_api-0.3.0 → hindsight_api-0.4.1}/hindsight_api/models.py +0 -2
  67. hindsight_api-0.4.1/hindsight_api/worker/__init__.py +11 -0
  68. hindsight_api-0.4.1/hindsight_api/worker/main.py +296 -0
  69. hindsight_api-0.4.1/hindsight_api/worker/poller.py +486 -0
  70. {hindsight_api-0.3.0 → hindsight_api-0.4.1}/pyproject.toml +15 -7
  71. hindsight_api-0.3.0/hindsight_api/api/mcp.py +0 -370
  72. hindsight_api-0.3.0/hindsight_api/engine/retain/bank_utils.py +0 -390
  73. hindsight_api-0.3.0/hindsight_api/engine/retain/observation_regeneration.py +0 -254
  74. hindsight_api-0.3.0/hindsight_api/engine/search/link_expansion_retrieval.py +0 -256
  75. hindsight_api-0.3.0/hindsight_api/engine/search/observation_utils.py +0 -125
  76. hindsight_api-0.3.0/hindsight_api/engine/search/scoring.py +0 -159
  77. hindsight_api-0.3.0/hindsight_api/engine/task_backend.py +0 -341
  78. hindsight_api-0.3.0/hindsight_api/engine/utils.py +0 -218
  79. {hindsight_api-0.3.0 → hindsight_api-0.4.1}/README.md +0 -0
  80. {hindsight_api-0.3.0 → hindsight_api-0.4.1}/hindsight_api/admin/__init__.py +0 -0
  81. {hindsight_api-0.3.0 → hindsight_api-0.4.1}/hindsight_api/alembic/README +0 -0
  82. {hindsight_api-0.3.0 → hindsight_api-0.4.1}/hindsight_api/alembic/env.py +0 -0
  83. {hindsight_api-0.3.0 → hindsight_api-0.4.1}/hindsight_api/alembic/script.py.mako +0 -0
  84. {hindsight_api-0.3.0 → hindsight_api-0.4.1}/hindsight_api/alembic/versions/5a366d414dce_initial_schema.py +0 -0
  85. {hindsight_api-0.3.0 → hindsight_api-0.4.1}/hindsight_api/alembic/versions/b7c4d8e9f1a2_add_chunks_table.py +0 -0
  86. {hindsight_api-0.3.0 → hindsight_api-0.4.1}/hindsight_api/alembic/versions/c8e5f2a3b4d1_add_retain_params_to_documents.py +0 -0
  87. {hindsight_api-0.3.0 → hindsight_api-0.4.1}/hindsight_api/alembic/versions/d9f6a3b4c5e2_rename_bank_to_interactions.py +0 -0
  88. {hindsight_api-0.3.0 → hindsight_api-0.4.1}/hindsight_api/alembic/versions/e0a1b2c3d4e5_disposition_to_3_traits.py +0 -0
  89. {hindsight_api-0.3.0 → hindsight_api-0.4.1}/hindsight_api/alembic/versions/f1a2b3c4d5e6_add_memory_links_composite_index.py +0 -0
  90. {hindsight_api-0.3.0 → hindsight_api-0.4.1}/hindsight_api/alembic/versions/g2a3b4c5d6e7_add_tags_column.py +0 -0
  91. {hindsight_api-0.3.0 → hindsight_api-0.4.1}/hindsight_api/alembic/versions/rename_personality_to_disposition.py +0 -0
  92. {hindsight_api-0.3.0 → hindsight_api-0.4.1}/hindsight_api/api/__init__.py +0 -0
  93. {hindsight_api-0.3.0 → hindsight_api-0.4.1}/hindsight_api/banner.py +0 -0
  94. {hindsight_api-0.3.0 → hindsight_api-0.4.1}/hindsight_api/daemon.py +0 -0
  95. {hindsight_api-0.3.0 → hindsight_api-0.4.1}/hindsight_api/engine/__init__.py +0 -0
  96. {hindsight_api-0.3.0 → hindsight_api-0.4.1}/hindsight_api/engine/db_budget.py +0 -0
  97. {hindsight_api-0.3.0 → hindsight_api-0.4.1}/hindsight_api/engine/db_utils.py +0 -0
  98. {hindsight_api-0.3.0 → hindsight_api-0.4.1}/hindsight_api/engine/entity_resolver.py +0 -0
  99. {hindsight_api-0.3.0 → hindsight_api-0.4.1}/hindsight_api/engine/query_analyzer.py +0 -0
  100. {hindsight_api-0.3.0 → hindsight_api-0.4.1}/hindsight_api/engine/retain/__init__.py +0 -0
  101. {hindsight_api-0.3.0 → hindsight_api-0.4.1}/hindsight_api/engine/retain/chunk_storage.py +0 -0
  102. {hindsight_api-0.3.0 → hindsight_api-0.4.1}/hindsight_api/engine/retain/deduplication.py +0 -0
  103. {hindsight_api-0.3.0 → hindsight_api-0.4.1}/hindsight_api/engine/retain/embedding_processing.py +0 -0
  104. {hindsight_api-0.3.0 → hindsight_api-0.4.1}/hindsight_api/engine/retain/embedding_utils.py +0 -0
  105. {hindsight_api-0.3.0 → hindsight_api-0.4.1}/hindsight_api/engine/retain/entity_processing.py +0 -0
  106. {hindsight_api-0.3.0 → hindsight_api-0.4.1}/hindsight_api/engine/retain/link_creation.py +0 -0
  107. {hindsight_api-0.3.0 → hindsight_api-0.4.1}/hindsight_api/engine/search/__init__.py +0 -0
  108. {hindsight_api-0.3.0 → hindsight_api-0.4.1}/hindsight_api/engine/search/fusion.py +0 -0
  109. {hindsight_api-0.3.0 → hindsight_api-0.4.1}/hindsight_api/engine/search/reranking.py +0 -0
  110. {hindsight_api-0.3.0 → hindsight_api-0.4.1}/hindsight_api/engine/search/tags.py +0 -0
  111. {hindsight_api-0.3.0 → hindsight_api-0.4.1}/hindsight_api/engine/search/temporal_extraction.py +0 -0
  112. {hindsight_api-0.3.0 → hindsight_api-0.4.1}/hindsight_api/extensions/base.py +0 -0
  113. {hindsight_api-0.3.0 → hindsight_api-0.4.1}/hindsight_api/extensions/builtin/__init__.py +0 -0
  114. {hindsight_api-0.3.0 → hindsight_api-0.4.1}/hindsight_api/extensions/context.py +0 -0
  115. {hindsight_api-0.3.0 → hindsight_api-0.4.1}/hindsight_api/extensions/http.py +0 -0
  116. {hindsight_api-0.3.0 → hindsight_api-0.4.1}/hindsight_api/extensions/loader.py +0 -0
  117. {hindsight_api-0.3.0 → hindsight_api-0.4.1}/hindsight_api/metrics.py +0 -0
  118. {hindsight_api-0.3.0 → hindsight_api-0.4.1}/hindsight_api/migrations.py +0 -0
  119. {hindsight_api-0.3.0 → hindsight_api-0.4.1}/hindsight_api/pg0.py +0 -0
  120. {hindsight_api-0.3.0 → hindsight_api-0.4.1}/hindsight_api/server.py +0 -0
@@ -29,7 +29,7 @@ nltk_data/
29
29
 
30
30
  # Monitoring stack (Prometheus/Grafana binaries and data)
31
31
  .monitoring/
32
- .pgbouncer
32
+ .pgbouncer/
33
33
 
34
34
  # Large benchmark datasets (will be downloaded automatically)
35
35
  **/longmemeval_s_cleaned.json
@@ -45,9 +45,12 @@ hindsight-docs/static/llms-full.txt
45
45
 
46
46
  hindsight-dev/benchmarks/locomo/results/
47
47
  hindsight-dev/benchmarks/longmemeval/results/
48
+ hindsight-dev/benchmarks/consolidation/results/
49
+ benchmarks/results/
48
50
  hindsight-cli/target
49
51
  hindsight-clients/rust/target
50
52
  .claude
51
53
  whats-next.md
52
54
  TASK.md
53
- CHANGELOG.md
55
+ # Changelog is now tracked in hindsight-docs/src/pages/changelog.md
56
+ # CHANGELOG.md
@@ -1,19 +1,23 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: hindsight-api
3
- Version: 0.3.0
3
+ Version: 0.4.1
4
4
  Summary: Hindsight: Agent Memory That Works Like Human Memory
5
5
  Requires-Python: >=3.11
6
+ Requires-Dist: aiohttp>=3.13.3
6
7
  Requires-Dist: alembic>=1.17.1
7
8
  Requires-Dist: anthropic>=0.40.0
8
9
  Requires-Dist: asyncpg>=0.29.0
10
+ Requires-Dist: authlib>=1.6.6
9
11
  Requires-Dist: cohere>=5.0.0
10
12
  Requires-Dist: dateparser>=1.2.2
11
13
  Requires-Dist: fastapi[standard]>=0.120.3
12
- Requires-Dist: fastmcp>=2.3.0
14
+ Requires-Dist: fastmcp>=2.14.0
15
+ Requires-Dist: filelock>=3.20.1
13
16
  Requires-Dist: flashrank>=0.2.0
14
17
  Requires-Dist: google-genai>=1.0.0
15
18
  Requires-Dist: greenlet>=3.2.4
16
19
  Requires-Dist: httpx>=0.27.0
20
+ Requires-Dist: langchain-core>=1.2.5
17
21
  Requires-Dist: langchain-text-splitters>=0.3.0
18
22
  Requires-Dist: openai>=1.0.0
19
23
  Requires-Dist: opentelemetry-api>=1.20.0
@@ -23,21 +27,23 @@ Requires-Dist: opentelemetry-sdk>=1.20.0
23
27
  Requires-Dist: pg0-embedded>=0.11.0
24
28
  Requires-Dist: pgvector>=0.4.1
25
29
  Requires-Dist: psycopg2-binary>=2.9.11
30
+ Requires-Dist: pyasn1>=0.6.2
26
31
  Requires-Dist: pydantic>=2.0.0
27
32
  Requires-Dist: python-dateutil>=2.8.0
28
33
  Requires-Dist: python-dotenv>=1.0.0
29
34
  Requires-Dist: rich>=13.0.0
30
- Requires-Dist: sentence-transformers<3.3.0,>=3.0.0
35
+ Requires-Dist: sentence-transformers>=3.3.0
31
36
  Requires-Dist: sqlalchemy>=2.0.44
32
37
  Requires-Dist: tiktoken>=0.12.0
33
- Requires-Dist: torch>=2.0.0
34
- Requires-Dist: transformers<4.46.0,>=4.30.0
38
+ Requires-Dist: torch>=2.6.0
39
+ Requires-Dist: transformers>=4.53.0
35
40
  Requires-Dist: typer>=0.9.0
41
+ Requires-Dist: urllib3>=2.6.3
36
42
  Requires-Dist: uvicorn>=0.38.0
37
43
  Requires-Dist: uvloop>=0.22.1
38
44
  Requires-Dist: wsproto>=1.0.0
39
45
  Provides-Extra: test
40
- Requires-Dist: filelock>=3.0.0; extra == 'test'
46
+ Requires-Dist: filelock>=3.20.1; extra == 'test'
41
47
  Requires-Dist: pytest-asyncio>=0.21.0; extra == 'test'
42
48
  Requires-Dist: pytest-timeout>=2.4.0; extra == 'test'
43
49
  Requires-Dist: pytest-xdist>=3.0.0; extra == 'test'
@@ -46,4 +46,4 @@ __all__ = [
46
46
  "RemoteTEICrossEncoder",
47
47
  "LLMConfig",
48
48
  ]
49
- __version__ = "0.1.0"
49
+ __version__ = "0.4.1"
@@ -244,6 +244,65 @@ def run_db_migration(
244
244
  typer.echo("Database migrations completed successfully")
245
245
 
246
246
 
247
+ async def _decommission_worker(db_url: str, worker_id: str, schema: str = "public") -> int:
248
+ """Release all tasks owned by a worker, setting them back to pending status."""
249
+ is_pg0, instance_name, _ = parse_pg0_url(db_url)
250
+ if is_pg0:
251
+ typer.echo(f"Starting embedded PostgreSQL (instance: {instance_name})...")
252
+ resolved_url = await resolve_database_url(db_url)
253
+
254
+ conn = await asyncpg.connect(resolved_url)
255
+ try:
256
+ table = _fq_table("async_operations", schema)
257
+ result = await conn.fetch(
258
+ f"""
259
+ UPDATE {table}
260
+ SET status = 'pending', worker_id = NULL, claimed_at = NULL, updated_at = now()
261
+ WHERE worker_id = $1 AND status = 'processing'
262
+ RETURNING operation_id
263
+ """,
264
+ worker_id,
265
+ )
266
+ return len(result)
267
+ finally:
268
+ await conn.close()
269
+
270
+
271
+ @app.command(name="decommission-worker")
272
+ def decommission_worker(
273
+ worker_id: str = typer.Argument(..., help="Worker ID to decommission"),
274
+ schema: str = typer.Option("public", "--schema", "-s", help="Database schema"),
275
+ yes: bool = typer.Option(False, "--yes", "-y", help="Skip confirmation prompt"),
276
+ ):
277
+ """Release all tasks owned by a worker (sets status back to pending).
278
+
279
+ Use this command when a worker has crashed or been removed without graceful shutdown.
280
+ All tasks that were being processed by the worker will be released back to the queue
281
+ so other workers can pick them up.
282
+ """
283
+ config = HindsightConfig.from_env()
284
+
285
+ if not config.database_url:
286
+ typer.echo("Error: Database URL not configured.", err=True)
287
+ typer.echo("Set HINDSIGHT_API_DATABASE_URL environment variable.", err=True)
288
+ raise typer.Exit(1)
289
+
290
+ if not yes:
291
+ typer.confirm(
292
+ f"This will release all tasks owned by worker '{worker_id}' back to pending. Continue?",
293
+ abort=True,
294
+ )
295
+
296
+ typer.echo(f"Decommissioning worker '{worker_id}' (schema: {schema})...")
297
+
298
+ count = asyncio.run(_decommission_worker(config.database_url, worker_id, schema))
299
+
300
+ if count > 0:
301
+ typer.echo(f"Released {count} task(s) from worker '{worker_id}'")
302
+ else:
303
+ typer.echo(f"No tasks found for worker '{worker_id}'")
304
+
305
+
247
306
  def main():
248
307
  app()
249
308
 
@@ -0,0 +1,112 @@
1
+ """mental_models_v4
2
+
3
+ Revision ID: h3c4d5e6f7g8
4
+ Revises: g2a3b4c5d6e7
5
+ Create Date: 2026-01-08 00:00:00.000000
6
+
7
+ This migration implements the v4 mental models system:
8
+ 1. Deletes existing observation memory_units (observations now in mental models)
9
+ 2. Adds mission column to banks (replacing background)
10
+ 3. Creates mental_models table with final schema
11
+
12
+ Mental models can reference entities when an entity is "promoted" to a mental model.
13
+ Summary content is stored as JSONB observations with per-observation fact attribution.
14
+ """
15
+
16
+ from collections.abc import Sequence
17
+
18
+ from alembic import context, op
19
+
20
+ # revision identifiers, used by Alembic.
21
+ revision: str = "h3c4d5e6f7g8"
22
+ down_revision: str | Sequence[str] | None = "g2a3b4c5d6e7"
23
+ branch_labels: str | Sequence[str] | None = None
24
+ depends_on: str | Sequence[str] | None = None
25
+
26
+
27
+ def _get_schema_prefix() -> str:
28
+ """Get schema prefix for table names (required for multi-tenant support)."""
29
+ schema = context.config.get_main_option("target_schema")
30
+ return f'"{schema}".' if schema else ""
31
+
32
+
33
+ def upgrade() -> None:
34
+ """Apply mental models v4 changes."""
35
+ schema = _get_schema_prefix()
36
+
37
+ # Step 1: Delete observation memory_units (cascades to unit_entities links)
38
+ # Observations are now handled through mental models, not memory_units
39
+ op.execute(f"DELETE FROM {schema}memory_units WHERE fact_type = 'observation'")
40
+
41
+ # Step 2: Drop observation-specific index (if it exists)
42
+ op.execute(f"DROP INDEX IF EXISTS {schema}idx_memory_units_observation_date")
43
+
44
+ # Step 3: Add mission column to banks (replacing background)
45
+ op.execute(f"ALTER TABLE {schema}banks ADD COLUMN IF NOT EXISTS mission TEXT")
46
+
47
+ # Migrate: copy background to mission if background column exists
48
+ # Use DO block to check column existence first (idempotent for re-runs)
49
+ schema_name = context.config.get_main_option("target_schema") or "public"
50
+ op.execute(f"""
51
+ DO $$
52
+ BEGIN
53
+ IF EXISTS (
54
+ SELECT 1 FROM information_schema.columns
55
+ WHERE table_schema = '{schema_name}' AND table_name = 'banks' AND column_name = 'background'
56
+ ) THEN
57
+ UPDATE {schema}banks
58
+ SET mission = background
59
+ WHERE mission IS NULL;
60
+ END IF;
61
+ END $$;
62
+ """)
63
+
64
+ # Remove background column (replaced by mission)
65
+ op.execute(f"ALTER TABLE {schema}banks DROP COLUMN IF EXISTS background")
66
+
67
+ # Step 4: Create mental_models table with final v4 schema (if not exists)
68
+ op.execute(f"""
69
+ CREATE TABLE IF NOT EXISTS {schema}mental_models (
70
+ id VARCHAR(64) NOT NULL,
71
+ bank_id VARCHAR(64) NOT NULL,
72
+ subtype VARCHAR(32) NOT NULL,
73
+ name VARCHAR(256) NOT NULL,
74
+ description TEXT NOT NULL,
75
+ entity_id UUID,
76
+ observations JSONB DEFAULT '{{"observations": []}}'::jsonb,
77
+ links VARCHAR[],
78
+ tags VARCHAR[] DEFAULT '{{}}',
79
+ last_updated TIMESTAMP WITH TIME ZONE,
80
+ created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(),
81
+ PRIMARY KEY (id, bank_id),
82
+ FOREIGN KEY (bank_id) REFERENCES {schema}banks(bank_id) ON DELETE CASCADE,
83
+ FOREIGN KEY (entity_id) REFERENCES {schema}entities(id) ON DELETE SET NULL,
84
+ CONSTRAINT ck_mental_models_subtype CHECK (subtype IN ('structural', 'emergent', 'pinned', 'learned'))
85
+ )
86
+ """)
87
+
88
+ # Step 5: Create indexes for efficient queries (if not exist)
89
+ op.execute(f"CREATE INDEX IF NOT EXISTS idx_mental_models_bank_id ON {schema}mental_models(bank_id)")
90
+ op.execute(f"CREATE INDEX IF NOT EXISTS idx_mental_models_subtype ON {schema}mental_models(bank_id, subtype)")
91
+ op.execute(f"CREATE INDEX IF NOT EXISTS idx_mental_models_entity_id ON {schema}mental_models(entity_id)")
92
+ # GIN index for efficient tags array filtering
93
+ op.execute(f"CREATE INDEX IF NOT EXISTS idx_mental_models_tags ON {schema}mental_models USING GIN(tags)")
94
+
95
+
96
+ def downgrade() -> None:
97
+ """Revert mental models v4 changes."""
98
+ schema = _get_schema_prefix()
99
+
100
+ # Drop mental_models table (cascades to indexes)
101
+ op.execute(f"DROP TABLE IF EXISTS {schema}mental_models CASCADE")
102
+
103
+ # Add back background column to banks
104
+ op.execute(f"ALTER TABLE {schema}banks ADD COLUMN IF NOT EXISTS background TEXT")
105
+
106
+ # Migrate mission back to background
107
+ op.execute(f"UPDATE {schema}banks SET background = mission WHERE background IS NULL")
108
+
109
+ # Remove mission column
110
+ op.execute(f"ALTER TABLE {schema}banks DROP COLUMN IF EXISTS mission")
111
+
112
+ # Note: Cannot restore deleted observations - they are lost on downgrade
@@ -0,0 +1,41 @@
1
+ """delete_opinions
2
+
3
+ Revision ID: i4d5e6f7g8h9
4
+ Revises: h3c4d5e6f7g8
5
+ Create Date: 2026-01-15 00:00:00.000000
6
+
7
+ This migration removes opinion facts from memory_units.
8
+ Opinions are no longer a separate fact type - they are now represented
9
+ through mental model observations with confidence scores.
10
+ """
11
+
12
+ from collections.abc import Sequence
13
+
14
+ from alembic import context, op
15
+
16
+ # revision identifiers, used by Alembic.
17
+ revision: str = "i4d5e6f7g8h9"
18
+ down_revision: str | Sequence[str] | None = "h3c4d5e6f7g8"
19
+ branch_labels: str | Sequence[str] | None = None
20
+ depends_on: str | Sequence[str] | None = None
21
+
22
+
23
+ def _get_schema_prefix() -> str:
24
+ """Get schema prefix for table names (required for multi-tenant support)."""
25
+ schema = context.config.get_main_option("target_schema")
26
+ return f'"{schema}".' if schema else ""
27
+
28
+
29
+ def upgrade() -> None:
30
+ """Delete opinion memory_units."""
31
+ schema = _get_schema_prefix()
32
+
33
+ # Delete opinion memory_units (cascades to unit_entities links)
34
+ # Opinions are now handled through mental model observations
35
+ op.execute(f"DELETE FROM {schema}memory_units WHERE fact_type = 'opinion'")
36
+
37
+
38
+ def downgrade() -> None:
39
+ """Cannot restore deleted opinions."""
40
+ # Note: Cannot restore deleted opinions - they are lost on downgrade
41
+ pass
@@ -0,0 +1,95 @@
1
+ """mental_model_versions
2
+
3
+ Revision ID: j5e6f7g8h9i0
4
+ Revises: i4d5e6f7g8h9
5
+ Create Date: 2026-01-16 00:00:00.000000
6
+
7
+ This migration adds versioning support for mental models:
8
+ 1. Creates mental_model_versions table to store observation snapshots
9
+ 2. Adds version column to mental_models for tracking current version
10
+
11
+ This enables changelog/diff functionality for mental model observations.
12
+ """
13
+
14
+ from collections.abc import Sequence
15
+
16
+ from alembic import context, op
17
+
18
+ # revision identifiers, used by Alembic.
19
+ revision: str = "j5e6f7g8h9i0"
20
+ down_revision: str | Sequence[str] | None = "i4d5e6f7g8h9"
21
+ branch_labels: str | Sequence[str] | None = None
22
+ depends_on: str | Sequence[str] | None = None
23
+
24
+
25
+ def _get_schema_prefix() -> str:
26
+ """Get schema prefix for table names (required for multi-tenant support)."""
27
+ schema = context.config.get_main_option("target_schema")
28
+ return f'"{schema}".' if schema else ""
29
+
30
+
31
+ def upgrade() -> None:
32
+ """Create mental_model_versions table and add version tracking."""
33
+ schema = _get_schema_prefix()
34
+
35
+ # Create mental_model_versions table for storing observation snapshots
36
+ op.execute(f"""
37
+ CREATE TABLE {schema}mental_model_versions (
38
+ id SERIAL PRIMARY KEY,
39
+ mental_model_id VARCHAR(64) NOT NULL,
40
+ bank_id VARCHAR(64) NOT NULL,
41
+ version INT NOT NULL,
42
+ observations JSONB NOT NULL DEFAULT '{{"observations": []}}'::jsonb,
43
+ created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(),
44
+ FOREIGN KEY (mental_model_id, bank_id)
45
+ REFERENCES {schema}mental_models(id, bank_id) ON DELETE CASCADE,
46
+ UNIQUE (mental_model_id, bank_id, version)
47
+ )
48
+ """)
49
+
50
+ # Index for efficient version queries (get latest, list versions)
51
+ op.execute(f"""
52
+ CREATE INDEX idx_mental_model_versions_lookup
53
+ ON {schema}mental_model_versions(mental_model_id, bank_id, version DESC)
54
+ """)
55
+
56
+ # Add version column to mental_models to track current version
57
+ op.execute(f"""
58
+ ALTER TABLE {schema}mental_models
59
+ ADD COLUMN IF NOT EXISTS version INT NOT NULL DEFAULT 0
60
+ """)
61
+
62
+ # Migrate existing mental models: create version 1 for any that have observations
63
+ op.execute(f"""
64
+ INSERT INTO {schema}mental_model_versions (mental_model_id, bank_id, version, observations, created_at)
65
+ SELECT id, bank_id, 1, observations, COALESCE(last_updated, created_at)
66
+ FROM {schema}mental_models
67
+ WHERE observations IS NOT NULL
68
+ AND observations != '{{"observations": []}}'::jsonb
69
+ AND (observations->'observations') IS NOT NULL
70
+ AND jsonb_array_length(observations->'observations') > 0
71
+ """)
72
+
73
+ # Update version to 1 for migrated mental models
74
+ op.execute(f"""
75
+ UPDATE {schema}mental_models
76
+ SET version = 1
77
+ WHERE observations IS NOT NULL
78
+ AND observations != '{{"observations": []}}'::jsonb
79
+ AND (observations->'observations') IS NOT NULL
80
+ AND jsonb_array_length(observations->'observations') > 0
81
+ """)
82
+
83
+
84
+ def downgrade() -> None:
85
+ """Remove mental_model_versions table and version column."""
86
+ schema = _get_schema_prefix()
87
+
88
+ # Drop index
89
+ op.execute(f"DROP INDEX IF EXISTS {schema}idx_mental_model_versions_lookup")
90
+
91
+ # Drop versions table
92
+ op.execute(f"DROP TABLE IF EXISTS {schema}mental_model_versions")
93
+
94
+ # Remove version column from mental_models
95
+ op.execute(f"ALTER TABLE {schema}mental_models DROP COLUMN IF EXISTS version")
@@ -0,0 +1,58 @@
1
+ """add_directive_subtype
2
+
3
+ Revision ID: k6f7g8h9i0j1
4
+ Revises: j5e6f7g8h9i0
5
+ Create Date: 2026-01-16 00:00:00.000000
6
+
7
+ This migration adds 'directive' to the mental_models subtype constraint.
8
+ Directives are hard rules with user-provided observations that the reflect agent must follow.
9
+ """
10
+
11
+ from collections.abc import Sequence
12
+
13
+ from alembic import context, op
14
+
15
+ # revision identifiers, used by Alembic.
16
+ revision: str = "k6f7g8h9i0j1"
17
+ down_revision: str | Sequence[str] | None = "j5e6f7g8h9i0"
18
+ branch_labels: str | Sequence[str] | None = None
19
+ depends_on: str | Sequence[str] | None = None
20
+
21
+
22
+ def _get_schema_prefix() -> str:
23
+ """Get schema prefix for table names (required for multi-tenant support)."""
24
+ schema = context.config.get_main_option("target_schema")
25
+ return f'"{schema}".' if schema else ""
26
+
27
+
28
+ def upgrade() -> None:
29
+ """Add 'directive' to mental_models subtype constraint."""
30
+ schema = _get_schema_prefix()
31
+
32
+ # Drop existing constraint
33
+ op.execute(f"ALTER TABLE {schema}mental_models DROP CONSTRAINT IF EXISTS ck_mental_models_subtype")
34
+
35
+ # Create new constraint with 'directive' added
36
+ op.execute(f"""
37
+ ALTER TABLE {schema}mental_models
38
+ ADD CONSTRAINT ck_mental_models_subtype
39
+ CHECK (subtype IN ('structural', 'emergent', 'pinned', 'learned', 'directive'))
40
+ """)
41
+
42
+
43
+ def downgrade() -> None:
44
+ """Remove 'directive' from mental_models subtype constraint."""
45
+ schema = _get_schema_prefix()
46
+
47
+ # First delete any directives (cannot downgrade if they exist)
48
+ op.execute(f"DELETE FROM {schema}mental_models WHERE subtype = 'directive'")
49
+
50
+ # Drop constraint with directive
51
+ op.execute(f"ALTER TABLE {schema}mental_models DROP CONSTRAINT IF EXISTS ck_mental_models_subtype")
52
+
53
+ # Recreate original constraint without directive
54
+ op.execute(f"""
55
+ ALTER TABLE {schema}mental_models
56
+ ADD CONSTRAINT ck_mental_models_subtype
57
+ CHECK (subtype IN ('structural', 'emergent', 'pinned', 'learned'))
58
+ """)
@@ -0,0 +1,109 @@
1
+ """add_worker_columns
2
+
3
+ Revision ID: l7g8h9i0j1k2
4
+ Revises: k6f7g8h9i0j1
5
+ Create Date: 2026-01-19 00:00:00.000000
6
+
7
+ This migration adds columns to async_operations for distributed worker support:
8
+ - worker_id: ID of the worker that claimed the task
9
+ - claimed_at: When the task was claimed
10
+ - retry_count: Number of retry attempts
11
+ - task_payload: The serialized task dictionary
12
+ """
13
+
14
+ from collections.abc import Sequence
15
+
16
+ import sqlalchemy as sa
17
+ from alembic import context, op
18
+ from sqlalchemy.dialects import postgresql
19
+
20
+ # revision identifiers, used by Alembic.
21
+ revision: str = "l7g8h9i0j1k2"
22
+ down_revision: str | Sequence[str] | None = "k6f7g8h9i0j1"
23
+ branch_labels: str | Sequence[str] | None = None
24
+ depends_on: str | Sequence[str] | None = None
25
+
26
+
27
+ def _get_schema_prefix() -> str:
28
+ """Get schema prefix for table names (required for multi-tenant support)."""
29
+ schema = context.config.get_main_option("target_schema")
30
+ return f'"{schema}".' if schema else ""
31
+
32
+
33
+ def upgrade() -> None:
34
+ """Add worker columns to async_operations."""
35
+ schema = _get_schema_prefix()
36
+
37
+ # Add worker_id column (ID of worker that claimed the task)
38
+ op.add_column(
39
+ "async_operations",
40
+ sa.Column("worker_id", sa.Text(), nullable=True),
41
+ schema=context.config.get_main_option("target_schema") or None,
42
+ )
43
+
44
+ # Add claimed_at column (when task was claimed by worker)
45
+ op.add_column(
46
+ "async_operations",
47
+ sa.Column("claimed_at", postgresql.TIMESTAMP(timezone=True), nullable=True),
48
+ schema=context.config.get_main_option("target_schema") or None,
49
+ )
50
+
51
+ # Add retry_count column (number of retry attempts)
52
+ op.add_column(
53
+ "async_operations",
54
+ sa.Column("retry_count", sa.Integer(), server_default="0", nullable=False),
55
+ schema=context.config.get_main_option("target_schema") or None,
56
+ )
57
+
58
+ # Add task_payload column (serialized task dictionary)
59
+ op.add_column(
60
+ "async_operations",
61
+ sa.Column(
62
+ "task_payload",
63
+ postgresql.JSONB(astext_type=sa.Text()),
64
+ nullable=True,
65
+ ),
66
+ schema=context.config.get_main_option("target_schema") or None,
67
+ )
68
+
69
+ # Add index for efficient worker polling (pending tasks ordered by creation time)
70
+ op.execute(
71
+ f"CREATE INDEX idx_async_operations_pending_claim ON {schema}async_operations (status, created_at) "
72
+ f"WHERE status = 'pending' AND task_payload IS NOT NULL"
73
+ )
74
+
75
+ # Add index for finding tasks by worker_id (for decommissioning)
76
+ op.execute(
77
+ f"CREATE INDEX idx_async_operations_worker_id ON {schema}async_operations (worker_id) WHERE worker_id IS NOT NULL"
78
+ )
79
+
80
+
81
+ def downgrade() -> None:
82
+ """Remove worker columns from async_operations."""
83
+ schema = _get_schema_prefix()
84
+
85
+ # Drop indexes
86
+ op.execute(f"DROP INDEX IF EXISTS {schema}idx_async_operations_pending_claim")
87
+ op.execute(f"DROP INDEX IF EXISTS {schema}idx_async_operations_worker_id")
88
+
89
+ # Drop columns
90
+ op.drop_column(
91
+ "async_operations",
92
+ "task_payload",
93
+ schema=context.config.get_main_option("target_schema") or None,
94
+ )
95
+ op.drop_column(
96
+ "async_operations",
97
+ "retry_count",
98
+ schema=context.config.get_main_option("target_schema") or None,
99
+ )
100
+ op.drop_column(
101
+ "async_operations",
102
+ "claimed_at",
103
+ schema=context.config.get_main_option("target_schema") or None,
104
+ )
105
+ op.drop_column(
106
+ "async_operations",
107
+ "worker_id",
108
+ schema=context.config.get_main_option("target_schema") or None,
109
+ )
@@ -0,0 +1,41 @@
1
+ """mental_model_id_to_text
2
+
3
+ Revision ID: m8h9i0j1k2l3
4
+ Revises: l7g8h9i0j1k2
5
+ Create Date: 2026-01-19 00:00:00.000000
6
+
7
+ This migration changes the mental_models.id column from VARCHAR(64) to TEXT
8
+ to support longer model IDs (e.g., entity names that exceed 64 characters).
9
+ """
10
+
11
+ from collections.abc import Sequence
12
+
13
+ from alembic import context, op
14
+
15
+ # revision identifiers, used by Alembic.
16
+ revision: str = "m8h9i0j1k2l3"
17
+ down_revision: str | Sequence[str] | None = "l7g8h9i0j1k2"
18
+ branch_labels: str | Sequence[str] | None = None
19
+ depends_on: str | Sequence[str] | None = None
20
+
21
+
22
+ def _get_schema_prefix() -> str:
23
+ """Get schema prefix for table names (required for multi-tenant support)."""
24
+ schema = context.config.get_main_option("target_schema")
25
+ return f'"{schema}".' if schema else ""
26
+
27
+
28
+ def upgrade() -> None:
29
+ """Change mental_models.id from VARCHAR(64) to TEXT."""
30
+ schema = _get_schema_prefix()
31
+
32
+ # Alter the id column type from VARCHAR(64) to TEXT
33
+ op.execute(f"ALTER TABLE {schema}mental_models ALTER COLUMN id TYPE TEXT")
34
+
35
+
36
+ def downgrade() -> None:
37
+ """Revert mental_models.id from TEXT to VARCHAR(64)."""
38
+ schema = _get_schema_prefix()
39
+
40
+ # Note: This may fail if any id values exceed 64 characters
41
+ op.execute(f"ALTER TABLE {schema}mental_models ALTER COLUMN id TYPE VARCHAR(64)")