hindsight-api 0.3.0__py3-none-any.whl → 0.4.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. hindsight_api/admin/cli.py +59 -0
  2. hindsight_api/alembic/versions/h3c4d5e6f7g8_mental_models_v4.py +112 -0
  3. hindsight_api/alembic/versions/i4d5e6f7g8h9_delete_opinions.py +41 -0
  4. hindsight_api/alembic/versions/j5e6f7g8h9i0_mental_model_versions.py +95 -0
  5. hindsight_api/alembic/versions/k6f7g8h9i0j1_add_directive_subtype.py +58 -0
  6. hindsight_api/alembic/versions/l7g8h9i0j1k2_add_worker_columns.py +109 -0
  7. hindsight_api/alembic/versions/m8h9i0j1k2l3_mental_model_id_to_text.py +41 -0
  8. hindsight_api/alembic/versions/n9i0j1k2l3m4_learnings_and_pinned_reflections.py +134 -0
  9. hindsight_api/alembic/versions/o0j1k2l3m4n5_migrate_mental_models_data.py +113 -0
  10. hindsight_api/alembic/versions/p1k2l3m4n5o6_new_knowledge_architecture.py +194 -0
  11. hindsight_api/alembic/versions/q2l3m4n5o6p7_fix_mental_model_fact_type.py +50 -0
  12. hindsight_api/alembic/versions/r3m4n5o6p7q8_add_reflect_response_to_reflections.py +47 -0
  13. hindsight_api/alembic/versions/s4n5o6p7q8r9_add_consolidated_at_to_memory_units.py +53 -0
  14. hindsight_api/alembic/versions/t5o6p7q8r9s0_rename_mental_models_to_observations.py +134 -0
  15. hindsight_api/alembic/versions/u6p7q8r9s0t1_mental_models_text_id.py +41 -0
  16. hindsight_api/alembic/versions/v7q8r9s0t1u2_add_max_tokens_to_mental_models.py +50 -0
  17. hindsight_api/api/http.py +1119 -93
  18. hindsight_api/api/mcp.py +11 -191
  19. hindsight_api/config.py +145 -45
  20. hindsight_api/engine/consolidation/__init__.py +5 -0
  21. hindsight_api/engine/consolidation/consolidator.py +859 -0
  22. hindsight_api/engine/consolidation/prompts.py +69 -0
  23. hindsight_api/engine/cross_encoder.py +114 -9
  24. hindsight_api/engine/directives/__init__.py +5 -0
  25. hindsight_api/engine/directives/models.py +37 -0
  26. hindsight_api/engine/embeddings.py +102 -5
  27. hindsight_api/engine/interface.py +32 -13
  28. hindsight_api/engine/llm_wrapper.py +505 -43
  29. hindsight_api/engine/memory_engine.py +2090 -1089
  30. hindsight_api/engine/mental_models/__init__.py +14 -0
  31. hindsight_api/engine/mental_models/models.py +53 -0
  32. hindsight_api/engine/reflect/__init__.py +18 -0
  33. hindsight_api/engine/reflect/agent.py +933 -0
  34. hindsight_api/engine/reflect/models.py +109 -0
  35. hindsight_api/engine/reflect/observations.py +186 -0
  36. hindsight_api/engine/reflect/prompts.py +483 -0
  37. hindsight_api/engine/reflect/tools.py +437 -0
  38. hindsight_api/engine/reflect/tools_schema.py +250 -0
  39. hindsight_api/engine/response_models.py +130 -4
  40. hindsight_api/engine/retain/bank_utils.py +79 -201
  41. hindsight_api/engine/retain/fact_extraction.py +81 -48
  42. hindsight_api/engine/retain/fact_storage.py +5 -8
  43. hindsight_api/engine/retain/link_utils.py +5 -8
  44. hindsight_api/engine/retain/orchestrator.py +1 -55
  45. hindsight_api/engine/retain/types.py +2 -2
  46. hindsight_api/engine/search/graph_retrieval.py +2 -2
  47. hindsight_api/engine/search/link_expansion_retrieval.py +164 -29
  48. hindsight_api/engine/search/mpfp_retrieval.py +1 -1
  49. hindsight_api/engine/search/retrieval.py +14 -14
  50. hindsight_api/engine/search/think_utils.py +41 -140
  51. hindsight_api/engine/search/trace.py +0 -1
  52. hindsight_api/engine/search/tracer.py +2 -5
  53. hindsight_api/engine/search/types.py +0 -3
  54. hindsight_api/engine/task_backend.py +112 -196
  55. hindsight_api/engine/utils.py +0 -151
  56. hindsight_api/extensions/__init__.py +10 -1
  57. hindsight_api/extensions/builtin/tenant.py +5 -1
  58. hindsight_api/extensions/operation_validator.py +81 -4
  59. hindsight_api/extensions/tenant.py +26 -0
  60. hindsight_api/main.py +16 -5
  61. hindsight_api/mcp_local.py +12 -53
  62. hindsight_api/mcp_tools.py +494 -0
  63. hindsight_api/models.py +0 -2
  64. hindsight_api/worker/__init__.py +11 -0
  65. hindsight_api/worker/main.py +296 -0
  66. hindsight_api/worker/poller.py +486 -0
  67. {hindsight_api-0.3.0.dist-info → hindsight_api-0.4.0.dist-info}/METADATA +12 -6
  68. hindsight_api-0.4.0.dist-info/RECORD +112 -0
  69. {hindsight_api-0.3.0.dist-info → hindsight_api-0.4.0.dist-info}/entry_points.txt +1 -0
  70. hindsight_api/engine/retain/observation_regeneration.py +0 -254
  71. hindsight_api/engine/search/observation_utils.py +0 -125
  72. hindsight_api/engine/search/scoring.py +0 -159
  73. hindsight_api-0.3.0.dist-info/RECORD +0 -82
  74. {hindsight_api-0.3.0.dist-info → hindsight_api-0.4.0.dist-info}/WHEEL +0 -0
@@ -0,0 +1,486 @@
1
+ """
2
+ Worker poller for distributed task execution.
3
+
4
+ Polls PostgreSQL for pending tasks and executes them using
5
+ FOR UPDATE SKIP LOCKED for safe concurrent claiming.
6
+ """
7
+
8
+ import asyncio
9
+ import json
10
+ import logging
11
+ import time
12
+ import traceback
13
+ from collections.abc import Awaitable, Callable
14
+ from dataclasses import dataclass
15
+ from typing import TYPE_CHECKING, Any
16
+
17
+ if TYPE_CHECKING:
18
+ import asyncpg
19
+
20
+ from hindsight_api.extensions.tenant import TenantExtension
21
+
22
+ logger = logging.getLogger(__name__)
23
+
24
+ # Progress logging interval in seconds
25
+ PROGRESS_LOG_INTERVAL = 30
26
+
27
+
28
+ def fq_table(table: str, schema: str | None = None) -> str:
29
+ """Get fully-qualified table name with optional schema prefix."""
30
+ if schema:
31
+ return f'"{schema}".{table}'
32
+ return table
33
+
34
+
35
+ @dataclass
36
+ class ClaimedTask:
37
+ """A task claimed from the database with its schema context."""
38
+
39
+ operation_id: str
40
+ task_dict: dict[str, Any]
41
+ schema: str | None
42
+
43
+
44
+ class WorkerPoller:
45
+ """
46
+ Polls PostgreSQL for pending tasks and executes them.
47
+
48
+ Uses FOR UPDATE SKIP LOCKED for safe distributed claiming,
49
+ allowing multiple workers to process tasks without conflicts.
50
+
51
+ Supports dynamic multi-tenant discovery via tenant_extension.
52
+ """
53
+
54
+ def __init__(
55
+ self,
56
+ pool: "asyncpg.Pool",
57
+ worker_id: str,
58
+ executor: Callable[[dict[str, Any]], Awaitable[None]],
59
+ poll_interval_ms: int = 500,
60
+ batch_size: int = 10,
61
+ max_retries: int = 3,
62
+ schema: str | None = None,
63
+ tenant_extension: "TenantExtension | None" = None,
64
+ ):
65
+ """
66
+ Initialize the worker poller.
67
+
68
+ Args:
69
+ pool: asyncpg connection pool
70
+ worker_id: Unique identifier for this worker
71
+ executor: Async function to execute tasks (typically MemoryEngine.execute_task)
72
+ poll_interval_ms: Interval between polls when no tasks found (milliseconds)
73
+ batch_size: Maximum number of tasks to claim per poll cycle
74
+ max_retries: Maximum retry attempts before marking task as failed
75
+ schema: Database schema for single-tenant support (ignored if tenant_extension is set)
76
+ tenant_extension: Extension for dynamic multi-tenant discovery. If set, list_tenants()
77
+ is called on each poll cycle to discover schemas dynamically.
78
+ """
79
+ self._pool = pool
80
+ self._worker_id = worker_id
81
+ self._executor = executor
82
+ self._poll_interval_ms = poll_interval_ms
83
+ self._batch_size = batch_size
84
+ self._max_retries = max_retries
85
+ self._schema = schema
86
+ self._tenant_extension = tenant_extension
87
+ self._shutdown = asyncio.Event()
88
+ self._current_tasks: set[asyncio.Task] = set()
89
+ self._in_flight_count = 0
90
+ self._in_flight_lock = asyncio.Lock()
91
+ self._last_progress_log = 0.0
92
+ self._tasks_completed_since_log = 0
93
+ # Track active tasks locally: operation_id -> (op_type, bank_id, schema)
94
+ self._active_tasks: dict[str, tuple[str, str, str | None]] = {}
95
+
96
+ async def _get_schemas(self) -> list[str | None]:
97
+ """Get list of schemas to poll. Returns [None] for public schema."""
98
+ if self._tenant_extension is not None:
99
+ tenants = await self._tenant_extension.list_tenants()
100
+ # Convert "public" to None for SQL compatibility, keep others as-is
101
+ return [t.schema if t.schema != "public" else None for t in tenants]
102
+ # Single schema mode
103
+ return [self._schema]
104
+
105
+ async def claim_batch(self) -> list[ClaimedTask]:
106
+ """
107
+ Claim up to batch_size pending tasks atomically across all tenant schemas.
108
+
109
+ Uses FOR UPDATE SKIP LOCKED to ensure no conflicts with other workers.
110
+
111
+ For consolidation tasks specifically, skips pending tasks if there's already
112
+ a processing consolidation for the same bank (to avoid duplicate work).
113
+
114
+ If tenant_extension is configured, dynamically discovers schemas on each call.
115
+
116
+ Returns:
117
+ List of ClaimedTask objects containing operation_id, task_dict, and schema
118
+ """
119
+ schemas = await self._get_schemas()
120
+ all_tasks: list[ClaimedTask] = []
121
+ remaining_batch = self._batch_size
122
+
123
+ for schema in schemas:
124
+ if remaining_batch <= 0:
125
+ break
126
+
127
+ tasks = await self._claim_batch_for_schema(schema, remaining_batch)
128
+ all_tasks.extend(tasks)
129
+ remaining_batch -= len(tasks)
130
+
131
+ return all_tasks
132
+
133
+ async def _claim_batch_for_schema(self, schema: str | None, limit: int) -> list[ClaimedTask]:
134
+ """Claim tasks from a specific schema."""
135
+ table = fq_table("async_operations", schema)
136
+
137
+ async with self._pool.acquire() as conn:
138
+ async with conn.transaction():
139
+ # Select and lock pending tasks
140
+ # For consolidation: skip if same bank already has one processing
141
+ rows = await conn.fetch(
142
+ f"""
143
+ SELECT operation_id, task_payload
144
+ FROM {table} AS pending
145
+ WHERE status = 'pending' AND task_payload IS NOT NULL
146
+ AND (
147
+ -- Non-consolidation tasks: always claimable
148
+ operation_type != 'consolidation'
149
+ OR
150
+ -- Consolidation: only if no other consolidation processing for same bank
151
+ NOT EXISTS (
152
+ SELECT 1 FROM {table} AS processing
153
+ WHERE processing.bank_id = pending.bank_id
154
+ AND processing.operation_type = 'consolidation'
155
+ AND processing.status = 'processing'
156
+ )
157
+ )
158
+ ORDER BY created_at
159
+ LIMIT $1
160
+ FOR UPDATE SKIP LOCKED
161
+ """,
162
+ limit,
163
+ )
164
+
165
+ if not rows:
166
+ return []
167
+
168
+ # Claim the tasks by updating status and worker_id
169
+ operation_ids = [row["operation_id"] for row in rows]
170
+ await conn.execute(
171
+ f"""
172
+ UPDATE {table}
173
+ SET status = 'processing', worker_id = $1, claimed_at = now(), updated_at = now()
174
+ WHERE operation_id = ANY($2)
175
+ """,
176
+ self._worker_id,
177
+ operation_ids,
178
+ )
179
+
180
+ # Parse and return task payloads with schema context
181
+ return [
182
+ ClaimedTask(
183
+ operation_id=str(row["operation_id"]),
184
+ task_dict=json.loads(row["task_payload"]),
185
+ schema=schema,
186
+ )
187
+ for row in rows
188
+ ]
189
+
190
+ async def _mark_completed(self, operation_id: str, schema: str | None):
191
+ """Mark a task as completed."""
192
+ table = fq_table("async_operations", schema)
193
+ await self._pool.execute(
194
+ f"""
195
+ UPDATE {table}
196
+ SET status = 'completed', completed_at = now(), updated_at = now()
197
+ WHERE operation_id = $1
198
+ """,
199
+ operation_id,
200
+ )
201
+
202
+ async def _mark_failed(self, operation_id: str, error_message: str, schema: str | None):
203
+ """Mark a task as failed with error message."""
204
+ table = fq_table("async_operations", schema)
205
+ # Truncate error message if too long (max 5000 chars in schema)
206
+ error_message = error_message[:5000] if len(error_message) > 5000 else error_message
207
+ await self._pool.execute(
208
+ f"""
209
+ UPDATE {table}
210
+ SET status = 'failed', error_message = $2, completed_at = now(), updated_at = now()
211
+ WHERE operation_id = $1
212
+ """,
213
+ operation_id,
214
+ error_message,
215
+ )
216
+
217
+ async def _retry_or_fail(self, operation_id: str, error_message: str, schema: str | None):
218
+ """Increment retry count or mark as failed if max retries exceeded."""
219
+ table = fq_table("async_operations", schema)
220
+
221
+ # Get current retry count
222
+ row = await self._pool.fetchrow(
223
+ f"SELECT retry_count FROM {table} WHERE operation_id = $1",
224
+ operation_id,
225
+ )
226
+
227
+ if row is None:
228
+ logger.warning(f"Operation {operation_id} not found, cannot retry")
229
+ return
230
+
231
+ retry_count = row["retry_count"]
232
+
233
+ if retry_count >= self._max_retries:
234
+ # Max retries exceeded, mark as failed
235
+ await self._mark_failed(
236
+ operation_id, f"Max retries ({self._max_retries}) exceeded. Last error: {error_message}", schema
237
+ )
238
+ logger.error(f"Task {operation_id} failed after {retry_count} retries")
239
+ else:
240
+ # Increment retry and reset to pending
241
+ await self._pool.execute(
242
+ f"""
243
+ UPDATE {table}
244
+ SET status = 'pending', worker_id = NULL, claimed_at = NULL,
245
+ retry_count = retry_count + 1, updated_at = now()
246
+ WHERE operation_id = $1
247
+ """,
248
+ operation_id,
249
+ )
250
+ logger.warning(f"Task {operation_id} failed, will retry (attempt {retry_count + 1}/{self._max_retries})")
251
+
252
+ async def execute_task(self, task: ClaimedTask):
253
+ """Execute a single task and update its status."""
254
+ task_type = task.task_dict.get("type", "unknown")
255
+ bank_id = task.task_dict.get("bank_id", "unknown")
256
+
257
+ # Track this task as active
258
+ async with self._in_flight_lock:
259
+ self._active_tasks[task.operation_id] = (task_type, bank_id, task.schema)
260
+
261
+ try:
262
+ schema_info = f", schema={task.schema}" if task.schema else ""
263
+ logger.debug(f"Executing task {task.operation_id} (type={task_type}, bank={bank_id}{schema_info})")
264
+ # Pass schema to executor so it can set the correct context
265
+ if task.schema:
266
+ task.task_dict["_schema"] = task.schema
267
+ await self._executor(task.task_dict)
268
+ await self._mark_completed(task.operation_id, task.schema)
269
+ logger.debug(f"Task {task.operation_id} completed successfully")
270
+ except Exception as e:
271
+ error_msg = f"{type(e).__name__}: {e}\n{traceback.format_exc()}"
272
+ logger.error(f"Task {task.operation_id} failed: {e}")
273
+ await self._retry_or_fail(task.operation_id, error_msg, task.schema)
274
+ finally:
275
+ # Remove from active tasks
276
+ async with self._in_flight_lock:
277
+ self._active_tasks.pop(task.operation_id, None)
278
+
279
+ async def recover_own_tasks(self) -> int:
280
+ """
281
+ Recover tasks that were assigned to this worker but not completed.
282
+
283
+ This handles the case where a worker crashes while processing tasks.
284
+ On startup, we reset any tasks stuck in 'processing' for this worker_id
285
+ back to 'pending' so they can be picked up again.
286
+
287
+ If tenant_extension is configured, recovers across all tenant schemas.
288
+
289
+ Returns:
290
+ Number of tasks recovered
291
+ """
292
+ schemas = await self._get_schemas()
293
+ total_count = 0
294
+
295
+ for schema in schemas:
296
+ table = fq_table("async_operations", schema)
297
+
298
+ result = await self._pool.execute(
299
+ f"""
300
+ UPDATE {table}
301
+ SET status = 'pending', worker_id = NULL, claimed_at = NULL, updated_at = now()
302
+ WHERE status = 'processing' AND worker_id = $1
303
+ """,
304
+ self._worker_id,
305
+ )
306
+
307
+ # Parse "UPDATE N" to get count
308
+ count = int(result.split()[-1]) if result else 0
309
+ total_count += count
310
+
311
+ if total_count > 0:
312
+ logger.info(f"Worker {self._worker_id} recovered {total_count} stale tasks from previous run")
313
+ return total_count
314
+
315
+ async def run(self):
316
+ """
317
+ Main polling loop.
318
+
319
+ Continuously polls for pending tasks, claims them, and executes them
320
+ until shutdown is signaled.
321
+
322
+ If tenant_extension is configured, dynamically discovers schemas on each poll.
323
+ """
324
+ # Recover any tasks from a previous crash before starting
325
+ await self.recover_own_tasks()
326
+
327
+ logger.info(f"Worker {self._worker_id} starting polling loop")
328
+
329
+ while not self._shutdown.is_set():
330
+ try:
331
+ # Claim a batch of tasks (across all tenant schemas if configured)
332
+ tasks = await self.claim_batch()
333
+
334
+ if tasks:
335
+ # Log batch info
336
+ task_types: dict[str, int] = {}
337
+ schemas_seen: set[str | None] = set()
338
+ for task in tasks:
339
+ t = task.task_dict.get("type", "unknown")
340
+ task_types[t] = task_types.get(t, 0) + 1
341
+ schemas_seen.add(task.schema)
342
+ types_str = ", ".join(f"{k}:{v}" for k, v in task_types.items())
343
+ schemas_str = ", ".join(s or "public" for s in schemas_seen)
344
+ logger.info(
345
+ f"Worker {self._worker_id} claimed {len(tasks)} tasks: {types_str} (schemas: {schemas_str})"
346
+ )
347
+
348
+ # Track in-flight tasks
349
+ async with self._in_flight_lock:
350
+ self._in_flight_count += len(tasks)
351
+
352
+ # Execute tasks concurrently
353
+ try:
354
+ await asyncio.gather(
355
+ *[self.execute_task(task) for task in tasks],
356
+ return_exceptions=True,
357
+ )
358
+ finally:
359
+ async with self._in_flight_lock:
360
+ self._in_flight_count -= len(tasks)
361
+ else:
362
+ # No tasks found, wait before polling again
363
+ try:
364
+ await asyncio.wait_for(
365
+ self._shutdown.wait(),
366
+ timeout=self._poll_interval_ms / 1000,
367
+ )
368
+ except asyncio.TimeoutError:
369
+ pass # Normal timeout, continue polling
370
+
371
+ # Log progress stats periodically
372
+ await self._log_progress_if_due()
373
+
374
+ except asyncio.CancelledError:
375
+ logger.info(f"Worker {self._worker_id} polling loop cancelled")
376
+ break
377
+ except Exception as e:
378
+ logger.error(f"Worker {self._worker_id} error in polling loop: {e}")
379
+ traceback.print_exc()
380
+ # Backoff on error
381
+ await asyncio.sleep(1)
382
+
383
+ logger.info(f"Worker {self._worker_id} polling loop stopped")
384
+
385
+ async def shutdown_graceful(self, timeout: float = 30.0):
386
+ """
387
+ Signal shutdown and wait for current tasks to complete.
388
+
389
+ Args:
390
+ timeout: Maximum time to wait for in-flight tasks (seconds)
391
+ """
392
+ logger.info(f"Worker {self._worker_id} initiating graceful shutdown")
393
+ self._shutdown.set()
394
+
395
+ # Wait for in-flight tasks to complete
396
+ start_time = asyncio.get_event_loop().time()
397
+ while asyncio.get_event_loop().time() - start_time < timeout:
398
+ async with self._in_flight_lock:
399
+ in_flight = self._in_flight_count
400
+
401
+ if in_flight == 0:
402
+ logger.info(f"Worker {self._worker_id} graceful shutdown complete")
403
+ return
404
+
405
+ logger.info(f"Worker {self._worker_id} waiting for {in_flight} in-flight tasks")
406
+ await asyncio.sleep(0.5)
407
+
408
+ logger.warning(f"Worker {self._worker_id} shutdown timeout after {timeout}s")
409
+
410
+ async def _log_progress_if_due(self):
411
+ """Log progress stats every PROGRESS_LOG_INTERVAL seconds."""
412
+ now = time.time()
413
+ if now - self._last_progress_log < PROGRESS_LOG_INTERVAL:
414
+ return
415
+
416
+ self._last_progress_log = now
417
+
418
+ try:
419
+ # Get local active tasks (this worker only)
420
+ async with self._in_flight_lock:
421
+ in_flight = self._in_flight_count
422
+ active_tasks = dict(self._active_tasks) # Copy to avoid holding lock
423
+
424
+ # Build local processing breakdown grouped by (op_type, bank_id)
425
+ task_groups: dict[tuple[str, str], int] = {}
426
+ for op_type, bank_id, _ in active_tasks.values():
427
+ key = (op_type, bank_id)
428
+ task_groups[key] = task_groups.get(key, 0) + 1
429
+
430
+ processing_info = [f"{op}:{bank}({cnt})" for (op, bank), cnt in task_groups.items()]
431
+ processing_str = ", ".join(processing_info[:10]) if processing_info else "none"
432
+ if len(processing_info) > 10:
433
+ processing_str += f" +{len(processing_info) - 10} more"
434
+
435
+ # Get global stats from DB across all schemas
436
+ schemas = await self._get_schemas()
437
+ global_pending = 0
438
+ all_worker_counts: dict[str, int] = {}
439
+
440
+ async with self._pool.acquire() as conn:
441
+ for schema in schemas:
442
+ table = fq_table("async_operations", schema)
443
+
444
+ row = await conn.fetchrow(f"SELECT COUNT(*) as count FROM {table} WHERE status = 'pending'")
445
+ global_pending += row["count"] if row else 0
446
+
447
+ # Get processing breakdown by worker
448
+ worker_rows = await conn.fetch(
449
+ f"""
450
+ SELECT worker_id, COUNT(*) as count
451
+ FROM {table}
452
+ WHERE status = 'processing'
453
+ GROUP BY worker_id
454
+ """
455
+ )
456
+ for wr in worker_rows:
457
+ wid = wr["worker_id"] or "unknown"
458
+ all_worker_counts[wid] = all_worker_counts.get(wid, 0) + wr["count"]
459
+
460
+ # Format other workers' processing counts
461
+ other_workers = []
462
+ for wid, cnt in all_worker_counts.items():
463
+ if wid != self._worker_id:
464
+ other_workers.append(f"{wid}:{cnt}")
465
+ others_str = ", ".join(other_workers) if other_workers else "none"
466
+
467
+ schemas_str = ", ".join(s or "public" for s in schemas)
468
+ logger.info(
469
+ f"[WORKER_STATS] worker={self._worker_id} in_flight={in_flight} | "
470
+ f"global: pending={global_pending} (schemas: {schemas_str}) | "
471
+ f"others: {others_str} | "
472
+ f"my_active: {processing_str}"
473
+ )
474
+
475
+ except Exception as e:
476
+ logger.debug(f"Failed to log progress stats: {e}")
477
+
478
+ @property
479
+ def worker_id(self) -> str:
480
+ """Get the worker ID."""
481
+ return self._worker_id
482
+
483
+ @property
484
+ def is_shutdown(self) -> bool:
485
+ """Check if shutdown has been signaled."""
486
+ return self._shutdown.is_set()
@@ -1,19 +1,23 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: hindsight-api
3
- Version: 0.3.0
3
+ Version: 0.4.0
4
4
  Summary: Hindsight: Agent Memory That Works Like Human Memory
5
5
  Requires-Python: >=3.11
6
+ Requires-Dist: aiohttp>=3.13.3
6
7
  Requires-Dist: alembic>=1.17.1
7
8
  Requires-Dist: anthropic>=0.40.0
8
9
  Requires-Dist: asyncpg>=0.29.0
10
+ Requires-Dist: authlib>=1.6.6
9
11
  Requires-Dist: cohere>=5.0.0
10
12
  Requires-Dist: dateparser>=1.2.2
11
13
  Requires-Dist: fastapi[standard]>=0.120.3
12
- Requires-Dist: fastmcp>=2.3.0
14
+ Requires-Dist: fastmcp>=2.14.0
15
+ Requires-Dist: filelock>=3.20.1
13
16
  Requires-Dist: flashrank>=0.2.0
14
17
  Requires-Dist: google-genai>=1.0.0
15
18
  Requires-Dist: greenlet>=3.2.4
16
19
  Requires-Dist: httpx>=0.27.0
20
+ Requires-Dist: langchain-core>=1.2.5
17
21
  Requires-Dist: langchain-text-splitters>=0.3.0
18
22
  Requires-Dist: openai>=1.0.0
19
23
  Requires-Dist: opentelemetry-api>=1.20.0
@@ -23,21 +27,23 @@ Requires-Dist: opentelemetry-sdk>=1.20.0
23
27
  Requires-Dist: pg0-embedded>=0.11.0
24
28
  Requires-Dist: pgvector>=0.4.1
25
29
  Requires-Dist: psycopg2-binary>=2.9.11
30
+ Requires-Dist: pyasn1>=0.6.2
26
31
  Requires-Dist: pydantic>=2.0.0
27
32
  Requires-Dist: python-dateutil>=2.8.0
28
33
  Requires-Dist: python-dotenv>=1.0.0
29
34
  Requires-Dist: rich>=13.0.0
30
- Requires-Dist: sentence-transformers<3.3.0,>=3.0.0
35
+ Requires-Dist: sentence-transformers>=3.3.0
31
36
  Requires-Dist: sqlalchemy>=2.0.44
32
37
  Requires-Dist: tiktoken>=0.12.0
33
- Requires-Dist: torch>=2.0.0
34
- Requires-Dist: transformers<4.46.0,>=4.30.0
38
+ Requires-Dist: torch>=2.6.0
39
+ Requires-Dist: transformers>=4.53.0
35
40
  Requires-Dist: typer>=0.9.0
41
+ Requires-Dist: urllib3>=2.6.3
36
42
  Requires-Dist: uvicorn>=0.38.0
37
43
  Requires-Dist: uvloop>=0.22.1
38
44
  Requires-Dist: wsproto>=1.0.0
39
45
  Provides-Extra: test
40
- Requires-Dist: filelock>=3.0.0; extra == 'test'
46
+ Requires-Dist: filelock>=3.20.1; extra == 'test'
41
47
  Requires-Dist: pytest-asyncio>=0.21.0; extra == 'test'
42
48
  Requires-Dist: pytest-timeout>=2.4.0; extra == 'test'
43
49
  Requires-Dist: pytest-xdist>=3.0.0; extra == 'test'
@@ -0,0 +1,112 @@
1
+ hindsight_api/__init__.py,sha256=lPhgtKMvT8qjORFKWlhlq-LVdwesIu0gbUYNPZQEFiI,1197
2
+ hindsight_api/banner.py,sha256=BXn-jhkXe4xi-YV4JeuaVvjYhTMs96O43XoOMv4Cd28,4591
3
+ hindsight_api/config.py,sha256=62AUvTZKnwTz_ow-vv_H78HYAx6_sstHNnFDhuWHAFQ,24882
4
+ hindsight_api/daemon.py,sha256=3CKcO_ENQ57dIWrTsmYUj-V4zvoAB1toNtVh3EVkg-c,5982
5
+ hindsight_api/main.py,sha256=xohICtRB-hPtwGgbL46UY0U-SfKCGU8tLHWpLYz7GUc,13396
6
+ hindsight_api/mcp_local.py,sha256=fJnCxMBc79GlBZrma94Ux6g-GVuh-W66194cqQdkKJQ,5613
7
+ hindsight_api/mcp_tools.py,sha256=KGzgDeRoChwgt3HB-OoUHcWgHz6ELequLIkw6u7kkyo,19669
8
+ hindsight_api/metrics.py,sha256=go3X7wyFAPkc55HFvu7esiaJXDrUsrSrC8Pq5NjcqU0,20692
9
+ hindsight_api/migrations.py,sha256=V4QL_N1cMe6kNF1ejJ3lPIPFXKU2Pzbaiviws7AyMIY,14624
10
+ hindsight_api/models.py,sha256=SzJ8uM2nGr3D6X-UEfE8VIT-PbS9J4DmRT_4lv5n9T8,12831
11
+ hindsight_api/pg0.py,sha256=XORoiemECidQgBP53EBSCF3i0PJegLRRWKl2hU5UPhE,6390
12
+ hindsight_api/server.py,sha256=MU2ZvKe3KWfxKYZq8EEJPgKMmq5diPkRqfQBaz-yOQI,2483
13
+ hindsight_api/admin/__init__.py,sha256=RvaczuwTxg6ajc_Jlk0EhVz5JqlNB3T8su060gRQwfs,26
14
+ hindsight_api/admin/cli.py,sha256=A1qkZ_9GWjz1qOIQYnmj-qUN005cIIlpFsvYH7tZdyc,11607
15
+ hindsight_api/alembic/README,sha256=MVlc9TYmr57RbhXET6QxgyCcwWP7w-vLkEsirENqiIQ,38
16
+ hindsight_api/alembic/env.py,sha256=I4sGdtUo8xcXe95MyD36JQeMod_Bvp9JUkW64Ve4XSM,5808
17
+ hindsight_api/alembic/script.py.mako,sha256=04kgeBtNMa4cCnG8CfQcKt6P6rnloIfj8wy0u_DBydM,704
18
+ hindsight_api/alembic/versions/5a366d414dce_initial_schema.py,sha256=g3G7fV70Z10PZxwTrTmR34OAlEZjQTLJKr-Ol54JqrQ,17665
19
+ hindsight_api/alembic/versions/b7c4d8e9f1a2_add_chunks_table.py,sha256=MaHFU4JczUIFLeUMBTKIV3ocuclil55N9fPPim-HRfk,2599
20
+ hindsight_api/alembic/versions/c8e5f2a3b4d1_add_retain_params_to_documents.py,sha256=ChqkHANauZb4-nBt2uepoZN3q0vRzN6aRsWTGueULiA,1146
21
+ hindsight_api/alembic/versions/d9f6a3b4c5e2_rename_bank_to_interactions.py,sha256=s5_B2D0JdaxO7WM-vWC5Yt6hAtTsAUzJhFGLFSkfuQU,1808
22
+ hindsight_api/alembic/versions/e0a1b2c3d4e5_disposition_to_3_traits.py,sha256=IdDP6fgsYj5fCXAF0QT-3t_wcKJsnf7B0mh7qS-cf_w,3806
23
+ hindsight_api/alembic/versions/f1a2b3c4d5e6_add_memory_links_composite_index.py,sha256=tqkOLVD_p1NXVsIRxAc1mBiNpEosU9WkwrNUEGbc9DY,1598
24
+ hindsight_api/alembic/versions/g2a3b4c5d6e7_add_tags_column.py,sha256=4P7OGJf2t9IWxI0wi8ibC3mrQzjWJaTZ5z5QPr67gig,1772
25
+ hindsight_api/alembic/versions/h3c4d5e6f7g8_mental_models_v4.py,sha256=Pus5r0o_cVlbMOJg8kf6ZxV0Z1n1CvjmmIrq_cqgKrs,4687
26
+ hindsight_api/alembic/versions/i4d5e6f7g8h9_delete_opinions.py,sha256=UAqopJkEkggNJO3aklK5pNr2WdFvqpvnRRAPTn0_6kE,1281
27
+ hindsight_api/alembic/versions/j5e6f7g8h9i0_mental_model_versions.py,sha256=k2JVQtgZnxYr7tbLDrtxgTvATsshA_dnC67A23zgxoA,3536
28
+ hindsight_api/alembic/versions/k6f7g8h9i0j1_add_directive_subtype.py,sha256=PophnJ_irT0TznA0f0acrm_OMPaukxJelWRjeRD5cvU,1990
29
+ hindsight_api/alembic/versions/l7g8h9i0j1k2_add_worker_columns.py,sha256=BjwvxbiwCSAc9OpPc8iZFvK-812xBhcZcjtdluH4dpg,3590
30
+ hindsight_api/alembic/versions/m8h9i0j1k2l3_mental_model_id_to_text.py,sha256=wBP2eQQLkq-EvRp3godGOgj3rWc1wc67v8GSnxHxTGU,1319
31
+ hindsight_api/alembic/versions/n9i0j1k2l3m4_learnings_and_pinned_reflections.py,sha256=Gv8QeFiUhhoeTW33606_Y38BsyJinQPBSgP_cRUPqwY,5008
32
+ hindsight_api/alembic/versions/o0j1k2l3m4n5_migrate_mental_models_data.py,sha256=iB2nKkzYoHFhH2m8VPlx3yGGFK0uypXXNIPxLZA-Sgo,4165
33
+ hindsight_api/alembic/versions/p1k2l3m4n5o6_new_knowledge_architecture.py,sha256=XnlmL-kAixgNfFPcGV7In9GfjrsstivSA9ukrY7a99I,7978
34
+ hindsight_api/alembic/versions/q2l3m4n5o6p7_fix_mental_model_fact_type.py,sha256=ve1pEaSiKnorifBeLFvh7rI33fUPV9CZCZV52eTcUzg,1740
35
+ hindsight_api/alembic/versions/r3m4n5o6p7q8_add_reflect_response_to_reflections.py,sha256=TPht7sDAQGxxJ6_Y62NrMygpWe_5MK3fvoJLVJUctdo,1385
36
+ hindsight_api/alembic/versions/rename_personality_to_disposition.py,sha256=A29-nDJ2Re4u9jdp2sUw29It808j4h6BpcA4wDHJMJ8,2765
37
+ hindsight_api/alembic/versions/s4n5o6p7q8r9_add_consolidated_at_to_memory_units.py,sha256=UYnaPSsagPWKSTISBErVPbGQXm7Cqf_yZjXnzfcen7k,1643
38
+ hindsight_api/alembic/versions/t5o6p7q8r9s0_rename_mental_models_to_observations.py,sha256=vUaQ2l3FUx4MJb_9m3H752FhTjrbsd82mBxe3Ti2nug,5448
39
+ hindsight_api/alembic/versions/u6p7q8r9s0t1_mental_models_text_id.py,sha256=uvil81f-4ag2dIxBXUGKZ5vxkqdNQRpxCWj_iVih09w,1355
40
+ hindsight_api/alembic/versions/v7q8r9s0t1u2_add_max_tokens_to_mental_models.py,sha256=Mw68uW8PK-SaHcYcqb41vWI0R22t70SSasNS2Myeoec,1656
41
+ hindsight_api/api/__init__.py,sha256=npF0AAy8WJhHF5a9ehkNn9_iYLk7RQOk2gdkdFb49Hk,3840
42
+ hindsight_api/api/http.py,sha256=s2v_tyc1-46qcztCNA-RNSShi2LdXSahyg0Jzkq5hrU,133313
43
+ hindsight_api/api/mcp.py,sha256=zV0TmkxKEqwhLIfNAdezYgsZ1PF9Lo8j5_lD73ULpKU,6707
44
+ hindsight_api/engine/__init__.py,sha256=-BwaSwG9fTT_BBO0c_2MBkxG6-tGdclSzIqsgHw4cnw,1633
45
+ hindsight_api/engine/cross_encoder.py,sha256=i_HWegrD5LyGJjb0VCt2SZRoiSMLcwR1A2CU4NvA6uM,35034
46
+ hindsight_api/engine/db_budget.py,sha256=1OmZiuszpuEaYz355QlOqwaupXPd9FrnbyENsFboBkg,8642
47
+ hindsight_api/engine/db_utils.py,sha256=Fq1pXETt8ZPhkWYjrcGbgL6glrwmCGWh3_lYJgHqQPo,3067
48
+ hindsight_api/engine/embeddings.py,sha256=bOt91cM2buXgZIH1qyA8lmaKeOcRzKtbrf_iNiGuW1E,29527
49
+ hindsight_api/engine/entity_resolver.py,sha256=qVvWJHnbGEfh0iUFtc1dbM3IUNwPMsQsmg2rMgiX2DY,23794
50
+ hindsight_api/engine/interface.py,sha256=rldxkBmp_bqEeTBD713uZeXvrqJB9Ix1L62gazlNEi0,16899
51
+ hindsight_api/engine/llm_wrapper.py,sha256=Mh38zSlNGhsbN0f2VA1JGZ52HRab_ndcKqvEhyajgK0,68084
52
+ hindsight_api/engine/memory_engine.py,sha256=G3AOo2t95X6q8yiU4dhk8gKKhghYdtZ36IDNqyzReAA,231516
53
+ hindsight_api/engine/query_analyzer.py,sha256=7APe0MjBcUxjivcMlM03PmMk_w5FjWvlEe20yAJlHlc,19741
54
+ hindsight_api/engine/response_models.py,sha256=1fNAFPztlmYfOaoRfwYyrhzdPBO9UL8QHFNXW6Lmjgg,16322
55
+ hindsight_api/engine/task_backend.py,sha256=zDH24tTwIH_59eFpQzepv0KkZXOIVMpmDkrg1Y5khDA,8172
56
+ hindsight_api/engine/utils.py,sha256=OtEFDViKcCpFmKN3Qir8YV4zp0kv7iaREcgDXCkwShw,2089
57
+ hindsight_api/engine/consolidation/__init__.py,sha256=qEUPy0R7akNoAooQL1TAt2rVasjvnXTcNzh2zpN0flc,160
58
+ hindsight_api/engine/consolidation/consolidator.py,sha256=VbAc1Kw3FRh5ppQ-6RzlcCEzpbCzSXvei8Kxc6_Zd1c,30536
59
+ hindsight_api/engine/consolidation/prompts.py,sha256=6N1E3_C8kgIhdybXLvVqO2-hvqfTCSd4JpZxhKCnF6c,3137
60
+ hindsight_api/engine/directives/__init__.py,sha256=5ZxaRqZVyJckbGElaI2DMRMBtnj-qYkxRKdnOHBwovA,118
61
+ hindsight_api/engine/directives/models.py,sha256=PKxvmhW1-fjBITAOBu7RKX5Lj61c2jdsTaX8ADelKag,1523
62
+ hindsight_api/engine/mental_models/__init__.py,sha256=TU6dSPyIsevFDgY6PLYctDsk5K4SA4pFSQnmQvbdRlA,488
63
+ hindsight_api/engine/mental_models/models.py,sha256=DjgumJE7LvbMVpv90aMkGhIWOZ3ZrXM2DFAqHuGerAs,2102
64
+ hindsight_api/engine/reflect/__init__.py,sha256=r70r-Y9LElHIL3EsvImO1KIL1sT_ubr1lC0IH5kH6O0,484
65
+ hindsight_api/engine/reflect/agent.py,sha256=mr0rUrwdnDISt9iyuspI_ZhL4qfyLTWAzJO9EAIEctM,37610
66
+ hindsight_api/engine/reflect/models.py,sha256=ZnMCi4sta5bSVGRRNatTA2jNSun59mWEVVq6Dkmjq1Q,5185
67
+ hindsight_api/engine/reflect/observations.py,sha256=TMivY5ujrJAOhG3OFFTGEuwbW27AOjyD5l0JlDfbyJM,6777
68
+ hindsight_api/engine/reflect/prompts.py,sha256=AewEMBJ_fk0pAVtWQagPMY_0lR9mHNJQ7kF4I5ruWmg,18442
69
+ hindsight_api/engine/reflect/tools.py,sha256=QQVIzEGf6gEDkEr3_PwoIaRJKDhmQ8uEFtfsLZ63HMU,14550
70
+ hindsight_api/engine/reflect/tools_schema.py,sha256=cdRmRwge5aHlVY19Y0Tlp5zEcFCnvxUc2GAXmwaNcMU,9944
71
+ hindsight_api/engine/retain/__init__.py,sha256=t6q3-_kf4iYTl9j2PVB6laqMSs6UuPeXBSYMW6HT1sA,1152
72
+ hindsight_api/engine/retain/bank_utils.py,sha256=LsFiB5rqyI1deL4rePAmPdmxMBb4paDA8UqMvY2Y40g,7696
73
+ hindsight_api/engine/retain/chunk_storage.py,sha256=zXAqbcFeYpjyWlOoi8zeK5G91zHpF75CUVF-6wsEJpU,2064
74
+ hindsight_api/engine/retain/deduplication.py,sha256=kqs7I7eIc_ppvgAF9GlzL6fSGuEEzrgw17-7NdyUDis,3099
75
+ hindsight_api/engine/retain/embedding_processing.py,sha256=R35oyKYIKjuqC-yZl5Ru56F8xRe0N6KW_9p5PZ9CBi0,1649
76
+ hindsight_api/engine/retain/embedding_utils.py,sha256=uulXIBiA7XNsj16K1VGawR3s5jV-hsAmvmoCi-IodpU,1565
77
+ hindsight_api/engine/retain/entity_processing.py,sha256=0x5b48Im7pWjeqg3xTMIRVhrzd4otc4rSkFBjxgOL9Y,3632
78
+ hindsight_api/engine/retain/fact_extraction.py,sha256=LdrXyoDERRWJhofHHCVlLrTi880RRIIeAk1AgZiDBAw,63187
79
+ hindsight_api/engine/retain/fact_storage.py,sha256=PUdMfNWaGuDA-DodeT3hs8ft81ldzXZedCMXys-sFf4,6690
80
+ hindsight_api/engine/retain/link_creation.py,sha256=KP2kGU2VCymJptgw0hjaSdsjvncBgNp3P_A4OB_qx-w,3082
81
+ hindsight_api/engine/retain/link_utils.py,sha256=eKa9Ecf7Mpqjl4laAEtRilQgu4fbsGWAjg98kdMDsDc,33078
82
+ hindsight_api/engine/retain/orchestrator.py,sha256=URQm9oXFWhLTmQjHlolnyWjcFDusEitn5UVbIvVdcXQ,20480
83
+ hindsight_api/engine/retain/types.py,sha256=zNkjqUA6oUAFe9a5SEbZfQC5PSmpYqTyBfgdmyqPpnw,7722
84
+ hindsight_api/engine/search/__init__.py,sha256=YPz_4g7IOabx078Xwg3RBfbOpJ649NRwNfe0gTI9P1U,802
85
+ hindsight_api/engine/search/fusion.py,sha256=cY81BH9U5RyWrPXbQnrDBghtelDMckZWCke9aqMyNnQ,4220
86
+ hindsight_api/engine/search/graph_retrieval.py,sha256=FCxyxXHv1FU6JlwEACcP3gL4F9u7RN3JDR7c3IzA0Sc,10161
87
+ hindsight_api/engine/search/link_expansion_retrieval.py,sha256=SkBU2bLqh699TwZoWhnmi6HI5blzfF5DAdYUTkAVj-8,16497
88
+ hindsight_api/engine/search/mpfp_retrieval.py,sha256=YJvQBPpBITVq3TcO9S8AFe9i6W4P2AtsZxEplRNtWxg,24461
89
+ hindsight_api/engine/search/reranking.py,sha256=hNwte352lTB8A7wlez8-05cdL2_Ghy2kbTs93sGyug4,3929
90
+ hindsight_api/engine/search/retrieval.py,sha256=kXUnVLZLisRYBrJc1rmD2UjtvPfZsh7i5hDAgvhHnD0,51572
91
+ hindsight_api/engine/search/tags.py,sha256=3oxpm3VonwvowyOXn1FPVby50PakVfxvTT1FuEI_iDo,5843
92
+ hindsight_api/engine/search/temporal_extraction.py,sha256=j7hPqpx2jMdR2BqgFrL-rrV2Hzq8HV24MtjYLJqVl2U,1732
93
+ hindsight_api/engine/search/think_utils.py,sha256=k2NBmb1eczTiDHuQZ7-VW4lsvlGt20VXz-pYLJzvN6k,9642
94
+ hindsight_api/engine/search/trace.py,sha256=RjvbkKBK-_MZBcVhVlaDTLD0yg6krngMNyC0_zLK05Y,11748
95
+ hindsight_api/engine/search/tracer.py,sha256=B75CZQjdoheN2UpNgqKbJkdXlDVKJjzVTdUhvBUFaLY,16212
96
+ hindsight_api/engine/search/types.py,sha256=meIoT8Q1coal1TmV_UiCqo9emjQI6af27EXPWVZL4h4,6418
97
+ hindsight_api/extensions/__init__.py,sha256=F8q_tH-2Hl8-4F8wzIieA4Cya2dMEE9gwPtz7Z56BYc,2151
98
+ hindsight_api/extensions/base.py,sha256=M7zXuM-tbqDnUwXX1mxAxiFs1eXOzNqIJutKLiUE4mU,2357
99
+ hindsight_api/extensions/context.py,sha256=Qq-uy3hhxO6ioDmf6dPXdnIjs_pdm7lTspDiEhJJmPU,4469
100
+ hindsight_api/extensions/http.py,sha256=c-a1g6R6rzibyReyR-WHz8DjRRGr4rVSyV9KB4UxVVU,2907
101
+ hindsight_api/extensions/loader.py,sha256=UwGM0XH7zHGng_xfHUY0VbOQemj9DmjuDaMst1TrFi8,4170
102
+ hindsight_api/extensions/operation_validator.py,sha256=ciXvTtlX4c5VcLze5cVbuaD6B-10IxnfgnNhbY8LGLc,13360
103
+ hindsight_api/extensions/tenant.py,sha256=0LraksQ1gzsOYLEGrx2q2F0or596Ywfo_MqD1FJMNRM,2617
104
+ hindsight_api/extensions/builtin/__init__.py,sha256=hLx2oFYZ1JtZhTWfab6AYcR02SWP2gIdbEqnZezT8ek,526
105
+ hindsight_api/extensions/builtin/tenant.py,sha256=fRYKxueoVbEA7zRPT7DNJpzcrMsNSSYq7qiA0tGvn30,1604
106
+ hindsight_api/worker/__init__.py,sha256=hzpMLvOfgL2KKrrik_9ouvEzCdvJSrH-pj5UdFK63J0,256
107
+ hindsight_api/worker/main.py,sha256=1OrQdHL-6u-311W0XMAoLHOXCu8MOETiQkR0TQ23qh8,9547
108
+ hindsight_api/worker/poller.py,sha256=l-y8xpekKZ7zcGo83osOsbFd_tBi49LqrAJsN-mxiMY,19306
109
+ hindsight_api-0.4.0.dist-info/METADATA,sha256=yuh2jSpF0sVTy81tN4k4MBVnp2k5cUf4RWCFNp5z25g,5760
110
+ hindsight_api-0.4.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
111
+ hindsight_api-0.4.0.dist-info/entry_points.txt,sha256=1-mxPbRGL_Byf9ZrHYkPW-TEgLYFcwCiSFCxOgI_3vM,206
112
+ hindsight_api-0.4.0.dist-info/RECORD,,
@@ -2,3 +2,4 @@
2
2
  hindsight-admin = hindsight_api.admin.cli:main
3
3
  hindsight-api = hindsight_api.main:main
4
4
  hindsight-local-mcp = hindsight_api.mcp_local:main
5
+ hindsight-worker = hindsight_api.worker.main:main