hindsight-api 0.1.10__py3-none-any.whl → 0.1.12__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- hindsight_api/__init__.py +2 -0
- hindsight_api/alembic/env.py +24 -1
- hindsight_api/alembic/versions/d9f6a3b4c5e2_rename_bank_to_interactions.py +14 -4
- hindsight_api/alembic/versions/e0a1b2c3d4e5_disposition_to_3_traits.py +54 -13
- hindsight_api/alembic/versions/rename_personality_to_disposition.py +18 -7
- hindsight_api/api/http.py +234 -228
- hindsight_api/api/mcp.py +14 -3
- hindsight_api/engine/__init__.py +12 -1
- hindsight_api/engine/entity_resolver.py +38 -37
- hindsight_api/engine/interface.py +592 -0
- hindsight_api/engine/llm_wrapper.py +176 -6
- hindsight_api/engine/memory_engine.py +993 -217
- hindsight_api/engine/retain/bank_utils.py +13 -12
- hindsight_api/engine/retain/chunk_storage.py +3 -2
- hindsight_api/engine/retain/fact_storage.py +10 -7
- hindsight_api/engine/retain/link_utils.py +17 -16
- hindsight_api/engine/retain/observation_regeneration.py +17 -16
- hindsight_api/engine/retain/orchestrator.py +2 -3
- hindsight_api/engine/retain/types.py +25 -8
- hindsight_api/engine/search/graph_retrieval.py +6 -5
- hindsight_api/engine/search/mpfp_retrieval.py +8 -7
- hindsight_api/engine/search/retrieval.py +12 -11
- hindsight_api/engine/search/think_utils.py +1 -1
- hindsight_api/engine/search/tracer.py +1 -1
- hindsight_api/engine/task_backend.py +32 -0
- hindsight_api/extensions/__init__.py +66 -0
- hindsight_api/extensions/base.py +81 -0
- hindsight_api/extensions/builtin/__init__.py +18 -0
- hindsight_api/extensions/builtin/tenant.py +33 -0
- hindsight_api/extensions/context.py +110 -0
- hindsight_api/extensions/http.py +89 -0
- hindsight_api/extensions/loader.py +125 -0
- hindsight_api/extensions/operation_validator.py +325 -0
- hindsight_api/extensions/tenant.py +63 -0
- hindsight_api/main.py +1 -1
- hindsight_api/mcp_local.py +7 -1
- hindsight_api/migrations.py +54 -10
- hindsight_api/models.py +15 -0
- hindsight_api/pg0.py +1 -1
- {hindsight_api-0.1.10.dist-info → hindsight_api-0.1.12.dist-info}/METADATA +1 -1
- hindsight_api-0.1.12.dist-info/RECORD +74 -0
- hindsight_api-0.1.10.dist-info/RECORD +0 -64
- {hindsight_api-0.1.10.dist-info → hindsight_api-0.1.12.dist-info}/WHEEL +0 -0
- {hindsight_api-0.1.10.dist-info → hindsight_api-0.1.12.dist-info}/entry_points.txt +0 -0
|
@@ -0,0 +1,325 @@
|
|
|
1
|
+
"""Operation Validator Extension for validating retain/recall/reflect operations."""
|
|
2
|
+
|
|
3
|
+
from abc import ABC, abstractmethod
|
|
4
|
+
from dataclasses import dataclass, field
|
|
5
|
+
from datetime import datetime
|
|
6
|
+
from typing import TYPE_CHECKING, Any
|
|
7
|
+
|
|
8
|
+
from hindsight_api.extensions.base import Extension
|
|
9
|
+
|
|
10
|
+
if TYPE_CHECKING:
|
|
11
|
+
from hindsight_api.engine.memory_engine import Budget
|
|
12
|
+
from hindsight_api.engine.response_models import RecallResult as RecallResultModel
|
|
13
|
+
from hindsight_api.engine.response_models import ReflectResult
|
|
14
|
+
from hindsight_api.models import RequestContext
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class OperationValidationError(Exception):
|
|
18
|
+
"""Raised when an operation fails validation."""
|
|
19
|
+
|
|
20
|
+
def __init__(self, reason: str):
|
|
21
|
+
self.reason = reason
|
|
22
|
+
super().__init__(f"Operation validation failed: {reason}")
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@dataclass
|
|
26
|
+
class ValidationResult:
|
|
27
|
+
"""Result of an operation validation."""
|
|
28
|
+
|
|
29
|
+
allowed: bool
|
|
30
|
+
reason: str | None = None
|
|
31
|
+
|
|
32
|
+
@classmethod
|
|
33
|
+
def accept(cls) -> "ValidationResult":
|
|
34
|
+
"""Create an accepted validation result."""
|
|
35
|
+
return cls(allowed=True)
|
|
36
|
+
|
|
37
|
+
@classmethod
|
|
38
|
+
def reject(cls, reason: str) -> "ValidationResult":
|
|
39
|
+
"""Create a rejected validation result with a reason."""
|
|
40
|
+
return cls(allowed=False, reason=reason)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
# =============================================================================
|
|
44
|
+
# Pre-operation Contexts (all user-provided parameters)
|
|
45
|
+
# =============================================================================
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
@dataclass
|
|
49
|
+
class RetainContext:
|
|
50
|
+
"""Context for a retain operation validation (pre-operation).
|
|
51
|
+
|
|
52
|
+
Contains ALL user-provided parameters for the retain operation.
|
|
53
|
+
"""
|
|
54
|
+
|
|
55
|
+
bank_id: str
|
|
56
|
+
contents: list[dict] # List of {content, context, event_date, document_id}
|
|
57
|
+
request_context: "RequestContext"
|
|
58
|
+
document_id: str | None = None
|
|
59
|
+
fact_type_override: str | None = None
|
|
60
|
+
confidence_score: float | None = None
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
@dataclass
|
|
64
|
+
class RecallContext:
|
|
65
|
+
"""Context for a recall operation validation (pre-operation).
|
|
66
|
+
|
|
67
|
+
Contains ALL user-provided parameters for the recall operation.
|
|
68
|
+
"""
|
|
69
|
+
|
|
70
|
+
bank_id: str
|
|
71
|
+
query: str
|
|
72
|
+
request_context: "RequestContext"
|
|
73
|
+
budget: "Budget | None" = None
|
|
74
|
+
max_tokens: int = 4096
|
|
75
|
+
enable_trace: bool = False
|
|
76
|
+
fact_types: list[str] = field(default_factory=list)
|
|
77
|
+
question_date: datetime | None = None
|
|
78
|
+
include_entities: bool = False
|
|
79
|
+
max_entity_tokens: int = 500
|
|
80
|
+
include_chunks: bool = False
|
|
81
|
+
max_chunk_tokens: int = 8192
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
@dataclass
|
|
85
|
+
class ReflectContext:
|
|
86
|
+
"""Context for a reflect operation validation (pre-operation).
|
|
87
|
+
|
|
88
|
+
Contains ALL user-provided parameters for the reflect operation.
|
|
89
|
+
"""
|
|
90
|
+
|
|
91
|
+
bank_id: str
|
|
92
|
+
query: str
|
|
93
|
+
request_context: "RequestContext"
|
|
94
|
+
budget: "Budget | None" = None
|
|
95
|
+
context: str | None = None
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
# =============================================================================
|
|
99
|
+
# Post-operation Contexts (includes results)
|
|
100
|
+
# =============================================================================
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
@dataclass
|
|
104
|
+
class RetainResult:
|
|
105
|
+
"""Result context for post-retain hook.
|
|
106
|
+
|
|
107
|
+
Contains the operation parameters and the result.
|
|
108
|
+
"""
|
|
109
|
+
|
|
110
|
+
bank_id: str
|
|
111
|
+
contents: list[dict]
|
|
112
|
+
request_context: "RequestContext"
|
|
113
|
+
document_id: str | None
|
|
114
|
+
fact_type_override: str | None
|
|
115
|
+
confidence_score: float | None
|
|
116
|
+
# Result
|
|
117
|
+
unit_ids: list[list[str]] # List of unit IDs per content item
|
|
118
|
+
success: bool = True
|
|
119
|
+
error: str | None = None
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
@dataclass
|
|
123
|
+
class RecallResult:
|
|
124
|
+
"""Result context for post-recall hook.
|
|
125
|
+
|
|
126
|
+
Contains the operation parameters and the result.
|
|
127
|
+
"""
|
|
128
|
+
|
|
129
|
+
bank_id: str
|
|
130
|
+
query: str
|
|
131
|
+
request_context: "RequestContext"
|
|
132
|
+
budget: "Budget | None"
|
|
133
|
+
max_tokens: int
|
|
134
|
+
enable_trace: bool
|
|
135
|
+
fact_types: list[str]
|
|
136
|
+
question_date: datetime | None
|
|
137
|
+
include_entities: bool
|
|
138
|
+
max_entity_tokens: int
|
|
139
|
+
include_chunks: bool
|
|
140
|
+
max_chunk_tokens: int
|
|
141
|
+
# Result
|
|
142
|
+
result: "RecallResultModel | None" = None
|
|
143
|
+
success: bool = True
|
|
144
|
+
error: str | None = None
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
@dataclass
|
|
148
|
+
class ReflectResultContext:
|
|
149
|
+
"""Result context for post-reflect hook.
|
|
150
|
+
|
|
151
|
+
Contains the operation parameters and the result.
|
|
152
|
+
"""
|
|
153
|
+
|
|
154
|
+
bank_id: str
|
|
155
|
+
query: str
|
|
156
|
+
request_context: "RequestContext"
|
|
157
|
+
budget: "Budget | None"
|
|
158
|
+
context: str | None
|
|
159
|
+
# Result
|
|
160
|
+
result: "ReflectResult | None" = None
|
|
161
|
+
success: bool = True
|
|
162
|
+
error: str | None = None
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
class OperationValidatorExtension(Extension, ABC):
|
|
166
|
+
"""
|
|
167
|
+
Validates and hooks into retain/recall/reflect operations.
|
|
168
|
+
|
|
169
|
+
This extension allows implementing custom logic such as:
|
|
170
|
+
- Rate limiting (pre-operation)
|
|
171
|
+
- Quota enforcement (pre-operation)
|
|
172
|
+
- Permission checks (pre-operation)
|
|
173
|
+
- Content filtering (pre-operation)
|
|
174
|
+
- Usage tracking (post-operation)
|
|
175
|
+
- Audit logging (post-operation)
|
|
176
|
+
- Metrics collection (post-operation)
|
|
177
|
+
|
|
178
|
+
Enable via environment variable:
|
|
179
|
+
HINDSIGHT_API_OPERATION_VALIDATOR_EXTENSION=mypackage.validators:MyValidator
|
|
180
|
+
|
|
181
|
+
Configuration is passed from prefixed environment variables:
|
|
182
|
+
HINDSIGHT_API_OPERATION_VALIDATOR_MAX_REQUESTS=100
|
|
183
|
+
-> config = {"max_requests": "100"}
|
|
184
|
+
|
|
185
|
+
Hook execution order:
|
|
186
|
+
1. validate_retain/validate_recall/validate_reflect (pre-operation)
|
|
187
|
+
2. [operation executes]
|
|
188
|
+
3. on_retain_complete/on_recall_complete/on_reflect_complete (post-operation)
|
|
189
|
+
"""
|
|
190
|
+
|
|
191
|
+
# =========================================================================
|
|
192
|
+
# Pre-operation validation hooks (abstract - must be implemented)
|
|
193
|
+
# =========================================================================
|
|
194
|
+
|
|
195
|
+
@abstractmethod
|
|
196
|
+
async def validate_retain(self, ctx: RetainContext) -> ValidationResult:
|
|
197
|
+
"""
|
|
198
|
+
Validate a retain operation before execution.
|
|
199
|
+
|
|
200
|
+
Called before the retain operation is processed. Return ValidationResult.reject()
|
|
201
|
+
to prevent the operation from executing.
|
|
202
|
+
|
|
203
|
+
Args:
|
|
204
|
+
ctx: Context containing all user-provided parameters:
|
|
205
|
+
- bank_id: Bank identifier
|
|
206
|
+
- contents: List of content dicts
|
|
207
|
+
- request_context: Request context with auth info
|
|
208
|
+
- document_id: Optional document ID
|
|
209
|
+
- fact_type_override: Optional fact type override
|
|
210
|
+
- confidence_score: Optional confidence score
|
|
211
|
+
|
|
212
|
+
Returns:
|
|
213
|
+
ValidationResult indicating whether the operation is allowed.
|
|
214
|
+
"""
|
|
215
|
+
...
|
|
216
|
+
|
|
217
|
+
@abstractmethod
|
|
218
|
+
async def validate_recall(self, ctx: RecallContext) -> ValidationResult:
|
|
219
|
+
"""
|
|
220
|
+
Validate a recall operation before execution.
|
|
221
|
+
|
|
222
|
+
Called before the recall operation is processed. Return ValidationResult.reject()
|
|
223
|
+
to prevent the operation from executing.
|
|
224
|
+
|
|
225
|
+
Args:
|
|
226
|
+
ctx: Context containing all user-provided parameters:
|
|
227
|
+
- bank_id: Bank identifier
|
|
228
|
+
- query: Search query
|
|
229
|
+
- request_context: Request context with auth info
|
|
230
|
+
- budget: Budget level
|
|
231
|
+
- max_tokens: Maximum tokens to return
|
|
232
|
+
- enable_trace: Whether to include trace info
|
|
233
|
+
- fact_types: List of fact types to search
|
|
234
|
+
- question_date: Optional date context for query
|
|
235
|
+
- include_entities: Whether to include entity data
|
|
236
|
+
- max_entity_tokens: Max tokens for entities
|
|
237
|
+
- include_chunks: Whether to include chunks
|
|
238
|
+
- max_chunk_tokens: Max tokens for chunks
|
|
239
|
+
|
|
240
|
+
Returns:
|
|
241
|
+
ValidationResult indicating whether the operation is allowed.
|
|
242
|
+
"""
|
|
243
|
+
...
|
|
244
|
+
|
|
245
|
+
@abstractmethod
|
|
246
|
+
async def validate_reflect(self, ctx: ReflectContext) -> ValidationResult:
|
|
247
|
+
"""
|
|
248
|
+
Validate a reflect operation before execution.
|
|
249
|
+
|
|
250
|
+
Called before the reflect operation is processed. Return ValidationResult.reject()
|
|
251
|
+
to prevent the operation from executing.
|
|
252
|
+
|
|
253
|
+
Args:
|
|
254
|
+
ctx: Context containing all user-provided parameters:
|
|
255
|
+
- bank_id: Bank identifier
|
|
256
|
+
- query: Question to answer
|
|
257
|
+
- request_context: Request context with auth info
|
|
258
|
+
- budget: Budget level
|
|
259
|
+
- context: Optional additional context
|
|
260
|
+
|
|
261
|
+
Returns:
|
|
262
|
+
ValidationResult indicating whether the operation is allowed.
|
|
263
|
+
"""
|
|
264
|
+
...
|
|
265
|
+
|
|
266
|
+
# =========================================================================
|
|
267
|
+
# Post-operation hooks (optional - override to implement)
|
|
268
|
+
# =========================================================================
|
|
269
|
+
|
|
270
|
+
async def on_retain_complete(self, result: RetainResult) -> None:
|
|
271
|
+
"""
|
|
272
|
+
Called after a retain operation completes (success or failure).
|
|
273
|
+
|
|
274
|
+
Override this method to implement post-operation logic such as:
|
|
275
|
+
- Usage tracking
|
|
276
|
+
- Audit logging
|
|
277
|
+
- Metrics collection
|
|
278
|
+
- Notifications
|
|
279
|
+
|
|
280
|
+
Args:
|
|
281
|
+
result: Result context containing:
|
|
282
|
+
- All original operation parameters
|
|
283
|
+
- unit_ids: List of created unit IDs (if success)
|
|
284
|
+
- success: Whether the operation succeeded
|
|
285
|
+
- error: Error message (if failed)
|
|
286
|
+
"""
|
|
287
|
+
pass
|
|
288
|
+
|
|
289
|
+
async def on_recall_complete(self, result: RecallResult) -> None:
|
|
290
|
+
"""
|
|
291
|
+
Called after a recall operation completes (success or failure).
|
|
292
|
+
|
|
293
|
+
Override this method to implement post-operation logic such as:
|
|
294
|
+
- Usage tracking
|
|
295
|
+
- Audit logging
|
|
296
|
+
- Metrics collection
|
|
297
|
+
- Query analytics
|
|
298
|
+
|
|
299
|
+
Args:
|
|
300
|
+
result: Result context containing:
|
|
301
|
+
- All original operation parameters
|
|
302
|
+
- result: RecallResultModel (if success)
|
|
303
|
+
- success: Whether the operation succeeded
|
|
304
|
+
- error: Error message (if failed)
|
|
305
|
+
"""
|
|
306
|
+
pass
|
|
307
|
+
|
|
308
|
+
async def on_reflect_complete(self, result: ReflectResultContext) -> None:
|
|
309
|
+
"""
|
|
310
|
+
Called after a reflect operation completes (success or failure).
|
|
311
|
+
|
|
312
|
+
Override this method to implement post-operation logic such as:
|
|
313
|
+
- Usage tracking
|
|
314
|
+
- Audit logging
|
|
315
|
+
- Metrics collection
|
|
316
|
+
- Response analytics
|
|
317
|
+
|
|
318
|
+
Args:
|
|
319
|
+
result: Result context containing:
|
|
320
|
+
- All original operation parameters
|
|
321
|
+
- result: ReflectResult (if success)
|
|
322
|
+
- success: Whether the operation succeeded
|
|
323
|
+
- error: Error message (if failed)
|
|
324
|
+
"""
|
|
325
|
+
pass
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
"""Tenant Extension for multi-tenancy and API key authentication."""
|
|
2
|
+
|
|
3
|
+
from abc import ABC, abstractmethod
|
|
4
|
+
from dataclasses import dataclass
|
|
5
|
+
|
|
6
|
+
from hindsight_api.extensions.base import Extension
|
|
7
|
+
from hindsight_api.models import RequestContext
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class AuthenticationError(Exception):
|
|
11
|
+
"""Raised when authentication fails."""
|
|
12
|
+
|
|
13
|
+
def __init__(self, reason: str):
|
|
14
|
+
self.reason = reason
|
|
15
|
+
super().__init__(f"Authentication failed: {reason}")
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@dataclass
|
|
19
|
+
class TenantContext:
|
|
20
|
+
"""
|
|
21
|
+
Tenant context returned by authentication.
|
|
22
|
+
|
|
23
|
+
Contains the PostgreSQL schema name for tenant isolation.
|
|
24
|
+
All database queries will use fully-qualified table names
|
|
25
|
+
with this schema (e.g., schema_name.memory_units).
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
schema_name: str
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class TenantExtension(Extension, ABC):
|
|
32
|
+
"""
|
|
33
|
+
Extension for multi-tenancy and API key authentication.
|
|
34
|
+
|
|
35
|
+
This extension validates incoming requests and returns the tenant context
|
|
36
|
+
including the PostgreSQL schema to use for database operations.
|
|
37
|
+
|
|
38
|
+
Built-in implementation:
|
|
39
|
+
hindsight_api.extensions.builtin.tenant.ApiKeyTenantExtension
|
|
40
|
+
|
|
41
|
+
Enable via environment variable:
|
|
42
|
+
HINDSIGHT_API_TENANT_EXTENSION=hindsight_api.extensions.builtin.tenant:ApiKeyTenantExtension
|
|
43
|
+
HINDSIGHT_API_TENANT_API_KEY=your-secret-key
|
|
44
|
+
|
|
45
|
+
The returned schema_name is used for fully-qualified table names in queries,
|
|
46
|
+
enabling tenant isolation at the database level.
|
|
47
|
+
"""
|
|
48
|
+
|
|
49
|
+
@abstractmethod
|
|
50
|
+
async def authenticate(self, context: RequestContext) -> TenantContext:
|
|
51
|
+
"""
|
|
52
|
+
Authenticate the action context and return tenant context.
|
|
53
|
+
|
|
54
|
+
Args:
|
|
55
|
+
context: The action context containing API key and other auth data.
|
|
56
|
+
|
|
57
|
+
Returns:
|
|
58
|
+
TenantContext with the schema_name for database operations.
|
|
59
|
+
|
|
60
|
+
Raises:
|
|
61
|
+
AuthenticationError: If authentication fails.
|
|
62
|
+
"""
|
|
63
|
+
...
|
hindsight_api/main.py
CHANGED
hindsight_api/mcp_local.py
CHANGED
|
@@ -87,6 +87,7 @@ def create_local_mcp_server(bank_id: str, memory=None) -> FastMCP:
|
|
|
87
87
|
from hindsight_api import MemoryEngine
|
|
88
88
|
from hindsight_api.engine.memory_engine import Budget
|
|
89
89
|
from hindsight_api.engine.response_models import VALID_RECALL_FACT_TYPES
|
|
90
|
+
from hindsight_api.models import RequestContext
|
|
90
91
|
|
|
91
92
|
# Create memory engine with pg0 embedded database if not provided
|
|
92
93
|
if memory is None:
|
|
@@ -115,7 +116,11 @@ def create_local_mcp_server(bank_id: str, memory=None) -> FastMCP:
|
|
|
115
116
|
|
|
116
117
|
async def _retain():
|
|
117
118
|
try:
|
|
118
|
-
await memory.retain_batch_async(
|
|
119
|
+
await memory.retain_batch_async(
|
|
120
|
+
bank_id=bank_id,
|
|
121
|
+
contents=[{"content": content, "context": context}],
|
|
122
|
+
request_context=RequestContext(),
|
|
123
|
+
)
|
|
119
124
|
except Exception as e:
|
|
120
125
|
logger.error(f"Error storing memory: {e}", exc_info=True)
|
|
121
126
|
|
|
@@ -142,6 +147,7 @@ def create_local_mcp_server(bank_id: str, memory=None) -> FastMCP:
|
|
|
142
147
|
fact_type=list(VALID_RECALL_FACT_TYPES),
|
|
143
148
|
budget=budget_enum,
|
|
144
149
|
max_tokens=max_tokens,
|
|
150
|
+
request_context=RequestContext(),
|
|
145
151
|
)
|
|
146
152
|
|
|
147
153
|
return search_result.model_dump()
|
hindsight_api/migrations.py
CHANGED
|
@@ -6,12 +6,16 @@ on application startup. It is designed to be safe for concurrent
|
|
|
6
6
|
execution using PostgreSQL advisory locks to coordinate between
|
|
7
7
|
distributed workers.
|
|
8
8
|
|
|
9
|
+
Supports multi-tenant schema isolation: migrations can target a specific
|
|
10
|
+
PostgreSQL schema, allowing each tenant to have isolated tables.
|
|
11
|
+
|
|
9
12
|
Important: All migrations must be backward-compatible to allow
|
|
10
13
|
safe rolling deployments.
|
|
11
14
|
|
|
12
15
|
No alembic.ini required - all configuration is done programmatically.
|
|
13
16
|
"""
|
|
14
17
|
|
|
18
|
+
import hashlib
|
|
15
19
|
import logging
|
|
16
20
|
import os
|
|
17
21
|
from pathlib import Path
|
|
@@ -26,11 +30,29 @@ logger = logging.getLogger(__name__)
|
|
|
26
30
|
MIGRATION_LOCK_ID = 123456789
|
|
27
31
|
|
|
28
32
|
|
|
29
|
-
def
|
|
33
|
+
def _get_schema_lock_id(schema: str) -> int:
|
|
34
|
+
"""
|
|
35
|
+
Generate a unique advisory lock ID for a schema.
|
|
36
|
+
|
|
37
|
+
Uses hash of schema name to create a deterministic lock ID.
|
|
38
|
+
"""
|
|
39
|
+
# Use hash to create a unique lock ID per schema
|
|
40
|
+
# Keep within PostgreSQL's bigint range
|
|
41
|
+
hash_bytes = hashlib.sha256(schema.encode()).digest()[:8]
|
|
42
|
+
return int.from_bytes(hash_bytes, byteorder="big") % (2**31)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _run_migrations_internal(database_url: str, script_location: str, schema: str | None = None) -> None:
|
|
30
46
|
"""
|
|
31
47
|
Internal function to run migrations without locking.
|
|
48
|
+
|
|
49
|
+
Args:
|
|
50
|
+
database_url: SQLAlchemy database URL
|
|
51
|
+
script_location: Path to alembic scripts
|
|
52
|
+
schema: Target schema (None for default/public)
|
|
32
53
|
"""
|
|
33
|
-
|
|
54
|
+
schema_name = schema or "public"
|
|
55
|
+
logger.info(f"Running database migrations to head for schema '{schema_name}'...")
|
|
34
56
|
logger.info(f"Database URL: {database_url}")
|
|
35
57
|
logger.info(f"Script location: {script_location}")
|
|
36
58
|
|
|
@@ -50,13 +72,22 @@ def _run_migrations_internal(database_url: str, script_location: str) -> None:
|
|
|
50
72
|
# Set path_separator to avoid deprecation warning
|
|
51
73
|
alembic_cfg.set_main_option("path_separator", "os")
|
|
52
74
|
|
|
53
|
-
#
|
|
75
|
+
# If targeting a specific schema, pass it to env.py via config
|
|
76
|
+
# env.py will handle setting search_path and version_table_schema
|
|
77
|
+
if schema:
|
|
78
|
+
alembic_cfg.set_main_option("target_schema", schema)
|
|
79
|
+
|
|
80
|
+
# Run migrations
|
|
54
81
|
command.upgrade(alembic_cfg, "head")
|
|
55
82
|
|
|
56
|
-
logger.info("Database migrations completed successfully")
|
|
83
|
+
logger.info(f"Database migrations completed successfully for schema '{schema_name}'")
|
|
57
84
|
|
|
58
85
|
|
|
59
|
-
def run_migrations(
|
|
86
|
+
def run_migrations(
|
|
87
|
+
database_url: str,
|
|
88
|
+
script_location: str | None = None,
|
|
89
|
+
schema: str | None = None,
|
|
90
|
+
) -> None:
|
|
60
91
|
"""
|
|
61
92
|
Run database migrations to the latest version using programmatic Alembic configuration.
|
|
62
93
|
|
|
@@ -65,19 +96,28 @@ def run_migrations(database_url: str, script_location: str | None = None) -> Non
|
|
|
65
96
|
- Other workers wait for the lock, then verify migrations are complete
|
|
66
97
|
- If schema is already up-to-date, this is a fast no-op
|
|
67
98
|
|
|
99
|
+
Supports multi-tenant schema isolation: when a schema is specified, migrations
|
|
100
|
+
run in that schema instead of public. This allows tenant extensions to provision
|
|
101
|
+
new tenant schemas with their own isolated tables.
|
|
102
|
+
|
|
68
103
|
Args:
|
|
69
104
|
database_url: SQLAlchemy database URL (e.g., "postgresql://user:pass@host/db")
|
|
70
105
|
script_location: Path to alembic migrations directory (e.g., "/path/to/alembic").
|
|
71
106
|
If None, defaults to hindsight-api/alembic directory.
|
|
107
|
+
schema: Target PostgreSQL schema name. If None, uses default (public).
|
|
108
|
+
When specified, creates the schema if needed and runs migrations there.
|
|
72
109
|
|
|
73
110
|
Raises:
|
|
74
111
|
RuntimeError: If migrations fail to complete
|
|
75
112
|
FileNotFoundError: If script_location doesn't exist
|
|
76
113
|
|
|
77
114
|
Example:
|
|
78
|
-
# Using default location
|
|
115
|
+
# Using default location and public schema
|
|
79
116
|
run_migrations("postgresql://user:pass@host/db")
|
|
80
117
|
|
|
118
|
+
# Run migrations for a specific tenant schema
|
|
119
|
+
run_migrations("postgresql://user:pass@host/db", schema="tenant_acme")
|
|
120
|
+
|
|
81
121
|
# Using custom location (when importing from another project)
|
|
82
122
|
run_migrations(
|
|
83
123
|
"postgresql://user:pass@host/db",
|
|
@@ -99,21 +139,25 @@ def run_migrations(database_url: str, script_location: str | None = None) -> Non
|
|
|
99
139
|
f"Alembic script location not found at {script_location}. Database migrations cannot be run."
|
|
100
140
|
)
|
|
101
141
|
|
|
142
|
+
# Use schema-specific lock ID for multi-tenant isolation
|
|
143
|
+
lock_id = _get_schema_lock_id(schema) if schema else MIGRATION_LOCK_ID
|
|
144
|
+
schema_name = schema or "public"
|
|
145
|
+
|
|
102
146
|
# Use PostgreSQL advisory lock to coordinate between distributed workers
|
|
103
147
|
engine = create_engine(database_url)
|
|
104
148
|
with engine.connect() as conn:
|
|
105
149
|
# pg_advisory_lock blocks until the lock is acquired
|
|
106
150
|
# The lock is automatically released when the connection closes
|
|
107
|
-
logger.debug(f"Acquiring migration advisory lock (id={
|
|
108
|
-
conn.execute(text(f"SELECT pg_advisory_lock({
|
|
151
|
+
logger.debug(f"Acquiring migration advisory lock for schema '{schema_name}' (id={lock_id})...")
|
|
152
|
+
conn.execute(text(f"SELECT pg_advisory_lock({lock_id})"))
|
|
109
153
|
logger.debug("Migration advisory lock acquired")
|
|
110
154
|
|
|
111
155
|
try:
|
|
112
156
|
# Run migrations while holding the lock
|
|
113
|
-
_run_migrations_internal(database_url, script_location)
|
|
157
|
+
_run_migrations_internal(database_url, script_location, schema=schema)
|
|
114
158
|
finally:
|
|
115
159
|
# Explicitly release the lock (also released on connection close)
|
|
116
|
-
conn.execute(text(f"SELECT pg_advisory_unlock({
|
|
160
|
+
conn.execute(text(f"SELECT pg_advisory_unlock({lock_id})"))
|
|
117
161
|
logger.debug("Migration advisory lock released")
|
|
118
162
|
|
|
119
163
|
except FileNotFoundError:
|
hindsight_api/models.py
CHANGED
|
@@ -2,9 +2,24 @@
|
|
|
2
2
|
SQLAlchemy models for the memory system.
|
|
3
3
|
"""
|
|
4
4
|
|
|
5
|
+
from dataclasses import dataclass
|
|
5
6
|
from datetime import datetime
|
|
6
7
|
from uuid import UUID as PyUUID
|
|
7
8
|
|
|
9
|
+
|
|
10
|
+
@dataclass
|
|
11
|
+
class RequestContext:
|
|
12
|
+
"""
|
|
13
|
+
Context for request authentication and authorization.
|
|
14
|
+
|
|
15
|
+
This dataclass carries authentication data from HTTP requests to the
|
|
16
|
+
memory engine operations. It can be extended to include additional
|
|
17
|
+
context like headers, tokens, user info, etc.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
api_key: str | None = None
|
|
21
|
+
|
|
22
|
+
|
|
8
23
|
from pgvector.sqlalchemy import Vector
|
|
9
24
|
from sqlalchemy import (
|
|
10
25
|
CheckConstraint,
|
hindsight_api/pg0.py
CHANGED
|
@@ -40,7 +40,7 @@ class EmbeddedPostgres:
|
|
|
40
40
|
# Only set port if explicitly specified
|
|
41
41
|
if self.port is not None:
|
|
42
42
|
kwargs["port"] = self.port
|
|
43
|
-
self._pg0 = Pg0(**kwargs)
|
|
43
|
+
self._pg0 = Pg0(**kwargs) # type: ignore[invalid-argument-type] - dict kwargs
|
|
44
44
|
return self._pg0
|
|
45
45
|
|
|
46
46
|
async def start(self, max_retries: int = 5, retry_delay: float = 4.0) -> str:
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
hindsight_api/__init__.py,sha256=lPhgtKMvT8qjORFKWlhlq-LVdwesIu0gbUYNPZQEFiI,1197
|
|
2
|
+
hindsight_api/banner.py,sha256=BXn-jhkXe4xi-YV4JeuaVvjYhTMs96O43XoOMv4Cd28,4591
|
|
3
|
+
hindsight_api/config.py,sha256=rqK0tNUcT-ddX8XRpsGx6x1sHVGusLW3m5OEdQz9sLs,6484
|
|
4
|
+
hindsight_api/main.py,sha256=pFPNndGHjdDMUasU_s7coBcyWbW4z14p8BMop09pyhI,6099
|
|
5
|
+
hindsight_api/mcp_local.py,sha256=fL2hpwQSNExcjIwZn1E5vy5No6iZFmw78yRNXxJzri0,7371
|
|
6
|
+
hindsight_api/metrics.py,sha256=sQI5MhC2xj9ONZ6Hdjf6r6r3NbYYd3ExyVOn1Uky49A,7239
|
|
7
|
+
hindsight_api/migrations.py,sha256=X5jYkrDhbeFzXOUoPRvPzkGHQsjlZ7oz_P71UI82VT4,9104
|
|
8
|
+
hindsight_api/models.py,sha256=LvOpCfuDjnVH-dEzOSolCOZnkoPiOZP_J9HK82sD1_0,12700
|
|
9
|
+
hindsight_api/pg0.py,sha256=SEIwYq8xp0s0YbV3CIy_ioZ5-Bfe8_rxdeP0YasAeXk,4677
|
|
10
|
+
hindsight_api/server.py,sha256=OrSd0G-79U07EXFc838c1vzUL-1O6wuxTMqUmMINpGY,1247
|
|
11
|
+
hindsight_api/alembic/README,sha256=MVlc9TYmr57RbhXET6QxgyCcwWP7w-vLkEsirENqiIQ,38
|
|
12
|
+
hindsight_api/alembic/env.py,sha256=I4sGdtUo8xcXe95MyD36JQeMod_Bvp9JUkW64Ve4XSM,5808
|
|
13
|
+
hindsight_api/alembic/script.py.mako,sha256=04kgeBtNMa4cCnG8CfQcKt6P6rnloIfj8wy0u_DBydM,704
|
|
14
|
+
hindsight_api/alembic/versions/5a366d414dce_initial_schema.py,sha256=g3G7fV70Z10PZxwTrTmR34OAlEZjQTLJKr-Ol54JqrQ,17665
|
|
15
|
+
hindsight_api/alembic/versions/b7c4d8e9f1a2_add_chunks_table.py,sha256=MaHFU4JczUIFLeUMBTKIV3ocuclil55N9fPPim-HRfk,2599
|
|
16
|
+
hindsight_api/alembic/versions/c8e5f2a3b4d1_add_retain_params_to_documents.py,sha256=ChqkHANauZb4-nBt2uepoZN3q0vRzN6aRsWTGueULiA,1146
|
|
17
|
+
hindsight_api/alembic/versions/d9f6a3b4c5e2_rename_bank_to_interactions.py,sha256=s5_B2D0JdaxO7WM-vWC5Yt6hAtTsAUzJhFGLFSkfuQU,1808
|
|
18
|
+
hindsight_api/alembic/versions/e0a1b2c3d4e5_disposition_to_3_traits.py,sha256=IdDP6fgsYj5fCXAF0QT-3t_wcKJsnf7B0mh7qS-cf_w,3806
|
|
19
|
+
hindsight_api/alembic/versions/rename_personality_to_disposition.py,sha256=A29-nDJ2Re4u9jdp2sUw29It808j4h6BpcA4wDHJMJ8,2765
|
|
20
|
+
hindsight_api/api/__init__.py,sha256=zoDWA86ttx-UriC35UIgdPswIrau7GuMWTN63wYsUdM,2916
|
|
21
|
+
hindsight_api/api/http.py,sha256=X4bi1sSq_RPR582RiK5IqFzHBFtPzFUZJw3IZR-QSkg,71537
|
|
22
|
+
hindsight_api/api/mcp.py,sha256=Iowo3ourjWx7ZqLiCwF9nvjMAJpRceBprF5cgn5M6fs,7853
|
|
23
|
+
hindsight_api/engine/__init__.py,sha256=-BwaSwG9fTT_BBO0c_2MBkxG6-tGdclSzIqsgHw4cnw,1633
|
|
24
|
+
hindsight_api/engine/cross_encoder.py,sha256=5WmUx9yfJdIwZ0nA218O-mMKQJ7EKaPOtwhMiDbG8KQ,10483
|
|
25
|
+
hindsight_api/engine/db_utils.py,sha256=0T5tL2SZ49JQihfyZYlTDThIfocKzkr1OpxQpJzPCGE,2687
|
|
26
|
+
hindsight_api/engine/embeddings.py,sha256=IEdP5-p6oTJRRKV2JzUEojByJGShUEmkInCyA9wM8tg,10219
|
|
27
|
+
hindsight_api/engine/entity_resolver.py,sha256=f-fbUDKCrM9a5Sz10J0rW3jV7dib7BmpyGyassspKXg,23510
|
|
28
|
+
hindsight_api/engine/interface.py,sha256=F6BgnjloH7EgL9_D2NpPuabR_zR-h_iEJBQ0ERC2P58,16090
|
|
29
|
+
hindsight_api/engine/llm_wrapper.py,sha256=nLdVAk2xtkbwxLFMQNmEU-JmHucdtQoh3ph0BWX4sDc,29140
|
|
30
|
+
hindsight_api/engine/memory_engine.py,sha256=aKGkqE8FhppSn0fQVlqeVjmFyMHu59yX9v8LLy7zdYw,165582
|
|
31
|
+
hindsight_api/engine/query_analyzer.py,sha256=DKFxmyyVVc59zwKbbGx4D22UVp6TxmD7jAa7cg9FGSU,19641
|
|
32
|
+
hindsight_api/engine/response_models.py,sha256=QeESHC7oh84SYPDrR6FqHjiGBZnTAzo61IDB-qwVTSY,8737
|
|
33
|
+
hindsight_api/engine/task_backend.py,sha256=txtcMUzHW1MigDCW7XsVZc5zqvM9FbR_xF_c9BKokBk,8054
|
|
34
|
+
hindsight_api/engine/utils.py,sha256=TwuipFRvN0Pu196JLakzQ71E3GAwySc5q6pByC81Ak4,6991
|
|
35
|
+
hindsight_api/engine/retain/__init__.py,sha256=t6q3-_kf4iYTl9j2PVB6laqMSs6UuPeXBSYMW6HT1sA,1152
|
|
36
|
+
hindsight_api/engine/retain/bank_utils.py,sha256=JjrTE-bixHZKaUyl4uPQ6FV9O7hMOOEijXUnqXhOB5g,14097
|
|
37
|
+
hindsight_api/engine/retain/chunk_storage.py,sha256=zXAqbcFeYpjyWlOoi8zeK5G91zHpF75CUVF-6wsEJpU,2064
|
|
38
|
+
hindsight_api/engine/retain/deduplication.py,sha256=kqs7I7eIc_ppvgAF9GlzL6fSGuEEzrgw17-7NdyUDis,3099
|
|
39
|
+
hindsight_api/engine/retain/embedding_processing.py,sha256=R35oyKYIKjuqC-yZl5Ru56F8xRe0N6KW_9p5PZ9CBi0,1649
|
|
40
|
+
hindsight_api/engine/retain/embedding_utils.py,sha256=uulXIBiA7XNsj16K1VGawR3s5jV-hsAmvmoCi-IodpU,1565
|
|
41
|
+
hindsight_api/engine/retain/entity_processing.py,sha256=5EYzyH_JjbhYQ0zQ8gX6xs0wCH6vmxMYUe6_qVJdvQA,2547
|
|
42
|
+
hindsight_api/engine/retain/fact_extraction.py,sha256=E9AswSrqx3X74gj5-qstbm2wqPv4kUMddkdn5yExKvI,50166
|
|
43
|
+
hindsight_api/engine/retain/fact_storage.py,sha256=zhIiccW1D4wkgnZMFcbxDeMeHy5v4JGKfEPBIFNLch4,5632
|
|
44
|
+
hindsight_api/engine/retain/link_creation.py,sha256=KP2kGU2VCymJptgw0hjaSdsjvncBgNp3P_A4OB_qx-w,3082
|
|
45
|
+
hindsight_api/engine/retain/link_utils.py,sha256=w8n_pPzs_rd3EMkb7nv4k_qSZttAKDig93hSSjl-Xbc,32854
|
|
46
|
+
hindsight_api/engine/retain/observation_regeneration.py,sha256=qE1-iSyH0lh5Zab1XIwSQSpxEArdOJOAC_yJY5iHLMQ,8143
|
|
47
|
+
hindsight_api/engine/retain/orchestrator.py,sha256=TY_xk-DbqvXs1KCV41jj8u7ba6WvI2yVeMv_Xq9fBY8,17620
|
|
48
|
+
hindsight_api/engine/retain/types.py,sha256=UzCXauLrMD26g5oZK3_oQ-gTaSSsd-Ttjh17le64HH4,6898
|
|
49
|
+
hindsight_api/engine/search/__init__.py,sha256=YPz_4g7IOabx078Xwg3RBfbOpJ649NRwNfe0gTI9P1U,802
|
|
50
|
+
hindsight_api/engine/search/fusion.py,sha256=cY81BH9U5RyWrPXbQnrDBghtelDMckZWCke9aqMyNnQ,4220
|
|
51
|
+
hindsight_api/engine/search/graph_retrieval.py,sha256=KV1LK_y8R_x4dYwikbZaJTVGPp7kXcrCy0IswaXCD4g,8625
|
|
52
|
+
hindsight_api/engine/search/mpfp_retrieval.py,sha256=mgUgHTj1uhjFWaz5vvqffyJPon01WgGjLq0A_gTWszw,13945
|
|
53
|
+
hindsight_api/engine/search/observation_utils.py,sha256=rlvGA4oFomMZNCZiJvPIQ0iwGaq9XqhRM530unqziCE,4243
|
|
54
|
+
hindsight_api/engine/search/reranking.py,sha256=RZSKe3JDkLfEdTAdgbS-xZka6Jq4mmTBPDXBpyH73zA,3278
|
|
55
|
+
hindsight_api/engine/search/retrieval.py,sha256=BfEYbVC3I9dMBKXzDOfcRaS3r2hKCIO9wg5POP-GXSo,25316
|
|
56
|
+
hindsight_api/engine/search/scoring.py,sha256=7jbBtdnow7JU0d8xdW-ZqYvP4s-TYX2tqPhu2DiqHUI,5132
|
|
57
|
+
hindsight_api/engine/search/temporal_extraction.py,sha256=j7hPqpx2jMdR2BqgFrL-rrV2Hzq8HV24MtjYLJqVl2U,1732
|
|
58
|
+
hindsight_api/engine/search/think_utils.py,sha256=9YAmM_GTSiGns08n6xL8eYW4fZnwm_2xwn5FX6g9xaI,13907
|
|
59
|
+
hindsight_api/engine/search/trace.py,sha256=UTCmNRfAvIvDFGm5ifkuUk6JOKYrLlA_rPA72Zz_DfI,11217
|
|
60
|
+
hindsight_api/engine/search/tracer.py,sha256=hjm8fEESqJnOhsQwmwmvO1gthIO87WC3Pd-iiLPCIEc,15466
|
|
61
|
+
hindsight_api/engine/search/types.py,sha256=2cK-5oynPTWc7UxnA7TFnwzNkcujCfOUvVf5VCk_srM,5594
|
|
62
|
+
hindsight_api/extensions/__init__.py,sha256=gt8RxBwz6JOjbwbPPJ1LGE7ugk1nYkEAlD-LN1ap7FE,1926
|
|
63
|
+
hindsight_api/extensions/base.py,sha256=M7zXuM-tbqDnUwXX1mxAxiFs1eXOzNqIJutKLiUE4mU,2357
|
|
64
|
+
hindsight_api/extensions/context.py,sha256=NXXoBd6Z_nhYWFHgzl6oxrWM_VfPvY99erYLrHR24CE,3640
|
|
65
|
+
hindsight_api/extensions/http.py,sha256=c-a1g6R6rzibyReyR-WHz8DjRRGr4rVSyV9KB4UxVVU,2907
|
|
66
|
+
hindsight_api/extensions/loader.py,sha256=UwGM0XH7zHGng_xfHUY0VbOQemj9DmjuDaMst1TrFi8,4170
|
|
67
|
+
hindsight_api/extensions/operation_validator.py,sha256=zQPD8pTMJJxQjpByxa4JxvGgD5i3A4PBaK9Z1BizL7o,10536
|
|
68
|
+
hindsight_api/extensions/tenant.py,sha256=gvngBMn3cJtUfd4P0P_288faNJq00T8zPQkeldEsD3g,1903
|
|
69
|
+
hindsight_api/extensions/builtin/__init__.py,sha256=hLx2oFYZ1JtZhTWfab6AYcR02SWP2gIdbEqnZezT8ek,526
|
|
70
|
+
hindsight_api/extensions/builtin/tenant.py,sha256=lsS0GDEUXmfPBzqhqk2FpN4Z_k5cA3Y3PFNYyiiuZjU,1444
|
|
71
|
+
hindsight_api-0.1.12.dist-info/METADATA,sha256=LTtW0e2A8m3zXZpy939-UwkPIYabvG6XsZGrLVIV_s8,5408
|
|
72
|
+
hindsight_api-0.1.12.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
|
|
73
|
+
hindsight_api-0.1.12.dist-info/entry_points.txt,sha256=vqZv5WLHbSx8vyec5RtMlUqtE_ul7DTgEVODSmou6Og,109
|
|
74
|
+
hindsight_api-0.1.12.dist-info/RECORD,,
|