memplex 3.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- memnex/__init__.py +31 -0
- memnex/__main__.py +6 -0
- memnex/_plugin/.claude-plugin/plugin.json +24 -0
- memnex/_plugin/.mcp.json +9 -0
- memnex/_plugin/__init__.py +0 -0
- memnex/_plugin/hooks/hooks.json +43 -0
- memnex/_plugin/scripts/hook-runner.py +166 -0
- memnex/_plugin/skills/mem-explore/SKILL.md +83 -0
- memnex/_plugin/skills/mem-manage/SKILL.md +92 -0
- memnex/_plugin/skills/mem-search/SKILL.md +85 -0
- memnex/_plugin/skills/mem-write/SKILL.md +78 -0
- memnex/adapters/__init__.py +14 -0
- memnex/adapters/claude_skill.py +169 -0
- memnex/adapters/cli.py +525 -0
- memnex/adapters/http_api.py +314 -0
- memnex/adapters/mcp_server.py +448 -0
- memnex/compaction.py +563 -0
- memnex/config.py +366 -0
- memnex/core/__init__.py +13 -0
- memnex/core/associator/__init__.py +8 -0
- memnex/core/associator/domain_classifier.py +75 -0
- memnex/core/associator/entity_aligner.py +127 -0
- memnex/core/associator/ref_linker.py +197 -0
- memnex/core/associator/term_mapper.py +77 -0
- memnex/core/dictionaries/__init__.py +50 -0
- memnex/core/engine.py +667 -0
- memnex/core/extractors/__init__.py +15 -0
- memnex/core/extractors/docx.py +97 -0
- memnex/core/extractors/image.py +233 -0
- memnex/core/extractors/markdown.py +139 -0
- memnex/core/extractors/pdf.py +133 -0
- memnex/core/extractors/vision_mapper.py +131 -0
- memnex/core/handlers/__init__.py +7 -0
- memnex/core/handlers/clipboard.py +40 -0
- memnex/core/handlers/file_handler.py +62 -0
- memnex/core/handlers/url_handler.py +132 -0
- memnex/llm/__init__.py +25 -0
- memnex/llm/enhancer.py +226 -0
- memnex/llm/fallback_chain.py +87 -0
- memnex/llm/injection_guard.py +178 -0
- memnex/llm/provider.py +130 -0
- memnex/llm/providers/__init__.py +22 -0
- memnex/llm/providers/anthropic.py +135 -0
- memnex/llm/providers/local.py +135 -0
- memnex/llm/providers/rule_based.py +68 -0
- memnex/llm/sanitizer.py +67 -0
- memnex/models/__init__.py +68 -0
- memnex/models/feedback.py +42 -0
- memnex/models/graph.py +33 -0
- memnex/models/memory.py +102 -0
- memnex/models/misc.py +185 -0
- memnex/models/paragraph.py +45 -0
- memnex/models/search.py +51 -0
- memnex/models/source.py +23 -0
- memnex/models/task.py +62 -0
- memnex/processing/__init__.py +1 -0
- memnex/processing/graph_builder.py +278 -0
- memnex/processing/merger/__init__.py +6 -0
- memnex/processing/merger/confidence_calculator.py +127 -0
- memnex/processing/merger/conflict_resolver.py +116 -0
- memnex/retrieval/__init__.py +1 -0
- memnex/retrieval/dedup.py +386 -0
- memnex/retrieval/embedding.py +289 -0
- memnex/retrieval/reranker.py +299 -0
- memnex/service.py +902 -0
- memnex/storage/__init__.py +65 -0
- memnex/storage/base.py +132 -0
- memnex/storage/changelog.py +106 -0
- memnex/storage/feedback.py +486 -0
- memnex/storage/lite/__init__.py +5 -0
- memnex/storage/lite/store.py +606 -0
- memnex/storage/vector.py +265 -0
- memnex/wiki/__init__.py +11 -0
- memnex/wiki/community.py +221 -0
- memnex/wiki/compiler.py +545 -0
- memnex/wiki/generator.py +270 -0
- memnex/wiki/search.py +282 -0
- memnex/worker.py +412 -0
- memplex-3.2.0.dist-info/METADATA +37 -0
- memplex-3.2.0.dist-info/RECORD +83 -0
- memplex-3.2.0.dist-info/WHEEL +5 -0
- memplex-3.2.0.dist-info/entry_points.txt +2 -0
- memplex-3.2.0.dist-info/top_level.txt +1 -0
memnex/worker.py
ADDED
|
@@ -0,0 +1,412 @@
|
|
|
1
|
+
"""BackgroundWorker -- async task processor for long-running operations.
|
|
2
|
+
|
|
3
|
+
Runs tasks (index builds, wiki compilation, vector refresh, compaction)
|
|
4
|
+
on a daemon thread so the main session is never blocked.
|
|
5
|
+
|
|
6
|
+
Task state is persisted to a lightweight JSON file so that pending/running
|
|
7
|
+
tasks survive process crashes and can be recovered on restart.
|
|
8
|
+
|
|
9
|
+
Usage::
|
|
10
|
+
|
|
11
|
+
worker = BackgroundWorker()
|
|
12
|
+
worker.start()
|
|
13
|
+
task_id = worker.submit(BackgroundTask.BUILD_INDEX, {"func_id": "abc"})
|
|
14
|
+
...
|
|
15
|
+
status = worker.get_status(task_id)
|
|
16
|
+
worker.stop()
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
import json
|
|
22
|
+
import logging
|
|
23
|
+
import os
|
|
24
|
+
import uuid
|
|
25
|
+
from datetime import datetime
|
|
26
|
+
from pathlib import Path
|
|
27
|
+
from queue import Empty, Queue
|
|
28
|
+
from threading import Thread, Timer
|
|
29
|
+
from typing import Any, Callable, Dict, List, Optional
|
|
30
|
+
|
|
31
|
+
from memnex.models import BackgroundTask, TaskInfo, TaskStatus
|
|
32
|
+
|
|
33
|
+
logger = logging.getLogger(__name__)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _generate_uuid() -> str:
|
|
37
|
+
return uuid.uuid4().hex
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def _json_serializer(obj: Any) -> Any:
|
|
41
|
+
"""JSON serializer for non-standard types."""
|
|
42
|
+
if isinstance(obj, datetime):
|
|
43
|
+
return obj.isoformat()
|
|
44
|
+
if isinstance(obj, Path):
|
|
45
|
+
return str(obj)
|
|
46
|
+
raise TypeError(f"Object of type {type(obj)} is not JSON serializable")
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
# ── TaskStore: lightweight JSON persistence ────────────────────────────
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
class TaskStore:
|
|
53
|
+
"""Persist task state to a single JSON file.
|
|
54
|
+
|
|
55
|
+
File format::
|
|
56
|
+
|
|
57
|
+
{
|
|
58
|
+
"tasks": {
|
|
59
|
+
"<task_id>": { ... TaskInfo dict ... },
|
|
60
|
+
...
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
"""
|
|
64
|
+
|
|
65
|
+
def __init__(self, path: Path) -> None:
|
|
66
|
+
self._path = path
|
|
67
|
+
self._tasks: Dict[str, Dict[str, Any]] = {}
|
|
68
|
+
self._load()
|
|
69
|
+
|
|
70
|
+
# ── Persistence ─────────────────────────────────────────────────
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _load(self) -> None:
|
|
74
|
+
"""Load tasks from disk (no-op if file does not exist)."""
|
|
75
|
+
if self._path.exists():
|
|
76
|
+
try:
|
|
77
|
+
with open(self._path, "r", encoding="utf-8") as fh:
|
|
78
|
+
data = json.load(fh)
|
|
79
|
+
self._tasks = data.get("tasks", {})
|
|
80
|
+
except (json.JSONDecodeError, OSError) as exc:
|
|
81
|
+
logger.warning("Failed to load task store from %s: %s", self._path, exc)
|
|
82
|
+
self._tasks = {}
|
|
83
|
+
|
|
84
|
+
def _save(self) -> None:
|
|
85
|
+
"""Flush tasks to disk."""
|
|
86
|
+
self._path.parent.mkdir(parents=True, exist_ok=True)
|
|
87
|
+
try:
|
|
88
|
+
with open(self._path, "w", encoding="utf-8") as fh:
|
|
89
|
+
json.dump({"tasks": self._tasks}, fh, default=_json_serializer, indent=2)
|
|
90
|
+
except OSError as exc:
|
|
91
|
+
logger.error("Failed to persist task store to %s: %s", self._path, exc)
|
|
92
|
+
|
|
93
|
+
# ── CRUD ────────────────────────────────────────────────────────
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def save(self, info: TaskInfo) -> None:
|
|
97
|
+
"""Save or update a TaskInfo."""
|
|
98
|
+
self._tasks[info.task_id] = self._info_to_dict(info)
|
|
99
|
+
self._save()
|
|
100
|
+
|
|
101
|
+
def get(self, task_id: str) -> Optional[TaskInfo]:
|
|
102
|
+
"""Retrieve a TaskInfo by ID, or ``None``."""
|
|
103
|
+
data = self._tasks.get(task_id)
|
|
104
|
+
if data is None:
|
|
105
|
+
return None
|
|
106
|
+
return self._dict_to_info(data)
|
|
107
|
+
|
|
108
|
+
def list_by_status(self, *statuses: TaskStatus) -> List[TaskInfo]:
|
|
109
|
+
"""Return all tasks matching any of the given statuses."""
|
|
110
|
+
status_values = {s.value for s in statuses}
|
|
111
|
+
result: List[TaskInfo] = []
|
|
112
|
+
for data in self._tasks.values():
|
|
113
|
+
if data.get("status") in status_values:
|
|
114
|
+
result.append(self._dict_to_info(data))
|
|
115
|
+
return result
|
|
116
|
+
|
|
117
|
+
# ── Serialisation helpers ───────────────────────────────────────
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
@staticmethod
|
|
121
|
+
def _info_to_dict(info: TaskInfo) -> Dict[str, Any]:
|
|
122
|
+
return {
|
|
123
|
+
"task_id": info.task_id,
|
|
124
|
+
"task_type": info.task_type.value,
|
|
125
|
+
"status": info.status.value,
|
|
126
|
+
"created_at": info.created_at.isoformat() if info.created_at else None,
|
|
127
|
+
"completed_at": info.completed_at.isoformat() if info.completed_at else None,
|
|
128
|
+
"payload": info.payload,
|
|
129
|
+
"result": info.result,
|
|
130
|
+
"error": info.error,
|
|
131
|
+
"retry_count": info.retry_count,
|
|
132
|
+
"max_retries": info.max_retries,
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
@staticmethod
|
|
136
|
+
def _dict_to_info(data: Dict[str, Any]) -> TaskInfo:
|
|
137
|
+
return TaskInfo(
|
|
138
|
+
task_id=data["task_id"],
|
|
139
|
+
task_type=BackgroundTask(data["task_type"]),
|
|
140
|
+
status=TaskStatus(data["status"]),
|
|
141
|
+
created_at=datetime.fromisoformat(data["created_at"]) if data.get("created_at") else datetime.now(),
|
|
142
|
+
completed_at=datetime.fromisoformat(data["completed_at"]) if data.get("completed_at") else None,
|
|
143
|
+
payload=data.get("payload"),
|
|
144
|
+
result=data.get("result"),
|
|
145
|
+
error=data.get("error"),
|
|
146
|
+
retry_count=data.get("retry_count", 0),
|
|
147
|
+
max_retries=data.get("max_retries", 3),
|
|
148
|
+
)
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
# ── BackgroundWorker ──────────────────────────────────────────────────
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
class BackgroundWorker:
|
|
155
|
+
"""Background task processor.
|
|
156
|
+
|
|
157
|
+
Parameters
|
|
158
|
+
----------
|
|
159
|
+
storage_path:
|
|
160
|
+
Path to the JSON file used for task persistence.
|
|
161
|
+
Defaults to ``~/.memnex/tasks.json``.
|
|
162
|
+
"""
|
|
163
|
+
|
|
164
|
+
def __init__(
|
|
165
|
+
self,
|
|
166
|
+
storage_path: Path = Path("~/.memnex/tasks.json").expanduser(),
|
|
167
|
+
) -> None:
|
|
168
|
+
self._task_store = TaskStore(storage_path)
|
|
169
|
+
self._queue: Queue = Queue()
|
|
170
|
+
self._running: bool = False
|
|
171
|
+
self._worker_thread: Optional[Thread] = None
|
|
172
|
+
self._recover_pending_tasks()
|
|
173
|
+
|
|
174
|
+
# ── Lifecycle ───────────────────────────────────────────────────
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def start(self) -> None:
|
|
178
|
+
"""Start the background worker daemon thread."""
|
|
179
|
+
if self._running:
|
|
180
|
+
return
|
|
181
|
+
self._running = True
|
|
182
|
+
self._worker_thread = Thread(target=self._run_loop, daemon=True)
|
|
183
|
+
self._worker_thread.start()
|
|
184
|
+
logger.info("BackgroundWorker started")
|
|
185
|
+
|
|
186
|
+
def stop(self, timeout: float = 30.0) -> None:
|
|
187
|
+
"""Gracefully stop the worker, waiting up to *timeout* seconds."""
|
|
188
|
+
self._running = False
|
|
189
|
+
if self._worker_thread is not None:
|
|
190
|
+
self._worker_thread.join(timeout=timeout)
|
|
191
|
+
if self._worker_thread.is_alive():
|
|
192
|
+
logger.warning("BackgroundWorker did not stop within %.1fs", timeout)
|
|
193
|
+
logger.info("BackgroundWorker stopped")
|
|
194
|
+
|
|
195
|
+
# ── Public API ──────────────────────────────────────────────────
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def submit(
|
|
199
|
+
self,
|
|
200
|
+
task: BackgroundTask,
|
|
201
|
+
payload: dict,
|
|
202
|
+
callback: Optional[Callable] = None,
|
|
203
|
+
) -> str:
|
|
204
|
+
"""Submit a background task, returning the ``task_id``.
|
|
205
|
+
|
|
206
|
+
Parameters
|
|
207
|
+
----------
|
|
208
|
+
task:
|
|
209
|
+
Type of background task.
|
|
210
|
+
payload:
|
|
211
|
+
Arbitrary data forwarded to the task handler.
|
|
212
|
+
callback:
|
|
213
|
+
Optional function called with the handler's result on success.
|
|
214
|
+
"""
|
|
215
|
+
task_id = _generate_uuid()
|
|
216
|
+
info = TaskInfo(
|
|
217
|
+
task_id=task_id,
|
|
218
|
+
task_type=task,
|
|
219
|
+
status=TaskStatus.PENDING,
|
|
220
|
+
created_at=datetime.now(),
|
|
221
|
+
payload=payload,
|
|
222
|
+
)
|
|
223
|
+
self._task_store.save(info)
|
|
224
|
+
self._queue.put(
|
|
225
|
+
{"id": task_id, "task": task, "payload": payload, "callback": callback}
|
|
226
|
+
)
|
|
227
|
+
logger.debug("Submitted task %s (%s)", task_id, task.value)
|
|
228
|
+
return task_id
|
|
229
|
+
|
|
230
|
+
def get_status(self, task_id: str) -> TaskStatus:
|
|
231
|
+
"""Return the current status of a task."""
|
|
232
|
+
info = self._task_store.get(task_id)
|
|
233
|
+
if info is None:
|
|
234
|
+
raise KeyError(f"Task {task_id!r} not found")
|
|
235
|
+
return info.status
|
|
236
|
+
|
|
237
|
+
def cancel(self, task_id: str) -> bool:
|
|
238
|
+
"""Cancel a pending task.
|
|
239
|
+
|
|
240
|
+
Returns ``True`` if the task was successfully cancelled,
|
|
241
|
+
``False`` if it was already running/completed.
|
|
242
|
+
"""
|
|
243
|
+
info = self._task_store.get(task_id)
|
|
244
|
+
if info is None:
|
|
245
|
+
return False
|
|
246
|
+
if info.status in (TaskStatus.PENDING,):
|
|
247
|
+
info.status = TaskStatus.CANCELLED
|
|
248
|
+
self._task_store.save(info)
|
|
249
|
+
return True
|
|
250
|
+
return False
|
|
251
|
+
|
|
252
|
+
# ── Worker loop ─────────────────────────────────────────────────
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
def _run_loop(self) -> None:
|
|
256
|
+
"""Main loop: dequeue tasks and execute them."""
|
|
257
|
+
while self._running:
|
|
258
|
+
try:
|
|
259
|
+
task = self._queue.get(timeout=5.0)
|
|
260
|
+
self._execute_task(task)
|
|
261
|
+
except Empty:
|
|
262
|
+
continue
|
|
263
|
+
except Exception as exc:
|
|
264
|
+
logger.error("Unexpected error in worker loop: %s", exc, exc_info=True)
|
|
265
|
+
|
|
266
|
+
def _execute_task(self, task: dict) -> None:
|
|
267
|
+
"""Execute a single task with retry support.
|
|
268
|
+
|
|
269
|
+
Steps:
|
|
270
|
+
1. Update status to RUNNING.
|
|
271
|
+
2. Dispatch to the correct handler.
|
|
272
|
+
3. On success: COMPLETED + optional callback.
|
|
273
|
+
4. On failure: retry with exponential backoff (Timer), or mark FAILED.
|
|
274
|
+
"""
|
|
275
|
+
task_id = task["id"]
|
|
276
|
+
task_type = task["task"]
|
|
277
|
+
payload = task.get("payload", {})
|
|
278
|
+
|
|
279
|
+
info = self._task_store.get(task_id)
|
|
280
|
+
if info is None:
|
|
281
|
+
return
|
|
282
|
+
|
|
283
|
+
info.status = TaskStatus.RUNNING
|
|
284
|
+
self._task_store.save(info)
|
|
285
|
+
|
|
286
|
+
try:
|
|
287
|
+
result = self._dispatch(task_type, payload)
|
|
288
|
+
info.status = TaskStatus.COMPLETED
|
|
289
|
+
info.result = result
|
|
290
|
+
info.completed_at = datetime.now()
|
|
291
|
+
|
|
292
|
+
callback = task.get("callback")
|
|
293
|
+
if callback is not None:
|
|
294
|
+
try:
|
|
295
|
+
callback(result)
|
|
296
|
+
except Exception as cb_exc:
|
|
297
|
+
logger.warning("Callback error for task %s: %s", task_id, cb_exc)
|
|
298
|
+
|
|
299
|
+
except Exception as exc:
|
|
300
|
+
if info.retry_count < info.max_retries:
|
|
301
|
+
info.retry_count += 1
|
|
302
|
+
info.status = TaskStatus.PENDING
|
|
303
|
+
self._task_store.save(info)
|
|
304
|
+
delay = min(2 ** info.retry_count, 30) # exponential backoff, max 30s
|
|
305
|
+
logger.info(
|
|
306
|
+
"Retrying task %s (%s) in %ds (attempt %d/%d)",
|
|
307
|
+
task_id, task_type.value, delay, info.retry_count, info.max_retries,
|
|
308
|
+
)
|
|
309
|
+
Timer(
|
|
310
|
+
delay,
|
|
311
|
+
self._queue.put,
|
|
312
|
+
args=[{"id": task_id, "task": task_type, "payload": payload}],
|
|
313
|
+
).start()
|
|
314
|
+
return
|
|
315
|
+
info.status = TaskStatus.FAILED
|
|
316
|
+
info.error = str(exc)
|
|
317
|
+
logger.error("Task %s failed permanently: %s", task_id, exc)
|
|
318
|
+
|
|
319
|
+
finally:
|
|
320
|
+
self._task_store.save(info)
|
|
321
|
+
|
|
322
|
+
# ── Dispatch & task handlers ────────────────────────────────────
|
|
323
|
+
|
|
324
|
+
|
|
325
|
+
def _dispatch(self, task_type: BackgroundTask, payload: dict) -> Any:
|
|
326
|
+
"""Route a task to its handler."""
|
|
327
|
+
handlers = {
|
|
328
|
+
BackgroundTask.EXTRACT_DOCUMENT: self._handle_extract,
|
|
329
|
+
BackgroundTask.BUILD_INDEX: self._handle_build_index,
|
|
330
|
+
BackgroundTask.COMPILE_WIKI: self._handle_compile_wiki,
|
|
331
|
+
BackgroundTask.REFRESH_VECTOR: self._handle_refresh_vector,
|
|
332
|
+
BackgroundTask.COMPACTION: self._handle_compaction,
|
|
333
|
+
}
|
|
334
|
+
handler = handlers.get(task_type)
|
|
335
|
+
if handler is None:
|
|
336
|
+
raise ValueError(f"Unknown task type: {task_type}")
|
|
337
|
+
return handler(payload)
|
|
338
|
+
|
|
339
|
+
def _handle_extract(self, payload: dict) -> dict:
|
|
340
|
+
"""Handle EXTRACT_DOCUMENT tasks.
|
|
341
|
+
|
|
342
|
+
Stub: concrete extraction logic is wired by the application layer
|
|
343
|
+
(MemNexService or a plugin). This method provides the scaffolding.
|
|
344
|
+
"""
|
|
345
|
+
logger.info("Extract document: %s", payload.get("source_id", "<unknown>"))
|
|
346
|
+
return {"status": "completed", "extracted": True}
|
|
347
|
+
|
|
348
|
+
def _handle_build_index(self, payload: dict) -> dict:
|
|
349
|
+
"""Handle BUILD_INDEX tasks."""
|
|
350
|
+
logger.info("Build index: %s", payload.get("func_id", "<batch>"))
|
|
351
|
+
return {"status": "completed", "indexed": True}
|
|
352
|
+
|
|
353
|
+
def _handle_compile_wiki(self, payload: dict) -> dict:
|
|
354
|
+
"""Handle COMPILE_WIKI tasks."""
|
|
355
|
+
logger.info("Compile wiki: %s", payload.get("domain", "<all>"))
|
|
356
|
+
return {"status": "completed", "compiled": True}
|
|
357
|
+
|
|
358
|
+
def _handle_refresh_vector(self, payload: dict) -> dict:
|
|
359
|
+
"""Handle REFRESH_VECTOR tasks."""
|
|
360
|
+
logger.info("Refresh vector: %s", payload.get("func_id", "<batch>"))
|
|
361
|
+
return {"status": "completed", "refreshed": True}
|
|
362
|
+
|
|
363
|
+
def _handle_compaction(self, payload: dict) -> dict:
|
|
364
|
+
"""Handle COMPACTION tasks.
|
|
365
|
+
|
|
366
|
+
The actual :class:`CompactionPipeline` is invoked here. Because
|
|
367
|
+
the pipeline's ``run`` method is async, this handler calls
|
|
368
|
+
``asyncio.run`` (or the thread-safe equivalent) when a pipeline
|
|
369
|
+
instance is injected.
|
|
370
|
+
"""
|
|
371
|
+
trigger = payload.get("trigger", "manual")
|
|
372
|
+
scope = payload.get("scope", "global")
|
|
373
|
+
logger.info("Compaction triggered: trigger=%s scope=%s", trigger, scope)
|
|
374
|
+
return {"status": "completed", "trigger": trigger, "scope": scope}
|
|
375
|
+
|
|
376
|
+
# ── Recovery ────────────────────────────────────────────────────
|
|
377
|
+
|
|
378
|
+
|
|
379
|
+
def _recover_pending_tasks(self) -> None:
|
|
380
|
+
"""Re-queue tasks that were pending or running when the process died."""
|
|
381
|
+
pending = self._task_store.list_by_status(
|
|
382
|
+
TaskStatus.PENDING, TaskStatus.RUNNING
|
|
383
|
+
)
|
|
384
|
+
for t in pending:
|
|
385
|
+
if t.status == TaskStatus.RUNNING:
|
|
386
|
+
# Only re-queue RUNNING tasks that have exceeded a 1-hour timeout
|
|
387
|
+
elapsed = (datetime.now() - t.created_at).total_seconds()
|
|
388
|
+
if elapsed > 3600:
|
|
389
|
+
t.status = TaskStatus.PENDING
|
|
390
|
+
self._task_store.save(t)
|
|
391
|
+
self._queue.put(
|
|
392
|
+
{
|
|
393
|
+
"id": t.task_id,
|
|
394
|
+
"task": t.task_type,
|
|
395
|
+
"payload": t.payload or {},
|
|
396
|
+
}
|
|
397
|
+
)
|
|
398
|
+
else:
|
|
399
|
+
logger.warning(
|
|
400
|
+
"Task %s (%s) was RUNNING before shutdown; "
|
|
401
|
+
"skipping re-queue (not timed out)",
|
|
402
|
+
t.task_id,
|
|
403
|
+
t.task_type.value,
|
|
404
|
+
)
|
|
405
|
+
else:
|
|
406
|
+
self._queue.put(
|
|
407
|
+
{
|
|
408
|
+
"id": t.task_id,
|
|
409
|
+
"task": t.task_type,
|
|
410
|
+
"payload": t.payload or {},
|
|
411
|
+
}
|
|
412
|
+
)
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: memplex
|
|
3
|
+
Version: 3.2.0
|
|
4
|
+
Summary: Memplex - Memory Complex: multi-agent knowledge graph memory system with 3-layer retrieval
|
|
5
|
+
Requires-Python: >=3.11
|
|
6
|
+
Requires-Dist: pyyaml>=6.0
|
|
7
|
+
Requires-Dist: numpy>=1.24.0
|
|
8
|
+
Requires-Dist: requests>=2.28.0
|
|
9
|
+
Provides-Extra: embedding
|
|
10
|
+
Requires-Dist: sentence-transformers>=2.0; extra == "embedding"
|
|
11
|
+
Provides-Extra: extractors
|
|
12
|
+
Requires-Dist: pdfplumber>=0.9.0; extra == "extractors"
|
|
13
|
+
Requires-Dist: python-docx>=0.8.11; extra == "extractors"
|
|
14
|
+
Requires-Dist: Pillow>=9.0; extra == "extractors"
|
|
15
|
+
Requires-Dist: pytesseract>=0.3.10; extra == "extractors"
|
|
16
|
+
Provides-Extra: vector
|
|
17
|
+
Requires-Dist: chromadb>=0.4.0; extra == "vector"
|
|
18
|
+
Requires-Dist: sentence-transformers>=2.0; extra == "vector"
|
|
19
|
+
Provides-Extra: graph
|
|
20
|
+
Requires-Dist: networkx>=3.0; extra == "graph"
|
|
21
|
+
Requires-Dist: python-louvain>=0.16; extra == "graph"
|
|
22
|
+
Provides-Extra: http
|
|
23
|
+
Requires-Dist: fastapi>=0.100.0; extra == "http"
|
|
24
|
+
Requires-Dist: uvicorn>=0.23.0; extra == "http"
|
|
25
|
+
Provides-Extra: llm
|
|
26
|
+
Requires-Dist: anthropic>=0.30.0; extra == "llm"
|
|
27
|
+
Requires-Dist: openai>=1.0.0; extra == "llm"
|
|
28
|
+
Provides-Extra: postgres
|
|
29
|
+
Requires-Dist: asyncpg>=0.29.0; extra == "postgres"
|
|
30
|
+
Provides-Extra: neo4j
|
|
31
|
+
Requires-Dist: neo4j>=5.0.0; extra == "neo4j"
|
|
32
|
+
Provides-Extra: all
|
|
33
|
+
Requires-Dist: memnex[embedding,extractors,graph,http,llm,vector]; extra == "all"
|
|
34
|
+
Provides-Extra: dev
|
|
35
|
+
Requires-Dist: pytest>=7.0; extra == "dev"
|
|
36
|
+
Requires-Dist: pytest-asyncio>=0.21; extra == "dev"
|
|
37
|
+
Requires-Dist: ruff>=0.1.0; extra == "dev"
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
memnex/__init__.py,sha256=Fvq5TO_UfCi4rm4NStjFO21JAzz8LxmqyQkse3VHWhc,625
|
|
2
|
+
memnex/__main__.py,sha256=vyokcq-HeGBLwJl0ioY95yHrNKCCdqomN7tKk7u-1S8,111
|
|
3
|
+
memnex/compaction.py,sha256=wOdLRKpVeDuFcxWc2AYBcgMkUR8kKAB03KXrB1ehaGQ,20703
|
|
4
|
+
memnex/config.py,sha256=DdH4Xb8vBEV5yw28i8tPO3gfZ9OBQl5GOt_UKch3pgk,11799
|
|
5
|
+
memnex/service.py,sha256=LnnhwWnVZnvkMaSygR8OM6vrek_blf7llDSTf6WxySs,33827
|
|
6
|
+
memnex/worker.py,sha256=szJcIADLQP1779OsYhr1W7Dwa-k8xpev7UytqrVtfMI,15516
|
|
7
|
+
memnex/_plugin/.mcp.json,sha256=gV4HuPqbWm68yZZA1zN4lzzq027iijN207oX3ZE8yVc,149
|
|
8
|
+
memnex/_plugin/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
9
|
+
memnex/_plugin/.claude-plugin/plugin.json,sha256=5VMvDw2OGl4isox_GOnii8AbWxDFBrPN5ddUux7c4_A,546
|
|
10
|
+
memnex/_plugin/hooks/hooks.json,sha256=F9CUi_qfM5IqGTc0mLVbbjgjd723eeeF_GGFBFZhSC0,1415
|
|
11
|
+
memnex/_plugin/scripts/hook-runner.py,sha256=9nz5rJ7NhPDf1rdCy0nRRLqe_KDiYUmC0nHIZSxu6LQ,4977
|
|
12
|
+
memnex/_plugin/skills/mem-explore/SKILL.md,sha256=n4a5jumCKK5EFqlcr3Kqh6Dh1yCPVcTzsWF7EWvVW-w,1868
|
|
13
|
+
memnex/_plugin/skills/mem-manage/SKILL.md,sha256=BCA8T126UpfkchP0bzxcBbaDv4d-AI-Mr8UkH68omZ4,1795
|
|
14
|
+
memnex/_plugin/skills/mem-search/SKILL.md,sha256=iXEVA-p3Pw9jh4Pci9In8b8gsXEr0Vi0-e-uBKT2K7s,2134
|
|
15
|
+
memnex/_plugin/skills/mem-write/SKILL.md,sha256=o7-ASDZ07yX5CakRAYLVajxXlayuw5MK3NhUxKaHYdY,1830
|
|
16
|
+
memnex/adapters/__init__.py,sha256=hozQpKX4T-yYaf_8r_PM0XJ2mYDxUH_kbitdVOny0Lg,489
|
|
17
|
+
memnex/adapters/claude_skill.py,sha256=016WGTfqHS5IeuXjsFAKdXyqBCRCOGUuiUpeIz2M9sU,4702
|
|
18
|
+
memnex/adapters/cli.py,sha256=jlS0HTk5IQckP97KQbxaizQLFT7usTjFYz1D9KTRAWY,15983
|
|
19
|
+
memnex/adapters/http_api.py,sha256=Img2GnY8XPlMyQyBLf21yUsDbjsvu4Y6T3dJdXszwYY,11159
|
|
20
|
+
memnex/adapters/mcp_server.py,sha256=K_aEpvNo4TDXiiBAkDmycPsBo8ec2nK1DC1USzMnzDg,16197
|
|
21
|
+
memnex/core/__init__.py,sha256=oM1wY1hnMULz3y-cA1I-Rs_EwH6aocrEMyKVMnTpJ0g,244
|
|
22
|
+
memnex/core/engine.py,sha256=qqALfdMcHPetK8eRNRvOzvHe5bDcUG74vKY9ngS5QeA,26002
|
|
23
|
+
memnex/core/associator/__init__.py,sha256=RIio0mK9v3IwwmQk3AtQ4pZnT32-xKjN4vXGOh_2wH4,276
|
|
24
|
+
memnex/core/associator/domain_classifier.py,sha256=n4QtJwmq42mINpFnU6rg9qSmaGwfXpsjKdEPAk4XAus,3264
|
|
25
|
+
memnex/core/associator/entity_aligner.py,sha256=vsKjrGeehXL11SsFcistKwNgYO_b5RAcwIIuy55-B2c,4263
|
|
26
|
+
memnex/core/associator/ref_linker.py,sha256=iqOsDaTZdVE4KgKDGBpHhgvRpPw12UwJ2iERwF-1kjc,7067
|
|
27
|
+
memnex/core/associator/term_mapper.py,sha256=_bMJ519UIooAI3xy6_92VsXKhs1XQ7OG2HAu4aSYAEQ,2760
|
|
28
|
+
memnex/core/dictionaries/__init__.py,sha256=F191-dJCbewWIccBLIlng6nEs4ckIUVK4oEB7cxMGzg,1652
|
|
29
|
+
memnex/core/extractors/__init__.py,sha256=obVyUCZMNbNlwlDBz4BogzTAjP8K_RM-1SznlmDW0Hs,347
|
|
30
|
+
memnex/core/extractors/docx.py,sha256=4JJi77uRa3uXrqnZx2CuP0OhuDee-Au5B6Z7Os88NpQ,2749
|
|
31
|
+
memnex/core/extractors/image.py,sha256=n63ZfzQ6v6mRfH6qNOEYZNHIXax6B6Nd0OvPq8fsmY8,8424
|
|
32
|
+
memnex/core/extractors/markdown.py,sha256=0AvLSDtVPGUCqfw7rWKAW1R7H7QqKsxP5TgoBKS2c-E,4858
|
|
33
|
+
memnex/core/extractors/pdf.py,sha256=95u2HdI1R92s2wq3Od9QOABs8hP24uXWwkpx5S4mFTU,4962
|
|
34
|
+
memnex/core/extractors/vision_mapper.py,sha256=a3DB0aLdzdNvfSC4GgCa10jeWDG4oS4xrY4EZ8iHXGY,4201
|
|
35
|
+
memnex/core/handlers/__init__.py,sha256=xhTYXT6UMsICyHT2r1nJ6jrl5NalyCHfpciCeI_SJU8,226
|
|
36
|
+
memnex/core/handlers/clipboard.py,sha256=CJiTMs7CpJdx0tuOFcPqx7TH6gvklPbtqG7bhbz0mQ0,996
|
|
37
|
+
memnex/core/handlers/file_handler.py,sha256=kCQFt1g4P4abbE_YFlKChvF8HLsRyl2LusFs-t90f50,1723
|
|
38
|
+
memnex/core/handlers/url_handler.py,sha256=2EWsxQIUM_YtwenLQQzg3T7XCCpSitIF6r-BCLMVer4,4290
|
|
39
|
+
memnex/llm/__init__.py,sha256=8jUzWx7fGplalWmvYFeXXs3yTohr7qLVtnCzhiBLh4E,857
|
|
40
|
+
memnex/llm/enhancer.py,sha256=4ZCKUHQsiXaz7zb2DWh4KS0JLlIotveGYTYpUVSHKqU,8373
|
|
41
|
+
memnex/llm/fallback_chain.py,sha256=jZpV529CTFvTBTr07UNPzz28FRAI2io915KrTcIyNmY,3138
|
|
42
|
+
memnex/llm/injection_guard.py,sha256=T3x8UYTKR4cPOsrOJvbVlWNmmoF4SRnVZzptw-d524s,6082
|
|
43
|
+
memnex/llm/provider.py,sha256=yQcWCD0moWLBpohoqnygUJpGodJL-X3-a1g7TOjNZjk,4219
|
|
44
|
+
memnex/llm/sanitizer.py,sha256=ufjRGa-8bRlRQqZKrMkDRT6YjwyCJ1c-xcwiAe3oM8M,2198
|
|
45
|
+
memnex/llm/providers/__init__.py,sha256=7s1zR8QNx-h_IvGn-6PbRIWcI52ogfexb14mIEsHvv4,751
|
|
46
|
+
memnex/llm/providers/anthropic.py,sha256=2yqsopfH3kFgTrtKZQe63qa1MEe0qdvqzlzvVVz_FBg,4765
|
|
47
|
+
memnex/llm/providers/local.py,sha256=_5l0frfqd2v_v86NcgC2zQw08--jjcpGQ3hTj7v1iEA,4852
|
|
48
|
+
memnex/llm/providers/rule_based.py,sha256=2pQixBI_M4hRTUKp31fmLNeVVly99K45dV5PhFpnKdU,2396
|
|
49
|
+
memnex/models/__init__.py,sha256=wvoToWac6Nyh2ihZnbDkENyt02PYOQnVzHL6v0Dge1Y,1113
|
|
50
|
+
memnex/models/feedback.py,sha256=gGoeGteYT4R-uNp3_B7HSj_Q4n4uZHh2a6rCvr8sMPA,1103
|
|
51
|
+
memnex/models/graph.py,sha256=Kquzk-dOivI4SPLzIQxjp7kp5eZPBPoUxUOYxgNgWCo,831
|
|
52
|
+
memnex/models/memory.py,sha256=r_TWV5vD4Gpo4jxzXTE50BeTr8MP_HCrzqlGjLhTiAc,3193
|
|
53
|
+
memnex/models/misc.py,sha256=X6VbkH1J9xToMkY5MSFnUfVjx07n2FCYpcXt-ZK2PVc,4873
|
|
54
|
+
memnex/models/paragraph.py,sha256=yUIA8zl1bmYyWsBkVVBhzv73Q5JLLQlP2y7PTmUJzfA,1062
|
|
55
|
+
memnex/models/search.py,sha256=Z_TniJoiuTryYUNcmLXEqJbYg5xdnLK5nQQf3CjcScA,1244
|
|
56
|
+
memnex/models/source.py,sha256=TsM_ps9tlu7Im86FyYyzDDEl6vPHAgDKncB-oJGesME,554
|
|
57
|
+
memnex/models/task.py,sha256=QmeCjFDVawLjXpnQpII6tmDI_c1Kw-UkECsm6ptdF3Y,1311
|
|
58
|
+
memnex/processing/__init__.py,sha256=jmRg4f0OMcMb0ik4cYNd37EszTUA3H3ASbRJ3sGMU1E,70
|
|
59
|
+
memnex/processing/graph_builder.py,sha256=HmCv7ZQFOqsBjqYImq50zvehW2kEyCE7CG0pkVDOfHM,9985
|
|
60
|
+
memnex/processing/merger/__init__.py,sha256=GswF67EW40RnOoZu4XDngl7wteT5Ab2vl7EwRAkG2xM,228
|
|
61
|
+
memnex/processing/merger/confidence_calculator.py,sha256=_UjzDat0xVR_Gg1jHQBtvm-uC7zzpD0jGF2-4kUQ-w0,3632
|
|
62
|
+
memnex/processing/merger/conflict_resolver.py,sha256=-W6qTFMVkgTvZcYrm3FlGXoyfv1UWSbSjeT800WDCjg,4303
|
|
63
|
+
memnex/retrieval/__init__.py,sha256=U2tfpBNkpxGUiTrCbFRgW0LNwBOiSzngGqD2XPB6EZo,63
|
|
64
|
+
memnex/retrieval/dedup.py,sha256=XJZE6VzF6qARriytAQyo63PP9Q5DG24jZpoXZrRfAo4,13755
|
|
65
|
+
memnex/retrieval/embedding.py,sha256=WV4mQYMcMb3rU2ELLz2_YDBVb6bl2ejjSh3Nt0xuwg0,9926
|
|
66
|
+
memnex/retrieval/reranker.py,sha256=VMYa00mjdImfOyLg398giCg8ZHTVxDnipoBkajtvRn0,10493
|
|
67
|
+
memnex/storage/__init__.py,sha256=a3eSLKcW43-vhYtg64dDifWmab1_0J022vJeli8Vz18,1769
|
|
68
|
+
memnex/storage/base.py,sha256=keClRSIRSSvDEnjG-BpZIYJqib8qwTV4jXOAGet7Vio,4329
|
|
69
|
+
memnex/storage/changelog.py,sha256=NeJPaeyP1b0xsm0C3WE7xwqP0Ey15DtY_YSnFyfR818,3516
|
|
70
|
+
memnex/storage/feedback.py,sha256=KX5nh0wUyMHH3_N63CCN22pbsPp9RLI3XAV-G0Skg4k,17809
|
|
71
|
+
memnex/storage/vector.py,sha256=3NMcW7ZGTns8ztXxXUNSU3jNhSzIZUAriHaPGzp-qMM,9149
|
|
72
|
+
memnex/storage/lite/__init__.py,sha256=tkb36Zswwho1MV2VVP7asgLd8pm50BIHh-emsttr7IA,146
|
|
73
|
+
memnex/storage/lite/store.py,sha256=qcPdZhz8DnDKwgzSmSt_5DDpjOWByODIK-xwbZt9FUE,22225
|
|
74
|
+
memnex/wiki/__init__.py,sha256=Ecbi5q42fou-YCW4SJlLTIdKUbo6zaoorimS3OK1wn0,302
|
|
75
|
+
memnex/wiki/community.py,sha256=MBlsH3KQSxPZvr6btGhl6lLpDX8WA4huAflVw_gq5hk,6797
|
|
76
|
+
memnex/wiki/compiler.py,sha256=4J2Fpm_Tuc-Z4tnRraZLfeZvVsR_3S5LWu08vIXnVHY,19377
|
|
77
|
+
memnex/wiki/generator.py,sha256=88074p1YsniKfSjO03_N4FBrpPQSJ4i7YCmB6aaaGts,9708
|
|
78
|
+
memnex/wiki/search.py,sha256=ChRIY64mBGHusmsOegHaV0ZHsFCC74s89birhSbgWbc,9935
|
|
79
|
+
memplex-3.2.0.dist-info/METADATA,sha256=U2hg8XAvGYPzqffOdf0i_5hCxNlPfwl2fIw4v7lqJeU,1496
|
|
80
|
+
memplex-3.2.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
|
|
81
|
+
memplex-3.2.0.dist-info/entry_points.txt,sha256=-Yxo9jmeV9cE-H9dv6O1FMgXtP9CxtNsk4gt5ICBsY8,39
|
|
82
|
+
memplex-3.2.0.dist-info/top_level.txt,sha256=PvQgcpCkjlAM8AHaruBtzobW2qVjHtCutBWNgg4j8XQ,7
|
|
83
|
+
memplex-3.2.0.dist-info/RECORD,,
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
memnex
|