memorius 0.4.1__tar.gz → 0.4.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {memorius-0.4.1/memorius.egg-info → memorius-0.4.2}/PKG-INFO +1 -1
- {memorius-0.4.1 → memorius-0.4.2}/memorius/__init__.py +1 -1
- {memorius-0.4.1 → memorius-0.4.2}/memorius/cli/main.py +87 -0
- {memorius-0.4.1 → memorius-0.4.2}/memorius/config.py +14 -0
- memorius-0.4.2/memorius/web_search.py +191 -0
- {memorius-0.4.1 → memorius-0.4.2/memorius.egg-info}/PKG-INFO +1 -1
- {memorius-0.4.1 → memorius-0.4.2}/memorius.egg-info/SOURCES.txt +3 -1
- {memorius-0.4.1 → memorius-0.4.2}/pyproject.toml +1 -1
- {memorius-0.4.1 → memorius-0.4.2}/tests/test_integration.py +27 -9
- memorius-0.4.2/tests/test_web_search.py +213 -0
- {memorius-0.4.1 → memorius-0.4.2}/LICENSE +0 -0
- {memorius-0.4.1 → memorius-0.4.2}/MANIFEST.in +0 -0
- {memorius-0.4.1 → memorius-0.4.2}/README.md +0 -0
- {memorius-0.4.1 → memorius-0.4.2}/examples/quickstart.py +0 -0
- {memorius-0.4.1 → memorius-0.4.2}/hooks/memorius-hook.sh +0 -0
- {memorius-0.4.1 → memorius-0.4.2}/manifest.yaml +0 -0
- {memorius-0.4.1 → memorius-0.4.2}/memorius/cli/__init__.py +0 -0
- {memorius-0.4.1 → memorius-0.4.2}/memorius/cli/obsidian.py +0 -0
- {memorius-0.4.1 → memorius-0.4.2}/memorius/consolidation.py +0 -0
- {memorius-0.4.1 → memorius-0.4.2}/memorius/context_inject.py +0 -0
- {memorius-0.4.1 → memorius-0.4.2}/memorius/data/LICENSE +0 -0
- {memorius-0.4.1 → memorius-0.4.2}/memorius/data/__init__.py +0 -0
- {memorius-0.4.1 → memorius-0.4.2}/memorius/data/examples/__init__.py +0 -0
- {memorius-0.4.1 → memorius-0.4.2}/memorius/data/examples/quickstart.py +0 -0
- {memorius-0.4.1 → memorius-0.4.2}/memorius/data/manifest.yaml +0 -0
- {memorius-0.4.1 → memorius-0.4.2}/memorius/data/memorius-hook.sh +0 -0
- {memorius-0.4.1 → memorius-0.4.2}/memorius/embeddings.py +0 -0
- {memorius-0.4.1 → memorius-0.4.2}/memorius/factcheck.py +0 -0
- {memorius-0.4.1 → memorius-0.4.2}/memorius/graph.py +0 -0
- {memorius-0.4.1 → memorius-0.4.2}/memorius/hooks/__init__.py +0 -0
- {memorius-0.4.1 → memorius-0.4.2}/memorius/hooks/adapters/__init__.py +0 -0
- {memorius-0.4.1 → memorius-0.4.2}/memorius/hooks/adapters/base.py +0 -0
- {memorius-0.4.1 → memorius-0.4.2}/memorius/hooks/adapters/claude_code.py +0 -0
- {memorius-0.4.1 → memorius-0.4.2}/memorius/hooks/adapters/codex.py +0 -0
- {memorius-0.4.1 → memorius-0.4.2}/memorius/hooks/adapters/gemini.py +0 -0
- {memorius-0.4.1 → memorius-0.4.2}/memorius/hooks/adapters/generic.py +0 -0
- {memorius-0.4.1 → memorius-0.4.2}/memorius/hooks/adapters/openclaude.py +0 -0
- {memorius-0.4.1 → memorius-0.4.2}/memorius/hooks/adapters/openclaw.py +0 -0
- {memorius-0.4.1 → memorius-0.4.2}/memorius/hooks/adapters/opencode.py +0 -0
- {memorius-0.4.1 → memorius-0.4.2}/memorius/hooks/adapters/pi.py +0 -0
- {memorius-0.4.1 → memorius-0.4.2}/memorius/hooks/cli.py +0 -0
- {memorius-0.4.1 → memorius-0.4.2}/memorius/hooks/engine.py +0 -0
- {memorius-0.4.1 → memorius-0.4.2}/memorius/hooks/models.py +0 -0
- {memorius-0.4.1 → memorius-0.4.2}/memorius/llm_extract.py +0 -0
- {memorius-0.4.1 → memorius-0.4.2}/memorius/mcp_server.py +0 -0
- {memorius-0.4.1 → memorius-0.4.2}/memorius/meta_store.py +0 -0
- {memorius-0.4.1 → memorius-0.4.2}/memorius/model_download.py +0 -0
- {memorius-0.4.1 → memorius-0.4.2}/memorius/models.py +0 -0
- {memorius-0.4.1 → memorius-0.4.2}/memorius/normalizers/__init__.py +0 -0
- {memorius-0.4.1 → memorius-0.4.2}/memorius/normalizers/cli.py +0 -0
- {memorius-0.4.1 → memorius-0.4.2}/memorius/plugin_gen/__init__.py +0 -0
- {memorius-0.4.1 → memorius-0.4.2}/memorius/plugin_gen/cli.py +0 -0
- {memorius-0.4.1 → memorius-0.4.2}/memorius/rest_server.py +0 -0
- {memorius-0.4.1 → memorius-0.4.2}/memorius/session.py +0 -0
- {memorius-0.4.1 → memorius-0.4.2}/memorius/temporal.py +0 -0
- {memorius-0.4.1 → memorius-0.4.2}/memorius/utils.py +0 -0
- {memorius-0.4.1 → memorius-0.4.2}/memorius/validation.py +0 -0
- {memorius-0.4.1 → memorius-0.4.2}/memorius/vault.py +0 -0
- {memorius-0.4.1 → memorius-0.4.2}/memorius/vector_store.py +0 -0
- {memorius-0.4.1 → memorius-0.4.2}/memorius.egg-info/dependency_links.txt +0 -0
- {memorius-0.4.1 → memorius-0.4.2}/memorius.egg-info/entry_points.txt +0 -0
- {memorius-0.4.1 → memorius-0.4.2}/memorius.egg-info/requires.txt +0 -0
- {memorius-0.4.1 → memorius-0.4.2}/memorius.egg-info/top_level.txt +0 -0
- {memorius-0.4.1 → memorius-0.4.2}/setup.cfg +0 -0
- {memorius-0.4.1 → memorius-0.4.2}/tests/test_core.py +0 -0
- {memorius-0.4.1 → memorius-0.4.2}/tests/test_encoding_regression.py +0 -0
- {memorius-0.4.1 → memorius-0.4.2}/tests/test_features.py +0 -0
- {memorius-0.4.1 → memorius-0.4.2}/tests/test_graph.py +0 -0
- {memorius-0.4.1 → memorius-0.4.2}/tests/test_polish_regressions.py +0 -0
- {memorius-0.4.1 → memorius-0.4.2}/tests/test_temporal.py +0 -0
- {memorius-0.4.1 → memorius-0.4.2}/tests/test_validation.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: memorius
|
|
3
|
-
Version: 0.4.
|
|
3
|
+
Version: 0.4.2
|
|
4
4
|
Summary: Self-contained memory vault for any AI agent — vector search, session diaries, and agent-agnostic hooks.
|
|
5
5
|
Author-email: dimonapatrick243 <dimona.patrick@gmail.com>
|
|
6
6
|
License-Expression: MIT
|
|
@@ -16,6 +16,7 @@ Usage:
|
|
|
16
16
|
memorius obsidian list List notes in an Obsidian vault
|
|
17
17
|
memorius obsidian import Import Obsidian notes as memories
|
|
18
18
|
memorius obsidian export Export memories as Obsidian notes
|
|
19
|
+
memorius web <query> Search the internet (web fallback)
|
|
19
20
|
"""
|
|
20
21
|
|
|
21
22
|
from __future__ import annotations
|
|
@@ -29,6 +30,7 @@ from pathlib import Path
|
|
|
29
30
|
from memorius import __version__
|
|
30
31
|
from memorius.config import load_config, DEFAULT_CONFIG_DIR, DEFAULT_CONFIG_PATH
|
|
31
32
|
from memorius.vault import VaultEngine
|
|
33
|
+
from memorius.web_search import web_fallback_enabled, should_fallback
|
|
32
34
|
|
|
33
35
|
logging.basicConfig(
|
|
34
36
|
level=logging.WARNING,
|
|
@@ -82,6 +84,7 @@ def main():
|
|
|
82
84
|
search_p.add_argument("--n", type=int, default=10, help="Number of results")
|
|
83
85
|
search_p.add_argument("--vault", default=None, help="Filter by vault")
|
|
84
86
|
search_p.add_argument("--shelf", default=None, help="Filter by shelf")
|
|
87
|
+
search_p.add_argument("--web", action="store_true", help="Fall back to web search if local recall is thin")
|
|
85
88
|
|
|
86
89
|
mine_p = subparsers.add_parser("mine", help="Mine memories from a transcript")
|
|
87
90
|
mine_p.add_argument("file", nargs="?", default=None, help="Transcript file path")
|
|
@@ -126,11 +129,18 @@ def main():
|
|
|
126
129
|
factcheck_p = subparsers.add_parser("factcheck", help="Fact-check against stored memories")
|
|
127
130
|
factcheck_p.add_argument("statement", nargs="?", default=None, help="Statement to verify")
|
|
128
131
|
factcheck_p.add_argument("--vault", default=None, help="Filter by vault")
|
|
132
|
+
factcheck_p.add_argument("--web", action="store_true", help="Cross-check uncertain claims against the web")
|
|
129
133
|
|
|
130
134
|
context_p = subparsers.add_parser("context", help="Get memory context for injection")
|
|
131
135
|
context_p.add_argument("query", nargs="?", default=None, help="Topic to search for")
|
|
132
136
|
context_p.add_argument("--vault", default=None, help="Filter by vault")
|
|
133
137
|
context_p.add_argument("--max", type=int, default=5, help="Max items")
|
|
138
|
+
context_p.add_argument("--web", action="store_true", help="Augment with web results if no local context")
|
|
139
|
+
|
|
140
|
+
web_p = subparsers.add_parser("web", help="Search the internet (web fallback)")
|
|
141
|
+
web_p.add_argument("query", nargs="?", default=None, help="Search query")
|
|
142
|
+
web_p.add_argument("--max", type=int, default=5, help="Max results")
|
|
143
|
+
web_p.add_argument("--provider", default=None, help="Provider override (duckduckgo|mock)")
|
|
134
144
|
|
|
135
145
|
profile_p = subparsers.add_parser("profile", help="Build session memory profile")
|
|
136
146
|
profile_p.add_argument("session_id", nargs="?", default=None, help="Session ID")
|
|
@@ -194,6 +204,7 @@ def main():
|
|
|
194
204
|
"extract": cmd_extract,
|
|
195
205
|
"factcheck": cmd_factcheck,
|
|
196
206
|
"context": cmd_context,
|
|
207
|
+
"web": cmd_web,
|
|
197
208
|
"profile": cmd_profile,
|
|
198
209
|
"stats": cmd_stats,
|
|
199
210
|
}
|
|
@@ -331,6 +342,24 @@ def cmd_search(engine, args, config):
|
|
|
331
342
|
print(" ...")
|
|
332
343
|
print()
|
|
333
344
|
|
|
345
|
+
# Web fallback — only if local recall is thin ("if needed").
|
|
346
|
+
if web_fallback_enabled(args, config) and should_fallback(len(results), config):
|
|
347
|
+
from memorius.web_search import get_web_provider
|
|
348
|
+
provider = get_web_provider(config)
|
|
349
|
+
if provider:
|
|
350
|
+
web = provider.search(
|
|
351
|
+
query,
|
|
352
|
+
max_results=config.get("retrieval", {}).get("web_max_results", 5),
|
|
353
|
+
)
|
|
354
|
+
print("Web results (from internet):")
|
|
355
|
+
if not web:
|
|
356
|
+
print(" (no web results)")
|
|
357
|
+
for r in web:
|
|
358
|
+
print(f" - {r.title}")
|
|
359
|
+
print(f" {r.url}")
|
|
360
|
+
if r.snippet:
|
|
361
|
+
print(f" {r.snippet[:160]}")
|
|
362
|
+
|
|
334
363
|
|
|
335
364
|
def cmd_mine(engine, args, config):
|
|
336
365
|
"""Mine memories from a transcript."""
|
|
@@ -503,6 +532,47 @@ def cmd_factcheck(engine, args, config):
|
|
|
503
532
|
for m in result.contradicting_memories:
|
|
504
533
|
print(f" - [{m['vault']}/{m['shelf']}] {m['content'][:100]}")
|
|
505
534
|
|
|
535
|
+
# Web cross-check when the vault can't settle the claim.
|
|
536
|
+
if result.verdict in ("uncertain", "no_match") and web_fallback_enabled(args, config):
|
|
537
|
+
from memorius.web_search import get_web_provider
|
|
538
|
+
provider = get_web_provider(config)
|
|
539
|
+
if provider:
|
|
540
|
+
web = provider.search(
|
|
541
|
+
result.statement,
|
|
542
|
+
max_results=config.get("retrieval", {}).get("web_max_results", 5),
|
|
543
|
+
)
|
|
544
|
+
if web:
|
|
545
|
+
print(" Web cross-check:")
|
|
546
|
+
for r in web[:3]:
|
|
547
|
+
print(f" - {r.title}: {r.url}")
|
|
548
|
+
|
|
549
|
+
|
|
550
|
+
def cmd_web(engine, args, config):
|
|
551
|
+
"""Search the internet (web fallback primitive)."""
|
|
552
|
+
from memorius.web_search import get_web_provider
|
|
553
|
+
|
|
554
|
+
query = args.query
|
|
555
|
+
if not query:
|
|
556
|
+
query = sys.stdin.read().strip()
|
|
557
|
+
if not query:
|
|
558
|
+
print("Error: query required (pass as argument or pipe to stdin)")
|
|
559
|
+
return
|
|
560
|
+
|
|
561
|
+
provider = get_web_provider(config, provider=args.provider)
|
|
562
|
+
if not provider:
|
|
563
|
+
print("Error: no web provider available")
|
|
564
|
+
return
|
|
565
|
+
|
|
566
|
+
results = provider.search(query, max_results=args.max)
|
|
567
|
+
print(f'Web search: "{query}"')
|
|
568
|
+
print(f"Results: {len(results)}")
|
|
569
|
+
print()
|
|
570
|
+
for i, r in enumerate(results, 1):
|
|
571
|
+
print(f"{i}. {r.title}")
|
|
572
|
+
print(f" {r.url}")
|
|
573
|
+
if r.snippet:
|
|
574
|
+
print(f" {r.snippet[:200]}")
|
|
575
|
+
|
|
506
576
|
|
|
507
577
|
def cmd_context(engine, args, config):
|
|
508
578
|
"""Get memory context for injection."""
|
|
@@ -516,6 +586,23 @@ def cmd_context(engine, args, config):
|
|
|
516
586
|
context = engine.get_context(query, vault=args.vault, max_items=args.max)
|
|
517
587
|
if context:
|
|
518
588
|
print(context)
|
|
589
|
+
elif web_fallback_enabled(args, config):
|
|
590
|
+
from memorius.web_search import get_web_provider
|
|
591
|
+
provider = get_web_provider(config)
|
|
592
|
+
if provider:
|
|
593
|
+
web = provider.search(
|
|
594
|
+
query,
|
|
595
|
+
max_results=config.get("retrieval", {}).get("web_max_results", 5),
|
|
596
|
+
)
|
|
597
|
+
if web:
|
|
598
|
+
print("No matching memories. Web results (from internet):")
|
|
599
|
+
for r in web:
|
|
600
|
+
print(f"- {r.title}")
|
|
601
|
+
print(f" {r.url}")
|
|
602
|
+
if r.snippet:
|
|
603
|
+
print(f" {r.snippet[:160]}")
|
|
604
|
+
else:
|
|
605
|
+
print("No relevant memories or web results found.")
|
|
519
606
|
else:
|
|
520
607
|
print("No relevant memories found.")
|
|
521
608
|
|
|
@@ -37,6 +37,12 @@ vault:
|
|
|
37
37
|
hooks:
|
|
38
38
|
enabled: true
|
|
39
39
|
config: ~/.memorius/hooks.yaml
|
|
40
|
+
|
|
41
|
+
retrieval:
|
|
42
|
+
web_fallback: false # opt-in: augment thin local recall with web (local-first / privacy)
|
|
43
|
+
web_provider: duckduckgo # duckduckgo (keyless) | mock (tests)
|
|
44
|
+
web_min_results: 1 # if ZERO local hits, fall back to web
|
|
45
|
+
web_max_results: 5 # max web results to return
|
|
40
46
|
"""
|
|
41
47
|
|
|
42
48
|
|
|
@@ -98,6 +104,12 @@ def _ensure_defaults(config: dict[str, Any]):
|
|
|
98
104
|
vault_cfg.setdefault("default", "main")
|
|
99
105
|
vault_cfg.setdefault("max_note_size", 1000)
|
|
100
106
|
|
|
107
|
+
retrieval = config.setdefault("retrieval", {})
|
|
108
|
+
retrieval.setdefault("web_fallback", False)
|
|
109
|
+
retrieval.setdefault("web_provider", "duckduckgo")
|
|
110
|
+
retrieval.setdefault("web_min_results", 1)
|
|
111
|
+
retrieval.setdefault("web_max_results", 5)
|
|
112
|
+
|
|
101
113
|
hooks = config.setdefault("hooks", {})
|
|
102
114
|
hooks.setdefault("enabled", True)
|
|
103
115
|
hooks.setdefault("config", "~/.memorius/hooks.yaml")
|
|
@@ -112,6 +124,8 @@ def _apply_env_overrides(config: dict[str, Any]):
|
|
|
112
124
|
"MEMORIUS_REST_PORT": ("server", "rest_port"),
|
|
113
125
|
"MEMORIUS_HOST": ("server", "host"),
|
|
114
126
|
"MEMORIUS_DEFAULT_VAULT": ("vault", "default"),
|
|
127
|
+
"MEMORIUS_WEB_FALLBACK": ("retrieval", "web_fallback"),
|
|
128
|
+
"MEMORIUS_WEB_PROVIDER": ("retrieval", "web_provider"),
|
|
115
129
|
}
|
|
116
130
|
for env_key, (section, key) in overrides.items():
|
|
117
131
|
val = os.environ.get(env_key)
|
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
"""Web search fallback for memorius.
|
|
2
|
+
|
|
3
|
+
When local recall is thin, optionally augment retrieval with live web
|
|
4
|
+
results. Local-first and opt-in: web fallback is OFF by default and
|
|
5
|
+
must be enabled via config (``retrieval.web_fallback``) or the ``--web``
|
|
6
|
+
flag — memorius never phones home unless you ask it to.
|
|
7
|
+
|
|
8
|
+
2026-aligned design (hybrid / grounded retrieval):
|
|
9
|
+
- Web results are returned as *cited* ``WebResult`` objects (title/url/snippet).
|
|
10
|
+
- The default provider (DuckDuckGo lite) needs no API key and uses
|
|
11
|
+
only the stdlib, so it works out-of-the-box without secrets.
|
|
12
|
+
- A ``MockProvider`` lets the full path be tested without network.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import html
|
|
18
|
+
import logging
|
|
19
|
+
import urllib.parse
|
|
20
|
+
import urllib.request
|
|
21
|
+
from abc import ABC, abstractmethod
|
|
22
|
+
from dataclasses import dataclass
|
|
23
|
+
from typing import Optional
|
|
24
|
+
|
|
25
|
+
logger = logging.getLogger("memorius.web")
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@dataclass
|
|
29
|
+
class WebResult:
|
|
30
|
+
"""A single cited web search result."""
|
|
31
|
+
|
|
32
|
+
title: str
|
|
33
|
+
url: str
|
|
34
|
+
snippet: str = ""
|
|
35
|
+
|
|
36
|
+
def __str__(self) -> str:
|
|
37
|
+
snippet = self.snippet[:160]
|
|
38
|
+
return f"- {self.title}\n {self.url}\n {snippet}"
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class WebSearchProvider(ABC):
|
|
42
|
+
"""Pluggable web search backend."""
|
|
43
|
+
|
|
44
|
+
name = "base"
|
|
45
|
+
|
|
46
|
+
@abstractmethod
|
|
47
|
+
def search(self, query: str, max_results: int = 5) -> list[WebResult]:
|
|
48
|
+
"""Return cited web results for ``query``."""
|
|
49
|
+
raise NotImplementedError
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
class DuckDuckGoProvider(WebSearchProvider):
|
|
53
|
+
"""Keyless DuckDuckGo *lite* scraper (stdlib only)."""
|
|
54
|
+
|
|
55
|
+
name = "duckduckgo"
|
|
56
|
+
_ENDPOINT = "https://lite.duckduckgo.com/lite/"
|
|
57
|
+
|
|
58
|
+
def __init__(
|
|
59
|
+
self,
|
|
60
|
+
timeout: float = 10.0,
|
|
61
|
+
user_agent: str = "memorius/0.4.2",
|
|
62
|
+
):
|
|
63
|
+
self.timeout = timeout
|
|
64
|
+
self.user_agent = user_agent
|
|
65
|
+
|
|
66
|
+
def search(self, query: str, max_results: int = 5) -> list[WebResult]:
|
|
67
|
+
data = urllib.parse.urlencode({"q": query}).encode()
|
|
68
|
+
req = urllib.request.Request(
|
|
69
|
+
self._ENDPOINT,
|
|
70
|
+
data=data,
|
|
71
|
+
headers={
|
|
72
|
+
"User-Agent": self.user_agent,
|
|
73
|
+
"Content-Type": "application/x-www-form-urlencoded",
|
|
74
|
+
},
|
|
75
|
+
)
|
|
76
|
+
try:
|
|
77
|
+
with urllib.request.urlopen(req, timeout=self.timeout) as resp:
|
|
78
|
+
raw = resp.read().decode("utf-8", "replace")
|
|
79
|
+
except Exception as e: # network/timeout — never crash the CLI
|
|
80
|
+
logger.warning("DuckDuckGo search failed: %s", e)
|
|
81
|
+
return []
|
|
82
|
+
return _parse_ddg_lite(raw, max_results)
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
class MockProvider(WebSearchProvider):
|
|
86
|
+
"""Test double: returns canned results from a ``query -> [WebResult]`` map."""
|
|
87
|
+
|
|
88
|
+
name = "mock"
|
|
89
|
+
|
|
90
|
+
def __init__(self, results: Optional[dict[str, list[WebResult]]] = None):
|
|
91
|
+
self.results = results or {}
|
|
92
|
+
|
|
93
|
+
def search(self, query: str, max_results: int = 5) -> list[WebResult]:
|
|
94
|
+
return self.results.get(query, [])[:max_results]
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def _parse_ddg_lite(html_text: str, max_results: int) -> list[WebResult]:
|
|
98
|
+
"""Extract result links + snippets from DuckDuckGo lite HTML.
|
|
99
|
+
|
|
100
|
+
lite emits a table where each row has a ``<a class="result-link">``
|
|
101
|
+
title followed by a ``<td class="result-snippet">`` cell. We pair
|
|
102
|
+
link anchors with the snippet cell at the same index.
|
|
103
|
+
"""
|
|
104
|
+
from html.parser import HTMLParser
|
|
105
|
+
|
|
106
|
+
class _Scraper(HTMLParser):
|
|
107
|
+
def __init__(self):
|
|
108
|
+
super().__init__()
|
|
109
|
+
self.links: list[tuple[str, str]] = []
|
|
110
|
+
self.snippets: list[str] = []
|
|
111
|
+
self._in_link = False
|
|
112
|
+
self._in_snippet = False
|
|
113
|
+
self._cur_text: list[str] = []
|
|
114
|
+
self._cur_href = ""
|
|
115
|
+
self._cur_snip: list[str] = []
|
|
116
|
+
|
|
117
|
+
def handle_starttag(self, tag, attrs):
|
|
118
|
+
d = dict(attrs)
|
|
119
|
+
cls = d.get("class", "")
|
|
120
|
+
if tag == "a" and "result-link" in cls:
|
|
121
|
+
self._in_link = True
|
|
122
|
+
self._cur_text = []
|
|
123
|
+
self._cur_href = d.get("href", "")
|
|
124
|
+
elif tag == "td" and "result-snippet" in cls:
|
|
125
|
+
self._in_snippet = True
|
|
126
|
+
self._cur_snip = []
|
|
127
|
+
|
|
128
|
+
def handle_endtag(self, tag):
|
|
129
|
+
if tag == "a" and self._in_link:
|
|
130
|
+
self._in_link = False
|
|
131
|
+
title = "".join(self._cur_text).strip()
|
|
132
|
+
if title and self._cur_href:
|
|
133
|
+
self.links.append((title, self._cur_href))
|
|
134
|
+
self._cur_text = []
|
|
135
|
+
elif tag == "td" and self._in_snippet:
|
|
136
|
+
self._in_snippet = False
|
|
137
|
+
self.snippets.append("".join(self._cur_snip).strip())
|
|
138
|
+
self._cur_snip = []
|
|
139
|
+
|
|
140
|
+
def handle_data(self, data):
|
|
141
|
+
if self._in_link:
|
|
142
|
+
self._cur_text.append(data)
|
|
143
|
+
elif self._in_snippet:
|
|
144
|
+
self._cur_snip.append(data)
|
|
145
|
+
|
|
146
|
+
parser = _Scraper()
|
|
147
|
+
parser.feed(html_text)
|
|
148
|
+
out: list[WebResult] = []
|
|
149
|
+
for i, (title, url) in enumerate(parser.links[:max_results]):
|
|
150
|
+
snippet = html.unescape(parser.snippets[i]) if i < len(parser.snippets) else ""
|
|
151
|
+
out.append(
|
|
152
|
+
WebResult(
|
|
153
|
+
title=html.unescape(title),
|
|
154
|
+
url=url,
|
|
155
|
+
snippet=snippet,
|
|
156
|
+
)
|
|
157
|
+
)
|
|
158
|
+
return out
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def get_web_provider(
|
|
162
|
+
config: dict, provider: Optional[str] = None
|
|
163
|
+
) -> Optional[WebSearchProvider]:
|
|
164
|
+
"""Build the configured web provider, or ``None`` if unavailable."""
|
|
165
|
+
retrieval = config.get("retrieval", {}) if isinstance(config, dict) else {}
|
|
166
|
+
name = (provider or retrieval.get("web_provider", "duckduckgo")).lower()
|
|
167
|
+
if name == "mock":
|
|
168
|
+
return MockProvider()
|
|
169
|
+
if name == "duckduckgo":
|
|
170
|
+
return DuckDuckGoProvider()
|
|
171
|
+
logger.warning("Unknown web_provider %r; using duckduckgo", name)
|
|
172
|
+
return DuckDuckGoProvider()
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def web_fallback_enabled(args, config) -> bool:
|
|
176
|
+
"""True if web fallback should be considered (``--web`` flag or config)."""
|
|
177
|
+
if getattr(args, "web", False):
|
|
178
|
+
return True
|
|
179
|
+
retrieval = config.get("retrieval", {}) if isinstance(config, dict) else {}
|
|
180
|
+
return bool(retrieval.get("web_fallback", False))
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
def should_fallback(local_count: int, config: dict) -> bool:
|
|
184
|
+
"""Decide whether local recall is too thin to skip the web.
|
|
185
|
+
|
|
186
|
+
Triggers when there are fewer local hits than ``web_min_results`` —
|
|
187
|
+
i.e. "search the internet *if needed*".
|
|
188
|
+
"""
|
|
189
|
+
retrieval = config.get("retrieval", {}) if isinstance(config, dict) else {}
|
|
190
|
+
min_results = retrieval.get("web_min_results", 1)
|
|
191
|
+
return local_count < min_results
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: memorius
|
|
3
|
-
Version: 0.4.
|
|
3
|
+
Version: 0.4.2
|
|
4
4
|
Summary: Self-contained memory vault for any AI agent — vector search, session diaries, and agent-agnostic hooks.
|
|
5
5
|
Author-email: dimonapatrick243 <dimona.patrick@gmail.com>
|
|
6
6
|
License-Expression: MIT
|
|
@@ -24,6 +24,7 @@ memorius/utils.py
|
|
|
24
24
|
memorius/validation.py
|
|
25
25
|
memorius/vault.py
|
|
26
26
|
memorius/vector_store.py
|
|
27
|
+
memorius/web_search.py
|
|
27
28
|
memorius.egg-info/PKG-INFO
|
|
28
29
|
memorius.egg-info/SOURCES.txt
|
|
29
30
|
memorius.egg-info/dependency_links.txt
|
|
@@ -64,4 +65,5 @@ tests/test_graph.py
|
|
|
64
65
|
tests/test_integration.py
|
|
65
66
|
tests/test_polish_regressions.py
|
|
66
67
|
tests/test_temporal.py
|
|
67
|
-
tests/test_validation.py
|
|
68
|
+
tests/test_validation.py
|
|
69
|
+
tests/test_web_search.py
|
|
@@ -6,20 +6,37 @@ import shutil
|
|
|
6
6
|
import subprocess
|
|
7
7
|
import sys
|
|
8
8
|
import tempfile
|
|
9
|
-
import threading
|
|
10
9
|
import time
|
|
11
10
|
from pathlib import Path
|
|
12
11
|
|
|
13
12
|
import pytest
|
|
14
13
|
|
|
15
14
|
|
|
15
|
+
def _safe_rmtree(path):
|
|
16
|
+
"""rmtree tolerant of Windows file locks.
|
|
17
|
+
|
|
18
|
+
On Windows, chroma keeps its index/data .bin files memory-mapped
|
|
19
|
+
open, so a naive cleanup races with the OS lock and raises
|
|
20
|
+
PermissionError (WinError 32). Retry briefly to let the lock
|
|
21
|
+
release, then force-ignore any stragglers (temp dir — safe to drop).
|
|
22
|
+
"""
|
|
23
|
+
for _ in range(5):
|
|
24
|
+
try:
|
|
25
|
+
shutil.rmtree(path)
|
|
26
|
+
return
|
|
27
|
+
except PermissionError:
|
|
28
|
+
time.sleep(0.3)
|
|
29
|
+
shutil.rmtree(path, ignore_errors=True)
|
|
30
|
+
|
|
31
|
+
|
|
16
32
|
@pytest.fixture
|
|
17
33
|
def isolated_env():
|
|
18
34
|
"""Set up a temporary memorius environment."""
|
|
19
35
|
old_home = os.environ.get("HOME", "")
|
|
20
36
|
old_storage = os.environ.pop("MEMORIUS_STORAGE_PATH", None)
|
|
21
37
|
|
|
22
|
-
|
|
38
|
+
tmpdir = tempfile.mkdtemp()
|
|
39
|
+
try:
|
|
23
40
|
# Sandbox HOME so memorius init writes to ~/.memorius inside tmpdir
|
|
24
41
|
fake_home = Path(tmpdir) / "home"
|
|
25
42
|
fake_home.mkdir()
|
|
@@ -27,13 +44,14 @@ def isolated_env():
|
|
|
27
44
|
os.environ["MEMORIUS_STORAGE_PATH"] = str(Path(tmpdir) / "data")
|
|
28
45
|
|
|
29
46
|
yield Path(tmpdir)
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
47
|
+
finally:
|
|
48
|
+
# Restore env
|
|
49
|
+
os.environ["HOME"] = old_home
|
|
50
|
+
if old_storage:
|
|
51
|
+
os.environ["MEMORIUS_STORAGE_PATH"] = old_storage
|
|
52
|
+
else:
|
|
53
|
+
os.environ.pop("MEMORIUS_STORAGE_PATH", None)
|
|
54
|
+
_safe_rmtree(tmpdir)
|
|
37
55
|
|
|
38
56
|
|
|
39
57
|
def run_memorius(*args, env=None, input_text=None, timeout=120):
|
|
@@ -0,0 +1,213 @@
|
|
|
1
|
+
"""Tests for the hybrid retrieval / web-fallback feature.
|
|
2
|
+
|
|
3
|
+
All web access is faked via ``MockProvider`` — no network required.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
import argparse
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
from memorius.web_search import (
|
|
10
|
+
DuckDuckGoProvider,
|
|
11
|
+
MockProvider,
|
|
12
|
+
WebResult,
|
|
13
|
+
_parse_ddg_lite,
|
|
14
|
+
get_web_provider,
|
|
15
|
+
should_fallback,
|
|
16
|
+
web_fallback_enabled,
|
|
17
|
+
)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
# Sample DuckDuckGo *lite* HTML (shape only) for parser coverage.
|
|
21
|
+
_DDG_LITE_HTML = """
|
|
22
|
+
<html><body>
|
|
23
|
+
<table>
|
|
24
|
+
<tr class="result-link-row">
|
|
25
|
+
<td class="result-link">
|
|
26
|
+
<a class="result-link" href="https://example.com/a">Example Article A</a>
|
|
27
|
+
</td>
|
|
28
|
+
<td class="result-snippet">A short snippet about topic A.</td>
|
|
29
|
+
</tr>
|
|
30
|
+
<tr class="result-link-row">
|
|
31
|
+
<td class="result-link">
|
|
32
|
+
<a class="result-link" href="https://example.com/b">Example Article B</a>
|
|
33
|
+
</td>
|
|
34
|
+
<td class="result-snippet">A short snippet about topic B.</td>
|
|
35
|
+
</tr>
|
|
36
|
+
</table>
|
|
37
|
+
</body></html>
|
|
38
|
+
"""
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _cfg(**overrides):
|
|
42
|
+
cfg = {
|
|
43
|
+
"retrieval": {
|
|
44
|
+
"web_fallback": False,
|
|
45
|
+
"web_provider": "duckduckgo",
|
|
46
|
+
"web_min_results": 1,
|
|
47
|
+
"web_max_results": 5,
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
for k, v in overrides.items():
|
|
51
|
+
cfg["retrieval"][k] = v
|
|
52
|
+
return cfg
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _args(web=False):
|
|
56
|
+
return argparse.Namespace(web=web)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
# ── Provider selection ────────────────────────────────────────────────────
|
|
60
|
+
def test_get_web_provider_default_is_duckduckgo():
|
|
61
|
+
p = get_web_provider(_cfg())
|
|
62
|
+
assert isinstance(p, DuckDuckGoProvider)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def test_get_web_provider_mock():
|
|
66
|
+
p = get_web_provider(_cfg(), provider="mock")
|
|
67
|
+
assert isinstance(p, MockProvider)
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def test_get_web_provider_unknown_falls_back_to_duckduckgo(caplog):
|
|
71
|
+
p = get_web_provider(_cfg(web_provider="bogus"))
|
|
72
|
+
assert isinstance(p, DuckDuckGoProvider)
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
# ── Fallback decision ─────────────────────────────────────────────────────
|
|
76
|
+
def test_should_fallback_true_when_local_empty():
|
|
77
|
+
assert should_fallback(0, _cfg()) is True
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def test_should_fallback_false_with_any_local_result():
|
|
81
|
+
# "if needed" = local found nothing; 1+ hit means no web fallback.
|
|
82
|
+
assert should_fallback(1, _cfg()) is False
|
|
83
|
+
assert should_fallback(5, _cfg()) is False
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def test_should_fallback_false_when_local_sufficient():
|
|
87
|
+
assert should_fallback(2, _cfg()) is False
|
|
88
|
+
assert should_fallback(5, _cfg()) is False
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def test_should_fallback_respects_min_results():
|
|
92
|
+
assert should_fallback(3, _cfg(web_min_results=4)) is True
|
|
93
|
+
assert should_fallback(4, _cfg(web_min_results=4)) is False
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
# ── Flag / config gating (local-first, opt-in) ──────────────────────────
|
|
97
|
+
def test_web_fallback_disabled_by_default():
|
|
98
|
+
assert web_fallback_enabled(_args(web=False), _cfg()) is False
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def test_web_fallback_enabled_via_flag():
|
|
102
|
+
assert web_fallback_enabled(_args(web=True), _cfg()) is True
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def test_web_fallback_enabled_via_config():
|
|
106
|
+
assert web_fallback_enabled(_args(web=False), _cfg(web_fallback=True)) is True
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
# ── DuckDuckGo lite parser ──────────────────────────────────────────────
|
|
110
|
+
def test_parse_ddg_lite_extracts_links_and_snippets():
|
|
111
|
+
provider = DuckDuckGoProvider()
|
|
112
|
+
results = _parse_ddg_lite(_DDG_LITE_HTML, 5)
|
|
113
|
+
assert len(results) == 2
|
|
114
|
+
assert results[0].title == "Example Article A"
|
|
115
|
+
assert results[0].url == "https://example.com/a"
|
|
116
|
+
assert "topic A" in results[0].snippet
|
|
117
|
+
assert results[1].url == "https://example.com/b"
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def test_parse_ddg_lite_respects_max_results():
|
|
121
|
+
provider = DuckDuckGoProvider()
|
|
122
|
+
results = _parse_ddg_lite(_DDG_LITE_HTML, 1)
|
|
123
|
+
assert len(results) == 1
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
# ── MockProvider ─────────────────────────────────────────────────────────
|
|
127
|
+
def test_mock_provider_returns_canned_results():
|
|
128
|
+
canned = {
|
|
129
|
+
"python 3.13": [
|
|
130
|
+
WebResult(title="Py 3.13 notes", url="https://docs/python/3.13"),
|
|
131
|
+
]
|
|
132
|
+
}
|
|
133
|
+
p = MockProvider(canned)
|
|
134
|
+
out = p.search("python 3.13")
|
|
135
|
+
assert len(out) == 1
|
|
136
|
+
assert out[0].url == "https://docs/python/3.13"
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def test_mock_provider_empty_for_unknown_query():
|
|
140
|
+
p = MockProvider({})
|
|
141
|
+
assert p.search("anything") == []
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
# ── CLI integration (no real vault / no network) ───────────────────────
|
|
145
|
+
def test_cmd_web_uses_mock_provider(monkeypatch, capsys):
|
|
146
|
+
from memorius.cli.main import cmd_web
|
|
147
|
+
|
|
148
|
+
canned = {"python 3.13": [
|
|
149
|
+
WebResult(title="Py 3.13", url="https://docs/python/3.13", snippet="New in 3.13"),
|
|
150
|
+
]}
|
|
151
|
+
monkeypatch.setattr(
|
|
152
|
+
"memorius.web_search.get_web_provider",
|
|
153
|
+
lambda config, provider=None: MockProvider(canned),
|
|
154
|
+
)
|
|
155
|
+
|
|
156
|
+
cmd_web(None, argparse.Namespace(query="python 3.13", max=5, provider="mock"), _cfg())
|
|
157
|
+
out = capsys.readouterr().out
|
|
158
|
+
assert "Py 3.13" in out
|
|
159
|
+
assert "https://docs/python/3.13" in out
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def test_cmd_search_web_fallback_when_local_empty(monkeypatch, capsys):
|
|
163
|
+
from memorius.cli.main import cmd_search
|
|
164
|
+
|
|
165
|
+
class FakeEngine:
|
|
166
|
+
def search(self, **kwargs):
|
|
167
|
+
return [] # thin local recall -> triggers fallback
|
|
168
|
+
|
|
169
|
+
canned = {"latest rust": [
|
|
170
|
+
WebResult(title="Rust 1.80", url="https://rust/1.80"),
|
|
171
|
+
]}
|
|
172
|
+
monkeypatch.setattr(
|
|
173
|
+
"memorius.web_search.get_web_provider",
|
|
174
|
+
lambda config, provider=None: MockProvider(canned),
|
|
175
|
+
)
|
|
176
|
+
|
|
177
|
+
cmd_search(
|
|
178
|
+
FakeEngine(),
|
|
179
|
+
argparse.Namespace(query="latest rust", n=10, vault=None, shelf=None, web=True),
|
|
180
|
+
_cfg(),
|
|
181
|
+
)
|
|
182
|
+
out = capsys.readouterr().out
|
|
183
|
+
assert "Web results (from internet):" in out
|
|
184
|
+
assert "Rust 1.80" in out
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def test_cmd_search_no_web_when_local_present(monkeypatch, capsys):
|
|
188
|
+
from memorius.cli.main import cmd_search
|
|
189
|
+
from memorius.models import Memory
|
|
190
|
+
|
|
191
|
+
class FakeEngine:
|
|
192
|
+
def search(self, **kwargs):
|
|
193
|
+
return [Memory(id="m1", vault="main", shelf="default",
|
|
194
|
+
folder="default", note="default",
|
|
195
|
+
content="Rust 1.80 released June 2024")]
|
|
196
|
+
|
|
197
|
+
# Web provider would return something, but fallback must NOT trigger
|
|
198
|
+
# because local recall is sufficient.
|
|
199
|
+
monkeypatch.setattr(
|
|
200
|
+
"memorius.web_search.get_web_provider",
|
|
201
|
+
lambda config, provider=None: MockProvider(
|
|
202
|
+
{"latest rust": [WebResult(title="SHOULD NOT APPEAR", url="x")]}
|
|
203
|
+
),
|
|
204
|
+
)
|
|
205
|
+
|
|
206
|
+
cmd_search(
|
|
207
|
+
FakeEngine(),
|
|
208
|
+
argparse.Namespace(query="latest rust", n=10, vault=None, shelf=None, web=True),
|
|
209
|
+
_cfg(),
|
|
210
|
+
)
|
|
211
|
+
out = capsys.readouterr().out
|
|
212
|
+
assert "SHOULD NOT APPEAR" not in out
|
|
213
|
+
assert "Web results" not in out
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|