memdate 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- memdate/__init__.py +78 -0
- memdate/adapters.py +174 -0
- memdate/capture.py +136 -0
- memdate/config.py +148 -0
- memdate/distill.py +142 -0
- memdate/drop.py +116 -0
- memdate/frontmatter.py +108 -0
- memdate/index.py +264 -0
- memdate-1.0.0.dist-info/METADATA +210 -0
- memdate-1.0.0.dist-info/RECORD +13 -0
- memdate-1.0.0.dist-info/WHEEL +5 -0
- memdate-1.0.0.dist-info/licenses/LICENSE +21 -0
- memdate-1.0.0.dist-info/top_level.txt +1 -0
memdate/drop.py
ADDED
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
"""DROP: the ingestion layer.
|
|
2
|
+
|
|
3
|
+
Files land in ``DROP/NEW/`` from anywhere — a phone note, an export, a
|
|
4
|
+
paste. ``sweep_new()`` routes each one: a leading ``domain: <name>``
|
|
5
|
+
line decides the destination; files with no routable domain, an unknown
|
|
6
|
+
domain, or interpretation markers go to ``DROP/QUARANTINE/`` instead of
|
|
7
|
+
being silently captured. Captured files move to ``DROP/PROCESSED/`` with
|
|
8
|
+
a timestamp prefix. Nothing is ever deleted by the sweep.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
from dataclasses import dataclass
|
|
14
|
+
from datetime import datetime
|
|
15
|
+
|
|
16
|
+
from .adapters import StorageAdapter, default_adapter
|
|
17
|
+
from .capture import CaptureError, InterpretationError, capture, scan_interpretation
|
|
18
|
+
from .config import ConfigError, MemoryConfig
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@dataclass
|
|
22
|
+
class SweepItem:
|
|
23
|
+
filename: str
|
|
24
|
+
outcome: str # "captured" | "quarantined"
|
|
25
|
+
domain: str = ""
|
|
26
|
+
reason: str = ""
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def ensure_layout(
|
|
30
|
+
config: MemoryConfig,
|
|
31
|
+
*,
|
|
32
|
+
adapter: StorageAdapter | None = None,
|
|
33
|
+
) -> None:
|
|
34
|
+
"""Create the root, DROP subdirs, and per-domain directories."""
|
|
35
|
+
store = adapter or default_adapter(config.root)
|
|
36
|
+
store.makedirs("")
|
|
37
|
+
for sub in config.drop_subdirs:
|
|
38
|
+
store.makedirs(f"DROP/{sub}")
|
|
39
|
+
for domain in config.domains:
|
|
40
|
+
store.makedirs(domain)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def sweep_new(
|
|
44
|
+
config: MemoryConfig,
|
|
45
|
+
*,
|
|
46
|
+
now: datetime | None = None,
|
|
47
|
+
adapter: StorageAdapter | None = None,
|
|
48
|
+
) -> list[SweepItem]:
|
|
49
|
+
"""Route every file in DROP/NEW/. Returns one item per file."""
|
|
50
|
+
store = adapter or default_adapter(config.root)
|
|
51
|
+
ensure_layout(config, adapter=store)
|
|
52
|
+
ts = (now or datetime.now().astimezone()).strftime("%Y%m%d-%H%M%S")
|
|
53
|
+
results: list[SweepItem] = []
|
|
54
|
+
|
|
55
|
+
for filename in store.list_files("DROP/NEW"):
|
|
56
|
+
src = f"DROP/NEW/{filename}"
|
|
57
|
+
text = store.read(src) or ""
|
|
58
|
+
domain, body = _split_domain_header(text)
|
|
59
|
+
|
|
60
|
+
if not domain:
|
|
61
|
+
_quarantine(store, src, filename, ts,
|
|
62
|
+
"no 'domain: <name>' header on the first line")
|
|
63
|
+
results.append(SweepItem(filename, "quarantined",
|
|
64
|
+
reason="missing domain header"))
|
|
65
|
+
continue
|
|
66
|
+
try:
|
|
67
|
+
name = config.require_domain(domain)
|
|
68
|
+
except ConfigError:
|
|
69
|
+
_quarantine(store, src, filename, ts,
|
|
70
|
+
f"unknown domain {domain!r}")
|
|
71
|
+
results.append(SweepItem(filename, "quarantined",
|
|
72
|
+
reason=f"unknown domain {domain!r}"))
|
|
73
|
+
continue
|
|
74
|
+
if not body.strip():
|
|
75
|
+
_quarantine(store, src, filename, ts, "empty body after header")
|
|
76
|
+
results.append(SweepItem(filename, "quarantined",
|
|
77
|
+
reason="empty body"))
|
|
78
|
+
continue
|
|
79
|
+
hits = scan_interpretation(body, config.interpretation_markers)
|
|
80
|
+
if hits:
|
|
81
|
+
_quarantine(store, src, filename, ts,
|
|
82
|
+
f"interpretation markers: {', '.join(hits)}")
|
|
83
|
+
results.append(SweepItem(filename, "quarantined",
|
|
84
|
+
domain=name,
|
|
85
|
+
reason="interpretation markers"))
|
|
86
|
+
continue
|
|
87
|
+
try:
|
|
88
|
+
result = capture(config, name, body,
|
|
89
|
+
source=f"DROP/NEW/{filename}",
|
|
90
|
+
now=now, adapter=store)
|
|
91
|
+
except (CaptureError, InterpretationError) as exc:
|
|
92
|
+
_quarantine(store, src, filename, ts, str(exc))
|
|
93
|
+
results.append(SweepItem(filename, "quarantined",
|
|
94
|
+
domain=name, reason=str(exc)))
|
|
95
|
+
continue
|
|
96
|
+
store.move(src, f"DROP/PROCESSED/{ts}-{filename}")
|
|
97
|
+
results.append(SweepItem(filename, "captured", domain=result.domain))
|
|
98
|
+
return results
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def _split_domain_header(text: str) -> tuple[str, str]:
|
|
102
|
+
"""Split an optional leading ``domain: <name>`` line from the body."""
|
|
103
|
+
lines = text.splitlines()
|
|
104
|
+
if not lines:
|
|
105
|
+
return "", ""
|
|
106
|
+
first = lines[0].strip()
|
|
107
|
+
if first.lower().startswith("domain:"):
|
|
108
|
+
return first.split(":", 1)[1].strip(), "\n".join(lines[1:])
|
|
109
|
+
return "", text
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def _quarantine(store: StorageAdapter, src: str, filename: str,
|
|
113
|
+
ts: str, reason: str) -> None:
|
|
114
|
+
store.write(f"DROP/QUARANTINE/{ts}-{filename}.reason.txt",
|
|
115
|
+
f"Quarantined {ts}: {reason}\n")
|
|
116
|
+
store.move(src, f"DROP/QUARANTINE/{ts}-{filename}")
|
memdate/frontmatter.py
ADDED
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
"""Frontmatter: parse, render, validate.
|
|
2
|
+
|
|
3
|
+
Derived artifacts (distilled.md, INDEX-cross-domain.md) carry provenance
|
|
4
|
+
frontmatter. raw.md deliberately does not — CAPTURE is an append-only
|
|
5
|
+
evidence log, and per-entry protocol metadata would pollute source
|
|
6
|
+
fidelity. The protocol stamp lives on the derived representation, never
|
|
7
|
+
on the raw record.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class FrontmatterError(ValueError):
|
|
14
|
+
"""Raised when frontmatter is missing or malformed."""
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def render(fields: dict[str, object]) -> str:
|
|
18
|
+
"""Render a frontmatter block from an ordered field mapping.
|
|
19
|
+
|
|
20
|
+
Lists render as one ``- item`` per line; dicts as ``key: value``
|
|
21
|
+
sub-lines; scalars inline. ``None`` values are skipped.
|
|
22
|
+
"""
|
|
23
|
+
lines = ["---"]
|
|
24
|
+
for key, value in fields.items():
|
|
25
|
+
if value is None:
|
|
26
|
+
continue
|
|
27
|
+
if isinstance(value, list):
|
|
28
|
+
lines.append(f"{key}:")
|
|
29
|
+
for item in value:
|
|
30
|
+
lines.append(f" - {item}")
|
|
31
|
+
elif isinstance(value, dict):
|
|
32
|
+
lines.append(f"{key}:")
|
|
33
|
+
for sub, subval in value.items():
|
|
34
|
+
lines.append(f" {sub}: {subval}")
|
|
35
|
+
else:
|
|
36
|
+
lines.append(f"{key}: {value}")
|
|
37
|
+
lines.append("---")
|
|
38
|
+
return "\n".join(lines) + "\n"
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def split(text: str) -> tuple[dict[str, object], str]:
|
|
42
|
+
"""Split ``(frontmatter dict, body)``. Raises if the block is absent."""
|
|
43
|
+
if not text.startswith("---"):
|
|
44
|
+
raise FrontmatterError("Document has no frontmatter block (must start with '---').")
|
|
45
|
+
lines = text.splitlines()
|
|
46
|
+
try:
|
|
47
|
+
end = lines.index("---", 1)
|
|
48
|
+
except ValueError:
|
|
49
|
+
raise FrontmatterError("Frontmatter block never closes (missing second '---').")
|
|
50
|
+
fields: dict[str, object] = {}
|
|
51
|
+
current: str | None = None
|
|
52
|
+
for lineno, line in enumerate(lines[1:end], 2):
|
|
53
|
+
stripped = line.strip()
|
|
54
|
+
if not stripped:
|
|
55
|
+
continue
|
|
56
|
+
if line[0] in (" ", "\t"):
|
|
57
|
+
if current is None:
|
|
58
|
+
raise FrontmatterError(f"Line {lineno}: indented line outside a list/dict.")
|
|
59
|
+
container = fields[current]
|
|
60
|
+
if stripped.startswith("- "):
|
|
61
|
+
if not isinstance(container, list):
|
|
62
|
+
raise FrontmatterError(f"Line {lineno}: list item under scalar key {current!r}.")
|
|
63
|
+
container.append(_scalar(stripped[2:].strip()))
|
|
64
|
+
elif ":" in stripped:
|
|
65
|
+
if isinstance(container, list):
|
|
66
|
+
if container:
|
|
67
|
+
raise FrontmatterError(
|
|
68
|
+
f"Line {lineno}: sub-key under list key {current!r}."
|
|
69
|
+
)
|
|
70
|
+
container = {}
|
|
71
|
+
fields[current] = container
|
|
72
|
+
if not isinstance(container, dict):
|
|
73
|
+
raise FrontmatterError(f"Line {lineno}: sub-key under non-dict key {current!r}.")
|
|
74
|
+
sub, _, subval = stripped.partition(":")
|
|
75
|
+
container[sub.strip()] = _scalar(subval.strip())
|
|
76
|
+
else:
|
|
77
|
+
raise FrontmatterError(f"Line {lineno}: cannot parse {line!r}.")
|
|
78
|
+
continue
|
|
79
|
+
if ":" not in stripped:
|
|
80
|
+
raise FrontmatterError(f"Line {lineno}: expected 'key: value'.")
|
|
81
|
+
key, _, value = stripped.partition(":")
|
|
82
|
+
key, value = key.strip(), value.strip()
|
|
83
|
+
if not key:
|
|
84
|
+
raise FrontmatterError(f"Line {lineno}: blank key.")
|
|
85
|
+
if value:
|
|
86
|
+
fields[key] = _scalar(value)
|
|
87
|
+
current = None
|
|
88
|
+
else:
|
|
89
|
+
# Could be a list or a dict; decide on first child line.
|
|
90
|
+
fields[key] = []
|
|
91
|
+
current = key
|
|
92
|
+
body = "\n".join(lines[end + 1:])
|
|
93
|
+
return fields, body
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def _scalar(value: str) -> object:
|
|
97
|
+
if len(value) >= 2 and value[0] == value[-1] and value[0] in ("'", '"'):
|
|
98
|
+
return value[1:-1]
|
|
99
|
+
return value
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def require(fields: dict[str, object], *keys: str, what: str = "document") -> None:
|
|
103
|
+
"""Assert required keys are present and non-blank."""
|
|
104
|
+
missing = [k for k in keys if not fields.get(k)]
|
|
105
|
+
if missing:
|
|
106
|
+
raise FrontmatterError(
|
|
107
|
+
f"{what} frontmatter missing required keys: {', '.join(missing)}."
|
|
108
|
+
)
|
memdate/index.py
ADDED
|
@@ -0,0 +1,264 @@
|
|
|
1
|
+
"""INDEX: regenerate the cross-domain connective tissue.
|
|
2
|
+
|
|
3
|
+
Runs after any full or multi-domain distill. A single-domain distill
|
|
4
|
+
skips the index with a recorded reason — the procedure is mechanical
|
|
5
|
+
and the skip rule is code, not advice.
|
|
6
|
+
|
|
7
|
+
Body-text conventions the index understands (documented in the README):
|
|
8
|
+
- A line starting with ``? `` inside distilled.md is an uncertainty flag.
|
|
9
|
+
- A line starting with ``! `` is an explicit raw<->distilled divergence note.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
from dataclasses import dataclass, field
|
|
15
|
+
from datetime import date
|
|
16
|
+
|
|
17
|
+
from . import frontmatter as fm
|
|
18
|
+
from .adapters import StorageAdapter, default_adapter
|
|
19
|
+
from .config import MemoryConfig
|
|
20
|
+
from .distill import ARTIFACT as DISTILL_ARTIFACT, read_distilled
|
|
21
|
+
|
|
22
|
+
ARTIFACT = "memdate-cross-domain-index"
|
|
23
|
+
PROTOCOL_VERSION = "memdate/1.0.0"
|
|
24
|
+
INDEX_PATH = "INDEX-cross-domain.md"
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class IndexError(ValueError):
|
|
28
|
+
"""Raised when the index cannot be built or verified."""
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@dataclass
|
|
32
|
+
class IndexResult:
|
|
33
|
+
path: str
|
|
34
|
+
skipped: bool
|
|
35
|
+
reason: str
|
|
36
|
+
cross_domain_entities: int = 0
|
|
37
|
+
open_questions: int = 0
|
|
38
|
+
hot_zones: int = 0
|
|
39
|
+
lifecycle: str = ""
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _uncertainty_flags(body: str) -> list[str]:
|
|
43
|
+
return [ln[2:].strip() for ln in body.splitlines() if ln.startswith("? ")]
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _divergence_notes(body: str) -> list[str]:
|
|
47
|
+
return [ln[2:].strip() for ln in body.splitlines() if ln.startswith("! ")]
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _chains(deps: dict[str, list[str]]) -> list[list[str]]:
|
|
51
|
+
"""Dependency paths of length >= 2 (the 'strongest chains' callout)."""
|
|
52
|
+
found: list[list[str]] = []
|
|
53
|
+
|
|
54
|
+
def walk(path: list[str]) -> None:
|
|
55
|
+
nxt = deps.get(path[-1], [])
|
|
56
|
+
for target in nxt:
|
|
57
|
+
if target in path:
|
|
58
|
+
continue
|
|
59
|
+
new_path = path + [target]
|
|
60
|
+
if len(new_path) >= 3:
|
|
61
|
+
found.append(new_path)
|
|
62
|
+
walk(new_path)
|
|
63
|
+
|
|
64
|
+
for start in deps:
|
|
65
|
+
walk([start])
|
|
66
|
+
# Deduplicate while keeping order.
|
|
67
|
+
seen: set[tuple[str, ...]] = set()
|
|
68
|
+
unique: list[list[str]] = []
|
|
69
|
+
for p in found:
|
|
70
|
+
key = tuple(p)
|
|
71
|
+
if key not in seen:
|
|
72
|
+
seen.add(key)
|
|
73
|
+
unique.append(p)
|
|
74
|
+
return unique
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def regenerate_index(
|
|
78
|
+
config: MemoryConfig,
|
|
79
|
+
touched: list[str],
|
|
80
|
+
*,
|
|
81
|
+
today: str | None = None,
|
|
82
|
+
adapter: StorageAdapter | None = None,
|
|
83
|
+
) -> IndexResult:
|
|
84
|
+
"""Rebuild INDEX-cross-domain.md from every domain's distilled.md.
|
|
85
|
+
|
|
86
|
+
``touched`` is the list of domains the distill pass updated. Fewer
|
|
87
|
+
than two distinct touched domains → the index is left alone and the
|
|
88
|
+
skip is recorded in the returned result.
|
|
89
|
+
"""
|
|
90
|
+
touched_names = sorted({config.require_domain(d) for d in touched})
|
|
91
|
+
store = adapter or default_adapter(config.root)
|
|
92
|
+
|
|
93
|
+
if len(touched_names) < 2:
|
|
94
|
+
return IndexResult(
|
|
95
|
+
path=INDEX_PATH,
|
|
96
|
+
skipped=True,
|
|
97
|
+
reason=(
|
|
98
|
+
f"Single-domain distill ({', '.join(touched_names) or 'none'}): "
|
|
99
|
+
f"index left untouched per the skip rule."
|
|
100
|
+
),
|
|
101
|
+
)
|
|
102
|
+
|
|
103
|
+
# 1. Collect.
|
|
104
|
+
distilled: dict[str, tuple[dict[str, object], str]] = {}
|
|
105
|
+
for domain in config.domains:
|
|
106
|
+
parsed = read_distilled(config, domain, adapter=store)
|
|
107
|
+
if parsed is not None:
|
|
108
|
+
distilled[domain] = parsed
|
|
109
|
+
if len(distilled) < 2:
|
|
110
|
+
raise IndexError(
|
|
111
|
+
f"Index needs at least two distilled domains; found "
|
|
112
|
+
f"{len(distilled)} ({', '.join(sorted(distilled)) or 'none'}). "
|
|
113
|
+
f"Distill more domains first."
|
|
114
|
+
)
|
|
115
|
+
|
|
116
|
+
# 2. Entities -> domains (cross-domain only).
|
|
117
|
+
entity_domains: dict[str, list[str]] = {}
|
|
118
|
+
entity_notes: dict[str, dict[str, str]] = {}
|
|
119
|
+
for domain, (fields, _body) in distilled.items():
|
|
120
|
+
for entity in _as_list(fields.get("entities")):
|
|
121
|
+
entity_domains.setdefault(entity, []).append(domain)
|
|
122
|
+
notes = fields.get("entity_notes")
|
|
123
|
+
if isinstance(notes, dict):
|
|
124
|
+
for entity, note in notes.items():
|
|
125
|
+
entity_notes.setdefault(str(entity), {})[domain] = str(note)
|
|
126
|
+
cross = {e: sorted(ds) for e, ds in entity_domains.items() if len(set(ds)) >= 2}
|
|
127
|
+
|
|
128
|
+
# 3. Dependencies.
|
|
129
|
+
deps: dict[str, list[str]] = {}
|
|
130
|
+
for domain, (fields, _body) in distilled.items():
|
|
131
|
+
informs = [d for d in _as_list(fields.get("informs")) if d in distilled]
|
|
132
|
+
if informs:
|
|
133
|
+
deps[domain] = sorted(set(informs))
|
|
134
|
+
chains = _chains(deps)
|
|
135
|
+
|
|
136
|
+
# 4. Open questions, prioritized by blast radius: questions sharing an
|
|
137
|
+
# entity with a cross-domain entity sort first.
|
|
138
|
+
cross_lower = {e.lower() for e in cross}
|
|
139
|
+
questions: list[tuple[str, str, bool]] = [] # (question, source, cross_domain)
|
|
140
|
+
for domain, (fields, _body) in distilled.items():
|
|
141
|
+
for q in _as_list(fields.get("open_questions")):
|
|
142
|
+
ql = q.lower()
|
|
143
|
+
hits_cross = any(e in ql for e in cross_lower)
|
|
144
|
+
questions.append((q, domain, hits_cross))
|
|
145
|
+
questions.sort(key=lambda item: (not item[2], item[1], item[0]))
|
|
146
|
+
|
|
147
|
+
# 5. Hot zones.
|
|
148
|
+
hot: list[str] = []
|
|
149
|
+
for domain, (_fields, body) in distilled.items():
|
|
150
|
+
for flag in _uncertainty_flags(body):
|
|
151
|
+
hot.append(f"[{domain}] uncertainty: {flag}")
|
|
152
|
+
for note in _divergence_notes(body):
|
|
153
|
+
hot.append(f"[{domain}] raw<->distilled divergence: {note}")
|
|
154
|
+
for entity, per_domain in entity_notes.items():
|
|
155
|
+
if entity in cross and len(set(per_domain.values())) > 1:
|
|
156
|
+
detail = "; ".join(f"{d}: {n}" for d, n in sorted(per_domain.items()))
|
|
157
|
+
hot.append(f"[tension] {entity}: differing status notes — {detail}")
|
|
158
|
+
|
|
159
|
+
# 6. Write.
|
|
160
|
+
day = today or date.today().isoformat()
|
|
161
|
+
lifecycle = "FINAL" if not questions else "ITERATIVE"
|
|
162
|
+
lines: list[str] = []
|
|
163
|
+
lines.append("# Cross-domain index")
|
|
164
|
+
lines.append("")
|
|
165
|
+
lines.append(f"_Regenerated {day} from {len(distilled)} distilled domains._")
|
|
166
|
+
lines.append("")
|
|
167
|
+
lines.append("## Entities -> Domains")
|
|
168
|
+
lines.append("")
|
|
169
|
+
if cross:
|
|
170
|
+
for entity in sorted(cross):
|
|
171
|
+
domains = cross[entity]
|
|
172
|
+
note = ""
|
|
173
|
+
per = entity_notes.get(entity, {})
|
|
174
|
+
if per and len(set(per.values())) == 1:
|
|
175
|
+
note = f" — {next(iter(per.values()))}"
|
|
176
|
+
lines.append(f"- **{entity}**: {', '.join(domains)}{note}")
|
|
177
|
+
else:
|
|
178
|
+
lines.append("- _No entity is referenced in more than one domain yet._")
|
|
179
|
+
lines.append("")
|
|
180
|
+
lines.append("## Dependencies")
|
|
181
|
+
lines.append("")
|
|
182
|
+
if deps:
|
|
183
|
+
for domain in sorted(deps):
|
|
184
|
+
lines.append(f"- {domain} informs: {', '.join(deps[domain])}")
|
|
185
|
+
if chains:
|
|
186
|
+
lines.append("")
|
|
187
|
+
lines.append("Strongest chains:")
|
|
188
|
+
for chain in chains:
|
|
189
|
+
lines.append(f"- {' -> '.join(chain)}")
|
|
190
|
+
else:
|
|
191
|
+
lines.append("- _No cross-domain dependencies declared yet._")
|
|
192
|
+
lines.append("")
|
|
193
|
+
lines.append("## Open Questions")
|
|
194
|
+
lines.append("")
|
|
195
|
+
if questions:
|
|
196
|
+
for q, source, is_cross in questions:
|
|
197
|
+
tag = " [cross-domain]" if is_cross else ""
|
|
198
|
+
lines.append(f"- {q} (from {source}){tag}")
|
|
199
|
+
else:
|
|
200
|
+
lines.append("- _None._")
|
|
201
|
+
lines.append("")
|
|
202
|
+
lines.append("## Hot Zones")
|
|
203
|
+
lines.append("")
|
|
204
|
+
if hot:
|
|
205
|
+
for item in hot:
|
|
206
|
+
lines.append(f"- {item}")
|
|
207
|
+
else:
|
|
208
|
+
lines.append("- _None flagged._")
|
|
209
|
+
lines.append("")
|
|
210
|
+
|
|
211
|
+
doc = fm.render({
|
|
212
|
+
"artifact": ARTIFACT,
|
|
213
|
+
"date": day,
|
|
214
|
+
"protocol": PROTOCOL_VERSION,
|
|
215
|
+
"lifecycle": lifecycle,
|
|
216
|
+
}) + "\n".join(lines)
|
|
217
|
+
store.write(INDEX_PATH, doc)
|
|
218
|
+
|
|
219
|
+
# 7. Verify: every indexed entity exists in >= 2 distilled files;
|
|
220
|
+
# every question traces to a distilled source. True by construction —
|
|
221
|
+
# assert it so a future refactor can't silently break the invariant.
|
|
222
|
+
for entity, domains in cross.items():
|
|
223
|
+
count = sum(
|
|
224
|
+
1 for d in domains
|
|
225
|
+
if entity in _as_list(distilled[d][0].get("entities"))
|
|
226
|
+
)
|
|
227
|
+
assert count >= 2, f"orphan entity in index: {entity}"
|
|
228
|
+
for _q, source, _c in questions:
|
|
229
|
+
assert source in distilled, f"orphan question source: {source}"
|
|
230
|
+
|
|
231
|
+
return IndexResult(
|
|
232
|
+
path=INDEX_PATH,
|
|
233
|
+
skipped=False,
|
|
234
|
+
reason=f"Regenerated from {len(distilled)} distilled domains.",
|
|
235
|
+
cross_domain_entities=len(cross),
|
|
236
|
+
open_questions=len(questions),
|
|
237
|
+
hot_zones=len(hot),
|
|
238
|
+
lifecycle=lifecycle,
|
|
239
|
+
)
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
def _as_list(value: object) -> list[str]:
|
|
243
|
+
if isinstance(value, list):
|
|
244
|
+
return [str(v) for v in value]
|
|
245
|
+
return []
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
def read_index(
|
|
249
|
+
config: MemoryConfig,
|
|
250
|
+
*,
|
|
251
|
+
adapter: StorageAdapter | None = None,
|
|
252
|
+
) -> tuple[dict[str, object], str] | None:
|
|
253
|
+
"""Return ``(frontmatter, body)`` for INDEX-cross-domain.md, or None."""
|
|
254
|
+
store = adapter or default_adapter(config.root)
|
|
255
|
+
text = store.read(INDEX_PATH)
|
|
256
|
+
if text is None:
|
|
257
|
+
return None
|
|
258
|
+
fields, body = fm.split(text)
|
|
259
|
+
fm.require(fields, "artifact", "date", "protocol", "lifecycle", what="index")
|
|
260
|
+
if fields.get("artifact") != ARTIFACT:
|
|
261
|
+
raise IndexError(
|
|
262
|
+
f"Index artifact is {fields.get('artifact')!r}, expected {ARTIFACT!r}."
|
|
263
|
+
)
|
|
264
|
+
return fields, body
|
|
@@ -0,0 +1,210 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: memdate
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: File-based memory system with a hard CAPTURE/DISTILL split: raw records are append-only evidence, distilled files are derived synthesis, and a cross-domain index keeps the connective tissue current
|
|
5
|
+
Author: Ryan Wesley Hitchcock
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/ohpersonperson/frontface
|
|
8
|
+
Keywords: memory,knowledge-management,journaling,capture,distillation,markdown
|
|
9
|
+
Classifier: Programming Language :: Python :: 3
|
|
10
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
11
|
+
Classifier: Operating System :: OS Independent
|
|
12
|
+
Requires-Python: >=3.10
|
|
13
|
+
Description-Content-Type: text/markdown
|
|
14
|
+
License-File: LICENSE
|
|
15
|
+
Dynamic: license-file
|
|
16
|
+
|
|
17
|
+
# memdate — file-based memory with a hard CAPTURE/DISTILL split
|
|
18
|
+
|
|
19
|
+
Plain Markdown memory for people or agents. One load-bearing invariant:
|
|
20
|
+
|
|
21
|
+
**CAPTURE preserves. DISTILL interprets. Never mix them.**
|
|
22
|
+
|
|
23
|
+
- **CAPTURE** appends timestamped raw records. No summarizing, no
|
|
24
|
+
synthesis, no editing yesterday's entries. Raw is the permanent record.
|
|
25
|
+
- **DISTILL** consolidates raw into `distilled.md` — and the library
|
|
26
|
+
checksums raw before and after, voiding the run if anything touched it.
|
|
27
|
+
- **INDEX** regenerates the cross-domain connective tissue
|
|
28
|
+
(entities→domains, dependencies, open questions, hot zones) after any
|
|
29
|
+
multi-domain distill. Derived, never authoritative over raw.
|
|
30
|
+
|
|
31
|
+
No prerequisites. You don't need to know anything about the project's
|
|
32
|
+
history. The mechanism is the whole product.
|
|
33
|
+
|
|
34
|
+
*(Lineage: front-facing conversion of the `memdate-v2` skill from Ryan's
|
|
35
|
+
skill-suite, MIT. What changed is documented in CHANGELOG.md — the
|
|
36
|
+
decoupling record.)*
|
|
37
|
+
|
|
38
|
+
## What it does
|
|
39
|
+
|
|
40
|
+
A memory store is a directory:
|
|
41
|
+
|
|
42
|
+
```
|
|
43
|
+
<root>/
|
|
44
|
+
├── INDEX-cross-domain.md ← regenerated after multi-domain distills
|
|
45
|
+
├── DROP/ ← ingestion: NEW / PROCESSED / PENDING / QUARANTINE
|
|
46
|
+
├── personal/
|
|
47
|
+
│ ├── raw.md ← append-only evidence log
|
|
48
|
+
│ └── distilled.md ← derived synthesis + provenance frontmatter
|
|
49
|
+
└── work/
|
|
50
|
+
├── raw.md
|
|
51
|
+
└── distilled.md
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
The domain list is configuration, not identity. This package ships with
|
|
55
|
+
neutral defaults (`personal`, `work`, `projects`, `reference`, `misc`) —
|
|
56
|
+
define your own via constructor, JSON, or YAML. Storage is a
|
|
57
|
+
`StorageAdapter`: the local filesystem by default, Google Drive as an
|
|
58
|
+
optional seam you can implement (it fails loudly until you do, instead of
|
|
59
|
+
pretending to work).
|
|
60
|
+
|
|
61
|
+
## Why it works
|
|
62
|
+
|
|
63
|
+
Most memory systems rot because recording and interpreting happen in the
|
|
64
|
+
same motion: you write down what happened *and* what it means, and a
|
|
65
|
+
month later you can't tell which was which. The split is enforced in
|
|
66
|
+
code, not just in prose:
|
|
67
|
+
|
|
68
|
+
- **Capture isolation** — `capture()` only appends; it has no code path
|
|
69
|
+
that edits prior entries (tested: old content is always a strict prefix
|
|
70
|
+
of new content). Entry text is scanned for interpretation markers
|
|
71
|
+
("in summary", "overall,", … — configurable); a hit raises unless you
|
|
72
|
+
pass `force=True`, and then the bypass is stamped into the entry
|
|
73
|
+
header. The exception is visible, never silent.
|
|
74
|
+
- **Distill isolation** — `distill()` checksums `raw.md` before writing
|
|
75
|
+
`distilled.md` and re-checks after. If raw changed mid-run, the run is
|
|
76
|
+
void. A test adapter that sabotages raw mid-distill proves the check
|
|
77
|
+
fires.
|
|
78
|
+
- **Index discipline** — the four required sections and the 7-step
|
|
79
|
+
regeneration procedure are code. Single-domain distills skip the index
|
|
80
|
+
with a recorded reason. The verify pass asserts no orphan entities and
|
|
81
|
+
no sourceless questions.
|
|
82
|
+
- **Raw wins conflicts** — stated in the docs, enforced by the
|
|
83
|
+
architecture: nothing in the distill path can write to raw.
|
|
84
|
+
|
|
85
|
+
## Quickstart
|
|
86
|
+
|
|
87
|
+
Zero dependencies beyond Python 3.10+.
|
|
88
|
+
|
|
89
|
+
```bash
|
|
90
|
+
cd memdate
|
|
91
|
+
PYTHONPATH=src python3 -m unittest discover -s tests # 72 offline tests
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
```bash
|
|
95
|
+
export PYTHONPATH=src
|
|
96
|
+
python3 examples/run_local.py --root /tmp/memo init
|
|
97
|
+
python3 examples/run_local.py --root /tmp/memo capture \
|
|
98
|
+
--domain work --text "Acme Corp emailed: contract renews March 1." --source email
|
|
99
|
+
python3 examples/run_local.py --root /tmp/memo capture \
|
|
100
|
+
--domain work --text "In summary, the deal is safe."
|
|
101
|
+
# InterpretationError: Entry looks like interpretation, not capture —
|
|
102
|
+
# markers found: in summary. Rewrite as raw record, or pass force=True
|
|
103
|
+
# to record the bypass explicitly.
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
Distill from a file holding your consolidated synthesis:
|
|
107
|
+
|
|
108
|
+
```bash
|
|
109
|
+
python3 examples/run_local.py --root /tmp/memo distill --domain work \
|
|
110
|
+
--file /tmp/work-distilled.md \
|
|
111
|
+
--entities "Acme Corp,Priya" --informs personal \
|
|
112
|
+
--questions "Does the freeze delay the paperwork?"
|
|
113
|
+
python3 examples/run_local.py --root /tmp/memo index --touched work,personal
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
Or use it as a library:
|
|
117
|
+
|
|
118
|
+
```python
|
|
119
|
+
from memdate import (
|
|
120
|
+
MemoryConfig, default_adapter, ensure_layout,
|
|
121
|
+
capture, distill, regenerate_index,
|
|
122
|
+
)
|
|
123
|
+
|
|
124
|
+
cfg = MemoryConfig(root="/tmp/memo", domains=("garden", "car"))
|
|
125
|
+
store = default_adapter(cfg.root)
|
|
126
|
+
ensure_layout(cfg, adapter=store)
|
|
127
|
+
|
|
128
|
+
capture(cfg, "garden", "Planted tomatoes, north bed.", adapter=store)
|
|
129
|
+
distill(cfg, "garden", "North bed: tomatoes planted.",
|
|
130
|
+
entities=["north bed"], lifecycle="ITERATIVE", adapter=store)
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
Drop a file in `DROP/NEW/` with a leading `domain: <name>` line and
|
|
134
|
+
`sweep` routes it — captured to the right domain's raw, or quarantined
|
|
135
|
+
with a reason file if it's unroutable or reads like synthesis:
|
|
136
|
+
|
|
137
|
+
```bash
|
|
138
|
+
printf 'domain: work\n\nFixed the flaky test.\n' > /tmp/memo/DROP/NEW/note.md
|
|
139
|
+
python3 examples/run_local.py --root /tmp/memo sweep
|
|
140
|
+
```
|
|
141
|
+
|
|
142
|
+
## Inputs, outputs
|
|
143
|
+
|
|
144
|
+
- **Input (capture):** domain, text, optional source. Output: timestamped
|
|
145
|
+
entry appended to `<domain>/raw.md`.
|
|
146
|
+
- **Input (distill):** domain, consolidated body, optional entities /
|
|
147
|
+
entity notes / informs / open questions / lifecycle. Output:
|
|
148
|
+
`<domain>/distilled.md` with provenance frontmatter.
|
|
149
|
+
- **Input (index):** list of domains the distill touched. Output:
|
|
150
|
+
`INDEX-cross-domain.md`, or a recorded skip for single-domain distills.
|
|
151
|
+
|
|
152
|
+
## Distilled body conventions
|
|
153
|
+
|
|
154
|
+
Two line-prefixes the index understands:
|
|
155
|
+
|
|
156
|
+
- `? ` — uncertainty flag → collected into Hot Zones.
|
|
157
|
+
- `! ` — explicit raw↔distilled divergence note → collected into Hot Zones.
|
|
158
|
+
|
|
159
|
+
Entity note contradictions across domains (work says "reliable", personal
|
|
160
|
+
says "misses deadlines") also land in Hot Zones as `[tension]` entries.
|
|
161
|
+
|
|
162
|
+
## Evidence-taxonomy mapping
|
|
163
|
+
|
|
164
|
+
This package shares a vocabulary with the `adversarial-ifs` evidence
|
|
165
|
+
taxonomy (ten tiers). The mapping is explicit — memdate doesn't tag
|
|
166
|
+
evidence itself, but its artifacts sit at defined tiers:
|
|
167
|
+
|
|
168
|
+
| memdate artifact | IFS tier | Why |
|
|
169
|
+
|---|---|---|
|
|
170
|
+
| `raw.md` entry (logged event) | `OBSERVATION` | Reported/logged, not independently verified |
|
|
171
|
+
| `raw.md` entry (someone's assertion) | `CLAIM` | Explicit assertion by a source |
|
|
172
|
+
| `distilled.md` synthesis | `INFERENCE` | Conclusion drawn from raw evidence |
|
|
173
|
+
| `distilled.md` `? ` flags, index open questions | `UNKNOWN` | Flagged as not presently determinable |
|
|
174
|
+
| distiller-verified fact | `FACT` | Directly established ground truth |
|
|
175
|
+
| provisional model in a distillation | `HYPOTHESIS` | Needs stress-testing before use |
|
|
176
|
+
|
|
177
|
+
No code dependency between the packages — the table is the contract, so
|
|
178
|
+
neither taxonomy can silently fork.
|
|
179
|
+
|
|
180
|
+
## Worked example
|
|
181
|
+
|
|
182
|
+
`examples/example-1-two-domains.md` — a full CAPTURE → DISTILL → INDEX
|
|
183
|
+
cycle on two neutral domains, with the actual file contents shown at
|
|
184
|
+
each step.
|
|
185
|
+
|
|
186
|
+
*The field is constructed for teaching — it shows the mechanism's shape,
|
|
187
|
+
not real findings.*
|
|
188
|
+
|
|
189
|
+
## FAQ
|
|
190
|
+
|
|
191
|
+
**Where's Google Drive?** The original skill lived in a Drive folder.
|
|
192
|
+
This package defaults to the local filesystem; `GoogleDriveAdapter`
|
|
193
|
+
documents the seam and raises `DriveNotConfigured` with setup pointers
|
|
194
|
+
until you implement it. Drive is an adapter, not the root.
|
|
195
|
+
|
|
196
|
+
**Can a model do the distilling?** Yes — that's the intended shape. The
|
|
197
|
+
model supplies the consolidated body; the library stamps provenance and
|
|
198
|
+
enforces that raw wasn't touched. The library never calls a model itself.
|
|
199
|
+
|
|
200
|
+
**Does capture need the index?** No. Capture works from the first entry;
|
|
201
|
+
distill works per-domain; the index appears once two domains are
|
|
202
|
+
distilled.
|
|
203
|
+
|
|
204
|
+
**What about the memory interviewer?** The guided-capture interviewer
|
|
205
|
+
(`meminqu`) builds on this library's capture contract — it converts next
|
|
206
|
+
and shares this package's domain config.
|
|
207
|
+
|
|
208
|
+
## License
|
|
209
|
+
|
|
210
|
+
MIT. See LICENSE.
|