ihmt-memory 1.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ihmt/__init__.py +134 -0
- ihmt/api.py +166 -0
- ihmt/chunkers/__init__.py +55 -0
- ihmt/chunkers/base.py +297 -0
- ihmt/chunkers/code.py +572 -0
- ihmt/chunkers/generic.py +19 -0
- ihmt/chunkers/narrative.py +142 -0
- ihmt/chunkers/process.py +79 -0
- ihmt/chunkers/temporal.py +187 -0
- ihmt/config.py +161 -0
- ihmt/conflict_resolver.py +468 -0
- ihmt/detectors.py +327 -0
- ihmt/models.py +566 -0
- ihmt/project_index.py +440 -0
- ihmt/recursive_summarizer.py +342 -0
- ihmt/semantic_navigator.py +847 -0
- ihmt/storage.py +589 -0
- ihmt/summarizers.py +264 -0
- ihmt/templates/memory-instructions.md +37 -0
- ihmt/textutils.py +273 -0
- ihmt/universal_ingestor.py +373 -0
- ihmt_launcher.py +54 -0
- ihmt_memory-1.2.0.dist-info/METADATA +609 -0
- ihmt_memory-1.2.0.dist-info/RECORD +28 -0
- ihmt_memory-1.2.0.dist-info/WHEEL +4 -0
- ihmt_memory-1.2.0.dist-info/entry_points.txt +2 -0
- ihmt_memory-1.2.0.dist-info/licenses/LICENSE +21 -0
- mcp_server.py +701 -0
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
"""Paragraph- and scene-aware splitting for prose.
|
|
2
|
+
|
|
3
|
+
A paragraph is atomic. Scene and chapter markers are hard boundaries, so a
|
|
4
|
+
chunk never straddles a scene change. The only case that ever cuts inside a
|
|
5
|
+
paragraph is a single paragraph larger than ``max_tokens``, which is split at
|
|
6
|
+
sentence boundaries — never mid-sentence.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import re
|
|
12
|
+
from typing import List, Pattern
|
|
13
|
+
|
|
14
|
+
from ..textutils import truncate
|
|
15
|
+
from .base import Block, Chunker, join_lines, split_lines
|
|
16
|
+
|
|
17
|
+
#: Lines that open a new scene, chapter or section.
|
|
18
|
+
SCENE_MARKERS: List[Pattern[str]] = [
|
|
19
|
+
re.compile(r"^\s*(?:\*\s*){3,}\s*$"),
|
|
20
|
+
re.compile(r"^\s*[-–—=~_]{3,}\s*$"),
|
|
21
|
+
re.compile(r"^\s*#{1,3}\s+\S"),
|
|
22
|
+
re.compile(r"^\s*(?:chapter|cap[ií]tulo|part|parte|scene|escena|act|acto)\b.{0,60}$", re.IGNORECASE),
|
|
23
|
+
re.compile(r"^\s*(?:prologue|pr[oó]logo|epilogue|ep[ií]logo)\b.{0,40}$", re.IGNORECASE),
|
|
24
|
+
]
|
|
25
|
+
|
|
26
|
+
_SENTENCE_BOUNDARY = re.compile(r"(?<=[.!?…])[\"'”’)\]]?\s+(?=[\"'“¿¡(\[]?[A-ZÀ-Ü0-9])")
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def is_scene_marker(line: str) -> bool:
|
|
30
|
+
"""Whether ``line`` opens a new scene/chapter/section."""
|
|
31
|
+
stripped = line.strip()
|
|
32
|
+
if not stripped:
|
|
33
|
+
return False
|
|
34
|
+
return any(pattern.match(stripped) for pattern in SCENE_MARKERS)
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def split_sentences_exact(text: str) -> List[str]:
|
|
38
|
+
"""Split into sentences without losing a single character.
|
|
39
|
+
|
|
40
|
+
Unlike :func:`ihmt.textutils.split_sentences`, the separating whitespace is
|
|
41
|
+
kept attached to the preceding sentence so that ``"".join(result) == text``.
|
|
42
|
+
"""
|
|
43
|
+
cuts = [match.end() for match in _SENTENCE_BOUNDARY.finditer(text)]
|
|
44
|
+
if not cuts:
|
|
45
|
+
return [text]
|
|
46
|
+
parts: List[str] = []
|
|
47
|
+
previous = 0
|
|
48
|
+
for cut in cuts:
|
|
49
|
+
parts.append(text[previous:cut])
|
|
50
|
+
previous = cut
|
|
51
|
+
if previous < len(text):
|
|
52
|
+
parts.append(text[previous:])
|
|
53
|
+
return parts
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
class NarrativeChunker(Chunker):
|
|
57
|
+
"""Splitter for stories, essays, articles and any free-flowing prose."""
|
|
58
|
+
|
|
59
|
+
name = "narrative"
|
|
60
|
+
|
|
61
|
+
def __init__(self, config, *, detect_scenes: bool = True) -> None:
|
|
62
|
+
super().__init__(config)
|
|
63
|
+
self.detect_scenes = detect_scenes
|
|
64
|
+
|
|
65
|
+
def blocks(self, text: str, *, source: str = "") -> List[Block]:
|
|
66
|
+
"""Return one block per paragraph, with scenes as hard boundaries."""
|
|
67
|
+
lines = split_lines(text)
|
|
68
|
+
blocks: List[Block] = []
|
|
69
|
+
index = 0
|
|
70
|
+
total = len(lines)
|
|
71
|
+
|
|
72
|
+
while index < total:
|
|
73
|
+
start = index
|
|
74
|
+
while index < total and not lines[index].strip():
|
|
75
|
+
index += 1
|
|
76
|
+
if index >= total:
|
|
77
|
+
# Trailing whitespace: keep it so the text round-trips exactly.
|
|
78
|
+
if blocks:
|
|
79
|
+
blocks[-1].text += join_lines(lines, start + 1, total)
|
|
80
|
+
blocks[-1].end_line = total
|
|
81
|
+
break
|
|
82
|
+
|
|
83
|
+
content_start = index
|
|
84
|
+
scene = self.detect_scenes and is_scene_marker(lines[content_start])
|
|
85
|
+
while index < total and lines[index].strip():
|
|
86
|
+
index += 1
|
|
87
|
+
|
|
88
|
+
block = Block(
|
|
89
|
+
text=join_lines(lines, start + 1, index),
|
|
90
|
+
title=truncate(lines[content_start].strip(), 70),
|
|
91
|
+
start_line=start + 1,
|
|
92
|
+
end_line=index,
|
|
93
|
+
kind="scene" if scene else "paragraph",
|
|
94
|
+
tags=["scene"] if scene else [],
|
|
95
|
+
hard_boundary=scene,
|
|
96
|
+
)
|
|
97
|
+
if block.tokens > self.config.max_tokens:
|
|
98
|
+
blocks.extend(self._split_long_paragraph(block))
|
|
99
|
+
else:
|
|
100
|
+
blocks.append(block)
|
|
101
|
+
|
|
102
|
+
return blocks
|
|
103
|
+
|
|
104
|
+
def _split_long_paragraph(self, block: Block) -> List[Block]:
|
|
105
|
+
"""Break an over-long paragraph at sentence boundaries."""
|
|
106
|
+
sentences = split_sentences_exact(block.text)
|
|
107
|
+
if len(sentences) < 2:
|
|
108
|
+
return [block]
|
|
109
|
+
|
|
110
|
+
out: List[Block] = []
|
|
111
|
+
buffer: List[str] = []
|
|
112
|
+
line_cursor = block.start_line
|
|
113
|
+
part = 1
|
|
114
|
+
target = self.config.target_tokens
|
|
115
|
+
|
|
116
|
+
def flush() -> None:
|
|
117
|
+
nonlocal buffer, line_cursor, part
|
|
118
|
+
if not buffer:
|
|
119
|
+
return
|
|
120
|
+
body = "".join(buffer)
|
|
121
|
+
span = body.count("\n")
|
|
122
|
+
out.append(
|
|
123
|
+
Block(
|
|
124
|
+
text=body,
|
|
125
|
+
title=f"{block.title} (part {part})",
|
|
126
|
+
start_line=line_cursor,
|
|
127
|
+
end_line=line_cursor + span,
|
|
128
|
+
kind="paragraph-part",
|
|
129
|
+
tags=list(block.tags) + ["split:sentence"],
|
|
130
|
+
)
|
|
131
|
+
)
|
|
132
|
+
line_cursor += span
|
|
133
|
+
part += 1
|
|
134
|
+
buffer = []
|
|
135
|
+
|
|
136
|
+
for sentence in sentences:
|
|
137
|
+
candidate = "".join(buffer) + sentence
|
|
138
|
+
if buffer and len(candidate) // 4 > target:
|
|
139
|
+
flush()
|
|
140
|
+
buffer.append(sentence)
|
|
141
|
+
flush()
|
|
142
|
+
return out or [block]
|
ihmt/chunkers/process.py
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
"""Section-aware splitting for procedures: recipes, protocols, runbooks, manuals.
|
|
2
|
+
|
|
3
|
+
A numbered step never becomes an orphan: steps attach to the section heading
|
|
4
|
+
that introduces them, and an ingredient/material list is kept whole with its
|
|
5
|
+
heading. This keeps "how to do X" retrievable as one coherent unit.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import re
|
|
11
|
+
from typing import List
|
|
12
|
+
|
|
13
|
+
from ..textutils import truncate
|
|
14
|
+
from .base import Block, Chunker, join_lines, split_lines
|
|
15
|
+
|
|
16
|
+
_MD_HEADING = re.compile(r"^\s*(#{1,6})\s+(\S.*)$")
|
|
17
|
+
_LABEL_HEADING = re.compile(r"^\s*([A-ZÀ-Ü][^:\n]{2,58}):\s*$")
|
|
18
|
+
_STEP_HEADING = re.compile(
|
|
19
|
+
r"^\s*(?:paso|step|fase|phase|etapa|stage)\s*\d+\s*[:.\-)]?\s*(.*)$", re.IGNORECASE
|
|
20
|
+
)
|
|
21
|
+
_LIST_ITEM = re.compile(r"^\s*(?:[-*•]|\d{1,3}[.)])\s+\S")
|
|
22
|
+
_MATERIALS = re.compile(
|
|
23
|
+
r"^\s*(ingredientes|ingredients|materiales|materials|requisitos|requirements|"
|
|
24
|
+
r"utensilios|tools|equipment)\b",
|
|
25
|
+
re.IGNORECASE,
|
|
26
|
+
)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class ProcessChunker(Chunker):
|
|
30
|
+
"""Splitter for step-based and section-based documents."""
|
|
31
|
+
|
|
32
|
+
name = "process"
|
|
33
|
+
|
|
34
|
+
def blocks(self, text: str, *, source: str = "") -> List[Block]:
|
|
35
|
+
"""Return one block per section, list items bound to their heading."""
|
|
36
|
+
lines = split_lines(text)
|
|
37
|
+
starts: List[int] = []
|
|
38
|
+
hard: set[int] = set()
|
|
39
|
+
|
|
40
|
+
for index, line in enumerate(lines, start=1):
|
|
41
|
+
if not line.strip() or _LIST_ITEM.match(line):
|
|
42
|
+
# List items belong to the section above them.
|
|
43
|
+
continue
|
|
44
|
+
heading = _MD_HEADING.match(line)
|
|
45
|
+
if heading:
|
|
46
|
+
starts.append(index)
|
|
47
|
+
if len(heading.group(1)) <= 2:
|
|
48
|
+
hard.add(index)
|
|
49
|
+
continue
|
|
50
|
+
if _LABEL_HEADING.match(line) or _STEP_HEADING.match(line):
|
|
51
|
+
starts.append(index)
|
|
52
|
+
if _MATERIALS.match(line):
|
|
53
|
+
hard.add(index)
|
|
54
|
+
|
|
55
|
+
if not starts:
|
|
56
|
+
return self._fallback_blocks(text)
|
|
57
|
+
if starts[0] != 1:
|
|
58
|
+
starts.insert(0, 1)
|
|
59
|
+
|
|
60
|
+
blocks: List[Block] = []
|
|
61
|
+
for position, start in enumerate(starts):
|
|
62
|
+
end = starts[position + 1] - 1 if position + 1 < len(starts) else len(lines)
|
|
63
|
+
if end < start:
|
|
64
|
+
continue
|
|
65
|
+
heading_text = lines[start - 1].strip()
|
|
66
|
+
is_materials = bool(_MATERIALS.match(heading_text))
|
|
67
|
+
is_step = bool(_STEP_HEADING.match(heading_text))
|
|
68
|
+
blocks.append(
|
|
69
|
+
Block(
|
|
70
|
+
text=join_lines(lines, start, end),
|
|
71
|
+
title=truncate(heading_text.lstrip("# ").rstrip(":"), 70) or f"section {position + 1}",
|
|
72
|
+
start_line=start,
|
|
73
|
+
end_line=end,
|
|
74
|
+
kind="materials" if is_materials else ("step" if is_step else "section"),
|
|
75
|
+
tags=["process"] + (["materials"] if is_materials else []) + (["step"] if is_step else []),
|
|
76
|
+
hard_boundary=start in hard,
|
|
77
|
+
)
|
|
78
|
+
)
|
|
79
|
+
return blocks
|
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
"""Date- and session-aware splitting for journals, chats and clinical notes.
|
|
2
|
+
|
|
3
|
+
One dated entry is atomic, and a change of date is a hard boundary: a leaf never
|
|
4
|
+
mixes two encounters or two diary days. The date found in the text becomes the
|
|
5
|
+
leaf timestamp, which is what makes the conflict resolver's recency weighting
|
|
6
|
+
reflect *when something was true* rather than when it happened to be ingested.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import re
|
|
12
|
+
from typing import Dict, List, Optional, Pattern
|
|
13
|
+
|
|
14
|
+
from ..textutils import truncate
|
|
15
|
+
from .base import Block, Chunker, join_lines, split_lines
|
|
16
|
+
|
|
17
|
+
MONTHS: Dict[str, int] = {
|
|
18
|
+
"enero": 1, "febrero": 2, "marzo": 3, "abril": 4, "mayo": 5, "junio": 6,
|
|
19
|
+
"julio": 7, "agosto": 8, "septiembre": 9, "setiembre": 9, "octubre": 10,
|
|
20
|
+
"noviembre": 11, "diciembre": 12,
|
|
21
|
+
"january": 1, "february": 2, "march": 3, "april": 4, "may": 5, "june": 6,
|
|
22
|
+
"july": 7, "august": 8, "september": 9, "october": 10, "november": 11, "december": 12,
|
|
23
|
+
"jan": 1, "feb": 2, "mar": 3, "apr": 4, "jun": 6, "jul": 7, "aug": 8,
|
|
24
|
+
"sep": 9, "sept": 9, "oct": 10, "nov": 11, "dec": 12,
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
_ISO_DATE = re.compile(r"\b(\d{4})-(\d{1,2})-(\d{1,2})\b")
|
|
28
|
+
_NUMERIC_DATE = re.compile(r"\b(\d{1,2})[/.\-](\d{1,2})[/.\-](\d{2,4})\b")
|
|
29
|
+
_TEXT_DATE_DMY = re.compile(
|
|
30
|
+
r"\b(\d{1,2})\s*(?:de\s+|\.|\s)\s*([A-Za-zÀ-ÿ]{3,12})\.?\s*(?:de\s+|,\s*|\s)\s*(\d{4})\b",
|
|
31
|
+
re.IGNORECASE,
|
|
32
|
+
)
|
|
33
|
+
_TEXT_DATE_MDY = re.compile(r"\b([A-Za-zÀ-ÿ]{3,12})\.?\s+(\d{1,2})(?:st|nd|rd|th)?,?\s+(\d{4})\b", re.IGNORECASE)
|
|
34
|
+
|
|
35
|
+
#: Line shapes that open a new dated entry.
|
|
36
|
+
ENTRY_MARKERS: List[Pattern[str]] = [
|
|
37
|
+
re.compile(r"^\s*#{1,4}\s+\S"),
|
|
38
|
+
re.compile(r"^\s*\[?\d{4}-\d{1,2}-\d{1,2}"),
|
|
39
|
+
re.compile(r"^\s*\[?\d{1,2}[/.\-]\d{1,2}[/.\-]\d{2,4}"),
|
|
40
|
+
re.compile(
|
|
41
|
+
r"^\s*(?:sesi[oó]n|session|entrada|entry|consulta|visita|visit|encounter|d[ií]a|day|nota|note)\b"
|
|
42
|
+
r"[^\n]{0,60}$",
|
|
43
|
+
re.IGNORECASE,
|
|
44
|
+
),
|
|
45
|
+
re.compile(r"^\s*[-=*]{3,}\s*$"),
|
|
46
|
+
]
|
|
47
|
+
|
|
48
|
+
#: Section headers typical of a clinical record.
|
|
49
|
+
CLINICAL_MARKERS = re.compile(
|
|
50
|
+
r"^\s*(paciente|patient|motivo de consulta|chief complaint|anamnesis|antecedentes|history|"
|
|
51
|
+
r"exploraci[oó]n|examination|diagn[oó]stico|diagnosis|tratamiento|treatment|plan|evoluci[oó]n|"
|
|
52
|
+
r"follow[- ]?up|alergias|allergies|medicaci[oó]n|medication)\s*:",
|
|
53
|
+
re.IGNORECASE,
|
|
54
|
+
)
|
|
55
|
+
|
|
56
|
+
#: Chat transcript turn, e.g. ``[21:30] Luis: ...`` or ``Luis: ...``.
|
|
57
|
+
CHAT_TURN = re.compile(r"^\s*(?:\[\s*\d{1,2}:\d{2}\s*\]\s*)?([A-ZÀ-Ü][\wÀ-ÿ .'-]{1,24}):\s")
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
#: A date opening the text: ``2026-03-15: ...``, ``# 2026-03-15``, ``[15/03/2026]``.
|
|
61
|
+
_LEADING_DATE = re.compile(r"^\s*[#*\[(]*\s*(\d{4}-\d{1,2}-\d{1,2}|\d{1,2}[/.\-]\d{1,2}[/.\-]\d{2,4})\b")
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def leading_date(text: str, *, day_first: bool = True) -> Optional[str]:
|
|
65
|
+
"""Date the text *opens with*, as an ISO-8601 UTC timestamp, else ``None``.
|
|
66
|
+
|
|
67
|
+
Stricter than :func:`extract_date`, which takes the first date anywhere:
|
|
68
|
+
only a date at the very start of the text says when the text was true.
|
|
69
|
+
This is how agents write memories ("2026-03-15: release procedure ..."),
|
|
70
|
+
whatever the material type.
|
|
71
|
+
"""
|
|
72
|
+
match = _LEADING_DATE.match(text)
|
|
73
|
+
return extract_date(match.group(1), day_first=day_first) if match else None
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def extract_date(text: str, *, day_first: bool = True) -> Optional[str]:
|
|
77
|
+
"""Extract the first date in ``text`` as an ISO-8601 UTC timestamp.
|
|
78
|
+
|
|
79
|
+
Args:
|
|
80
|
+
text: Text to scan (usually the first lines of an entry).
|
|
81
|
+
day_first: Interpret ambiguous ``d/m/y`` as day-first (European
|
|
82
|
+
convention). Unambiguous values (first component > 12) always win.
|
|
83
|
+
|
|
84
|
+
Returns:
|
|
85
|
+
``"YYYY-MM-DDT00:00:00Z"``, or ``None`` if no plausible date is found.
|
|
86
|
+
"""
|
|
87
|
+
|
|
88
|
+
def build(year: int, month: int, day: int) -> Optional[str]:
|
|
89
|
+
if not (1 <= month <= 12 and 1 <= day <= 31 and 1000 <= year <= 9999):
|
|
90
|
+
return None
|
|
91
|
+
return f"{year:04d}-{month:02d}-{day:02d}T00:00:00Z"
|
|
92
|
+
|
|
93
|
+
match = _ISO_DATE.search(text)
|
|
94
|
+
if match:
|
|
95
|
+
return build(int(match.group(1)), int(match.group(2)), int(match.group(3)))
|
|
96
|
+
|
|
97
|
+
match = _TEXT_DATE_DMY.search(text)
|
|
98
|
+
if match:
|
|
99
|
+
month = MONTHS.get(match.group(2).lower())
|
|
100
|
+
if month:
|
|
101
|
+
return build(int(match.group(3)), month, int(match.group(1)))
|
|
102
|
+
|
|
103
|
+
match = _TEXT_DATE_MDY.search(text)
|
|
104
|
+
if match:
|
|
105
|
+
month = MONTHS.get(match.group(1).lower())
|
|
106
|
+
if month:
|
|
107
|
+
return build(int(match.group(3)), month, int(match.group(2)))
|
|
108
|
+
|
|
109
|
+
match = _NUMERIC_DATE.search(text)
|
|
110
|
+
if match:
|
|
111
|
+
first, second, raw_year = (int(match.group(1)), int(match.group(2)), match.group(3))
|
|
112
|
+
year = int(raw_year) if len(raw_year) == 4 else 2000 + int(raw_year)
|
|
113
|
+
day, month = (first, second) if (day_first or first > 12) else (second, first)
|
|
114
|
+
if month > 12 and day <= 12:
|
|
115
|
+
day, month = month, day
|
|
116
|
+
return build(year, month, day)
|
|
117
|
+
|
|
118
|
+
return None
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
class TemporalChunker(Chunker):
|
|
122
|
+
"""Splitter for personal logs, chat transcripts and clinical records."""
|
|
123
|
+
|
|
124
|
+
name = "temporal"
|
|
125
|
+
|
|
126
|
+
def __init__(self, config, *, clinical: bool = False) -> None:
|
|
127
|
+
super().__init__(config)
|
|
128
|
+
self.clinical = clinical
|
|
129
|
+
|
|
130
|
+
def blocks(self, text: str, *, source: str = "") -> List[Block]:
|
|
131
|
+
"""Return one block per entry, tagged with its inherited date."""
|
|
132
|
+
lines = split_lines(text)
|
|
133
|
+
starts = self._entry_starts(lines)
|
|
134
|
+
if not starts:
|
|
135
|
+
return self._fallback_blocks(text)
|
|
136
|
+
|
|
137
|
+
blocks: List[Block] = []
|
|
138
|
+
current_date: Optional[str] = None
|
|
139
|
+
for position, start in enumerate(starts):
|
|
140
|
+
end = starts[position + 1] - 1 if position + 1 < len(starts) else len(lines)
|
|
141
|
+
body = join_lines(lines, start, end)
|
|
142
|
+
head = "".join(lines[start - 1 : min(start + 1, len(lines))])
|
|
143
|
+
found = extract_date(head) or extract_date(body[:400])
|
|
144
|
+
is_new_day = bool(found) and found != current_date
|
|
145
|
+
if found:
|
|
146
|
+
current_date = found
|
|
147
|
+
|
|
148
|
+
speaker = CHAT_TURN.match(lines[start - 1])
|
|
149
|
+
tags = ["entry"]
|
|
150
|
+
if speaker:
|
|
151
|
+
tags.append(f"speaker:{speaker.group(1).strip()}")
|
|
152
|
+
if self.clinical:
|
|
153
|
+
tags.append("clinical")
|
|
154
|
+
if current_date:
|
|
155
|
+
tags.append(f"date:{current_date[:10]}")
|
|
156
|
+
|
|
157
|
+
blocks.append(
|
|
158
|
+
Block(
|
|
159
|
+
text=body,
|
|
160
|
+
title=truncate(lines[start - 1].strip() or f"entry {position + 1}", 70),
|
|
161
|
+
start_line=start,
|
|
162
|
+
end_line=end,
|
|
163
|
+
kind="encounter" if self.clinical else "entry",
|
|
164
|
+
tags=tags,
|
|
165
|
+
timestamp=current_date,
|
|
166
|
+
hard_boundary=is_new_day,
|
|
167
|
+
)
|
|
168
|
+
)
|
|
169
|
+
return blocks
|
|
170
|
+
|
|
171
|
+
def _entry_starts(self, lines: List[str]) -> List[int]:
|
|
172
|
+
"""1-based line numbers where a new entry begins."""
|
|
173
|
+
starts: List[int] = []
|
|
174
|
+
for index, line in enumerate(lines, start=1):
|
|
175
|
+
if not line.strip():
|
|
176
|
+
continue
|
|
177
|
+
is_start = (
|
|
178
|
+
any(pattern.match(line) for pattern in ENTRY_MARKERS)
|
|
179
|
+
or bool(CHAT_TURN.match(line))
|
|
180
|
+
or (self.clinical and bool(CLINICAL_MARKERS.match(line)))
|
|
181
|
+
)
|
|
182
|
+
if is_start:
|
|
183
|
+
starts.append(index)
|
|
184
|
+
if starts and starts[0] != 1:
|
|
185
|
+
# Preamble before the first marker becomes its own leading entry.
|
|
186
|
+
starts.insert(0, 1)
|
|
187
|
+
return starts
|
ihmt/config.py
ADDED
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
"""Configuration for an IHMT store.
|
|
2
|
+
|
|
3
|
+
The configuration is persisted inside the memory directory itself
|
|
4
|
+
(``ihmt_memory/ihmt.config.json``) so that a store is fully self-describing and
|
|
5
|
+
can be relocated as a single directory.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import json
|
|
11
|
+
from dataclasses import asdict, dataclass, field, fields
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
from typing import Any, Dict
|
|
14
|
+
|
|
15
|
+
MEMORY_DIRNAME = "ihmt_memory"
|
|
16
|
+
CONFIG_FILENAME = "ihmt.config.json"
|
|
17
|
+
SCHEMA_VERSION = 1
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
@dataclass
|
|
21
|
+
class IHMTConfig:
|
|
22
|
+
"""Tunable parameters governing chunking, tree shape and retrieval.
|
|
23
|
+
|
|
24
|
+
Attributes:
|
|
25
|
+
base_dir: Directory that *contains* the ``ihmt_memory`` store.
|
|
26
|
+
branch_factor: Number of children accumulated before a Summarization
|
|
27
|
+
Event fires. Also the logarithm base of the retrieval cost.
|
|
28
|
+
target_tokens: Desired size of a layer-0 leaf, in estimated tokens.
|
|
29
|
+
max_tokens: Hard ceiling before a chunk is flagged ``oversized``.
|
|
30
|
+
Never enforced by cutting a syntactic block in half.
|
|
31
|
+
min_tokens: Chunks smaller than this are merged with their neighbour
|
|
32
|
+
when the merge does not violate a block boundary.
|
|
33
|
+
beam_width: Branches kept alive per level during the tree descent.
|
|
34
|
+
confidence_threshold: Minimum normalized score for an answer to be
|
|
35
|
+
returned without asking the user for a clue.
|
|
36
|
+
min_coverage: Fraction of the query's distinct terms the best leaf
|
|
37
|
+
must contain to count as a match at all. Below it the search
|
|
38
|
+
reports no clear match instead of asking for a clue.
|
|
39
|
+
ambiguity_margin: Minimum score gap between the best and second-best
|
|
40
|
+
candidate branch. A smaller gap is treated as ambiguity.
|
|
41
|
+
max_clue_rounds: Maximum iterations of the interactive clue loop.
|
|
42
|
+
summarizer_backend: ``"heuristic"`` (offline, default) or
|
|
43
|
+
``"anthropic"`` (optional, requires the ``anthropic`` package).
|
|
44
|
+
summarizer_model: Model identifier used by LLM-backed summarizers.
|
|
45
|
+
excerpt_chars: Length of the child excerpt embedded in a parent node.
|
|
46
|
+
code_chunk_mode: ``"pack"`` (default) merges consecutive code blocks up
|
|
47
|
+
to ``target_tokens``; ``"symbol"`` stores one leaf per class member,
|
|
48
|
+
which is what a project index wants — a lookup then returns one
|
|
49
|
+
method instead of a whole file.
|
|
50
|
+
"""
|
|
51
|
+
|
|
52
|
+
base_dir: Path = field(default_factory=Path.cwd)
|
|
53
|
+
branch_factor: int = 8
|
|
54
|
+
target_tokens: int = 2000
|
|
55
|
+
max_tokens: int = 3000
|
|
56
|
+
min_tokens: int = 120
|
|
57
|
+
beam_width: int = 3
|
|
58
|
+
confidence_threshold: float = 0.45
|
|
59
|
+
min_coverage: float = 0.3
|
|
60
|
+
ambiguity_margin: float = 0.18
|
|
61
|
+
max_clue_rounds: int = 3
|
|
62
|
+
summarizer_backend: str = "heuristic"
|
|
63
|
+
summarizer_model: str = "claude-sonnet-5"
|
|
64
|
+
excerpt_chars: int = 320
|
|
65
|
+
code_chunk_mode: str = "pack"
|
|
66
|
+
schema_version: int = SCHEMA_VERSION
|
|
67
|
+
|
|
68
|
+
# ------------------------------------------------------------------ paths
|
|
69
|
+
@property
|
|
70
|
+
def memory_dir(self) -> Path:
|
|
71
|
+
"""Root of the on-disk store."""
|
|
72
|
+
return self.base_dir / MEMORY_DIRNAME
|
|
73
|
+
|
|
74
|
+
@property
|
|
75
|
+
def layer0_dir(self) -> Path:
|
|
76
|
+
"""Directory holding raw ``.txt`` leaves, partitioned by domain."""
|
|
77
|
+
return self.memory_dir / "layer_0"
|
|
78
|
+
|
|
79
|
+
@property
|
|
80
|
+
def layers_dir(self) -> Path:
|
|
81
|
+
"""Directory holding the numbered branch layers (``1``..``N``)."""
|
|
82
|
+
return self.memory_dir / "layers"
|
|
83
|
+
|
|
84
|
+
@property
|
|
85
|
+
def state_dir(self) -> Path:
|
|
86
|
+
"""Directory holding the catalog and the fact timeline."""
|
|
87
|
+
return self.memory_dir / "state"
|
|
88
|
+
|
|
89
|
+
@property
|
|
90
|
+
def root_path(self) -> Path:
|
|
91
|
+
"""Path of ``root.json``, the trunk of the tree."""
|
|
92
|
+
return self.memory_dir / "root.json"
|
|
93
|
+
|
|
94
|
+
@property
|
|
95
|
+
def catalog_path(self) -> Path:
|
|
96
|
+
return self.state_dir / "catalog.json"
|
|
97
|
+
|
|
98
|
+
@property
|
|
99
|
+
def facts_path(self) -> Path:
|
|
100
|
+
return self.state_dir / "facts.json"
|
|
101
|
+
|
|
102
|
+
@property
|
|
103
|
+
def config_path(self) -> Path:
|
|
104
|
+
return self.memory_dir / CONFIG_FILENAME
|
|
105
|
+
|
|
106
|
+
def layer_dir(self, layer: int) -> Path:
|
|
107
|
+
"""Return the directory for a given branch layer (``layer >= 1``)."""
|
|
108
|
+
if layer < 1:
|
|
109
|
+
raise ValueError(f"branch layers start at 1, got {layer}")
|
|
110
|
+
return self.layers_dir / str(layer)
|
|
111
|
+
|
|
112
|
+
# ------------------------------------------------------------- (de)serial
|
|
113
|
+
def to_dict(self) -> Dict[str, Any]:
|
|
114
|
+
data = asdict(self)
|
|
115
|
+
data.pop("base_dir", None)
|
|
116
|
+
return data
|
|
117
|
+
|
|
118
|
+
@classmethod
|
|
119
|
+
def from_dict(cls, data: Dict[str, Any], base_dir: Path) -> "IHMTConfig":
|
|
120
|
+
"""Build a config from a dict, ignoring unknown/legacy keys."""
|
|
121
|
+
known = {f.name for f in fields(cls)} - {"base_dir"}
|
|
122
|
+
kwargs = {k: v for k, v in data.items() if k in known}
|
|
123
|
+
return cls(base_dir=Path(base_dir), **kwargs)
|
|
124
|
+
|
|
125
|
+
def save(self) -> None:
|
|
126
|
+
"""Persist the configuration next to the store."""
|
|
127
|
+
self.config_path.parent.mkdir(parents=True, exist_ok=True)
|
|
128
|
+
tmp = self.config_path.with_suffix(".json.tmp")
|
|
129
|
+
tmp.write_text(
|
|
130
|
+
json.dumps(self.to_dict(), indent=2, ensure_ascii=False) + "\n",
|
|
131
|
+
encoding="utf-8",
|
|
132
|
+
)
|
|
133
|
+
tmp.replace(self.config_path)
|
|
134
|
+
|
|
135
|
+
@classmethod
|
|
136
|
+
def load(cls, base_dir: Path | str = ".") -> "IHMTConfig":
|
|
137
|
+
"""Load the configuration for ``base_dir``, falling back to defaults."""
|
|
138
|
+
base = Path(base_dir).expanduser().resolve()
|
|
139
|
+
path = base / MEMORY_DIRNAME / CONFIG_FILENAME
|
|
140
|
+
if not path.exists():
|
|
141
|
+
return cls(base_dir=base)
|
|
142
|
+
try:
|
|
143
|
+
data = json.loads(path.read_text(encoding="utf-8"))
|
|
144
|
+
except (OSError, json.JSONDecodeError) as exc: # pragma: no cover
|
|
145
|
+
raise RuntimeError(f"corrupt IHMT config at {path}: {exc}") from exc
|
|
146
|
+
return cls.from_dict(data, base)
|
|
147
|
+
|
|
148
|
+
def validate(self) -> None:
|
|
149
|
+
"""Raise :class:`ValueError` if the parameters are incoherent."""
|
|
150
|
+
if self.branch_factor < 2:
|
|
151
|
+
raise ValueError("branch_factor must be >= 2 for the tree to converge")
|
|
152
|
+
if self.target_tokens <= 0 or self.max_tokens < self.target_tokens:
|
|
153
|
+
raise ValueError("require 0 < target_tokens <= max_tokens")
|
|
154
|
+
if self.beam_width < 1:
|
|
155
|
+
raise ValueError("beam_width must be >= 1")
|
|
156
|
+
if not 0.0 <= self.confidence_threshold <= 1.0:
|
|
157
|
+
raise ValueError("confidence_threshold must lie in [0, 1]")
|
|
158
|
+
if not 0.0 <= self.min_coverage <= 1.0:
|
|
159
|
+
raise ValueError("min_coverage must lie in [0, 1]")
|
|
160
|
+
if self.code_chunk_mode not in ("pack", "symbol"):
|
|
161
|
+
raise ValueError('code_chunk_mode must be "pack" or "symbol"')
|