everalgo-boundary 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- everalgo_boundary-0.1.0/.gitignore +58 -0
- everalgo_boundary-0.1.0/CHANGELOG.md +24 -0
- everalgo_boundary-0.1.0/LICENSE +21 -0
- everalgo_boundary-0.1.0/PKG-INFO +113 -0
- everalgo_boundary-0.1.0/README.md +93 -0
- everalgo_boundary-0.1.0/pyproject.toml +33 -0
- everalgo_boundary-0.1.0/src/everalgo/boundary/__init__.py +24 -0
- everalgo_boundary-0.1.0/src/everalgo/boundary/chat.py +221 -0
- everalgo_boundary-0.1.0/src/everalgo/boundary/prompts/__init__.py +6 -0
- everalgo_boundary-0.1.0/src/everalgo/boundary/prompts/en/__init__.py +1 -0
- everalgo_boundary-0.1.0/src/everalgo/boundary/prompts/en/chat.py +99 -0
- everalgo_boundary-0.1.0/src/everalgo/boundary/prompts/zh/__init__.py +1 -0
- everalgo_boundary-0.1.0/src/everalgo/boundary/prompts/zh/chat.py +99 -0
- everalgo_boundary-0.1.0/src/everalgo/boundary/py.typed +0 -0
- everalgo_boundary-0.1.0/src/everalgo/boundary/workspace.py +42 -0
- everalgo_boundary-0.1.0/tests/boundary/__init__.py +0 -0
- everalgo_boundary-0.1.0/tests/boundary/test_boundary_public_api.py +66 -0
- everalgo_boundary-0.1.0/tests/boundary/test_chat.py +567 -0
- everalgo_boundary-0.1.0/tests/boundary/test_tokenize.py +160 -0
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
# Byte-compiled / optimized / DLL files
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
*$py.class
|
|
5
|
+
|
|
6
|
+
# Distribution / packaging
|
|
7
|
+
build/
|
|
8
|
+
dist/
|
|
9
|
+
*.egg-info/
|
|
10
|
+
*.egg
|
|
11
|
+
wheels/
|
|
12
|
+
|
|
13
|
+
# Environments
|
|
14
|
+
.env
|
|
15
|
+
.env.local
|
|
16
|
+
.venv
|
|
17
|
+
env/
|
|
18
|
+
venv/
|
|
19
|
+
|
|
20
|
+
# uv
|
|
21
|
+
.uv/
|
|
22
|
+
|
|
23
|
+
# Test / coverage / cache
|
|
24
|
+
.pytest_cache/
|
|
25
|
+
.cache
|
|
26
|
+
.coverage
|
|
27
|
+
.coverage.*
|
|
28
|
+
coverage.xml
|
|
29
|
+
report.xml
|
|
30
|
+
htmlcov/
|
|
31
|
+
.tox/
|
|
32
|
+
.nox/
|
|
33
|
+
.hypothesis/
|
|
34
|
+
|
|
35
|
+
# Type checkers / linters
|
|
36
|
+
.mypy_cache/
|
|
37
|
+
.dmypy.json
|
|
38
|
+
.pyre/
|
|
39
|
+
.pytype/
|
|
40
|
+
.ruff_cache/
|
|
41
|
+
|
|
42
|
+
# Jupyter
|
|
43
|
+
.ipynb_checkpoints/
|
|
44
|
+
|
|
45
|
+
# IDE / OS
|
|
46
|
+
.vscode/
|
|
47
|
+
.idea/
|
|
48
|
+
*.swp
|
|
49
|
+
.DS_Store
|
|
50
|
+
|
|
51
|
+
# Logs
|
|
52
|
+
*.log
|
|
53
|
+
|
|
54
|
+
# Project local artifacts (placed under local/ per project convention)
|
|
55
|
+
local/
|
|
56
|
+
|
|
57
|
+
# Claude Code session-local state (scheduled tasks, agent caches, locks)
|
|
58
|
+
.claude/
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to this package are documented here. Format follows
|
|
4
|
+
[Keep a Changelog 1.1.0](https://keepachangelog.com/en/1.1.0/). Versioning
|
|
5
|
+
follows [Semantic Versioning 2.0](https://semver.org/spec/v2.0.0.html).
|
|
6
|
+
|
|
7
|
+
## [Unreleased]
|
|
8
|
+
|
|
9
|
+
### Added
|
|
10
|
+
|
|
11
|
+
- `detect_boundaries(messages, *, llm, tail) -> DetectionResult`: async public entry point that splits a `list[ChatMessage]` into `MemCell` slices using a single LLM call; no retry.
|
|
12
|
+
- `DetectionResult` `NamedTuple`: `(cells: list[MemCell], tail: list[ChatMessage])` where `tail` is the unconsumed carry-forward window.
|
|
13
|
+
- `WorkspaceMemCellExtractor`: stub extractor for Jira / Email / Confluence inputs (EXPERIMENTAL; implementation pending).
|
|
14
|
+
- `count_tokens(text: str) -> int`: token count using OpenAI `o200k_base` encoding via `tiktoken`.
|
|
15
|
+
- `force_split(text: str, *, max_tokens: int) -> list[str]`: token-bounded chunking for caller-side prompt fitting.
|
|
16
|
+
- English and Chinese boundary detection prompts under `boundary/prompts/{en,zh}/`.
|
|
17
|
+
- `BoundaryDetector` (in `everalgo-user-memory`) and `AgentBoundaryDetector` (in `everalgo-agent-memory`) are the facade classes; `detect_boundaries` is this package's low-level public primitive.
|
|
18
|
+
- 81 unit tests across `test_chat.py` (boundary logic), `test_boundary_public_api.py` (public contract), and `test_tokenize.py` (token helpers).
|
|
19
|
+
|
|
20
|
+
### Removed
|
|
21
|
+
|
|
22
|
+
- 5-retry loop in the internal `_detect_boundaries` helper: the function now raises `ValueError` immediately on malformed LLM JSON instead of silently retrying.
|
|
23
|
+
|
|
24
|
+
[Unreleased]: https://github.com/EverMind-AI/EverAlgo/compare/main...HEAD
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 EverMind
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: everalgo-boundary
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: EverAlgo boundary: MemCell extractors (chat / workspace / agent).
|
|
5
|
+
Project-URL: Homepage, https://github.com/EverMind-AI/EverAlgo
|
|
6
|
+
Project-URL: Repository, https://github.com/EverMind-AI/EverAlgo
|
|
7
|
+
Project-URL: Issues, https://github.com/EverMind-AI/EverAlgo/issues
|
|
8
|
+
Project-URL: Documentation, https://github.com/EverMind-AI/EverAlgo/tree/main/packages/everalgo-boundary
|
|
9
|
+
Project-URL: Changelog, https://github.com/EverMind-AI/EverAlgo/blob/main/packages/everalgo-boundary/CHANGELOG.md
|
|
10
|
+
Author: EverMind
|
|
11
|
+
License-Expression: MIT
|
|
12
|
+
License-File: LICENSE
|
|
13
|
+
Classifier: Development Status :: 3 - Alpha
|
|
14
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
16
|
+
Requires-Python: >=3.12
|
|
17
|
+
Requires-Dist: asgiref>=3.0
|
|
18
|
+
Requires-Dist: everalgo-core<2.0.0,>=0.1.0
|
|
19
|
+
Description-Content-Type: text/markdown
|
|
20
|
+
|
|
21
|
+
# everalgo-boundary
|
|
22
|
+
|
|
23
|
+
Chat boundary detection for EverAlgo — segments a flat list of `ChatMessage` objects into coherent `MemCell` slices using an LLM-based batch algorithm.
|
|
24
|
+
|
|
25
|
+
See the umbrella project: [EverAlgo monorepo](../../README.md) and the architecture document at [`docs/concepts/architecture.md`](../../docs/concepts/architecture.md).
|
|
26
|
+
|
|
27
|
+
## Install
|
|
28
|
+
|
|
29
|
+
```bash
|
|
30
|
+
pip install everalgo-boundary
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
For the user-scenario class facade, install [`everalgo-user-memory`](../everalgo-user-memory/) instead — it re-exports `BoundaryDetector` which wraps this package.
|
|
34
|
+
|
|
35
|
+
## What this distribution provides
|
|
36
|
+
|
|
37
|
+
| Symbol | Role |
|
|
38
|
+
|---|---|
|
|
39
|
+
| `detect_boundaries` | Low-level async function: `(list[ChatMessage], *, llm, is_final, ...) → DetectionResult` |
|
|
40
|
+
| `DetectionResult` | `NamedTuple(cells: list[MemCell], tail: list[ChatMessage])` |
|
|
41
|
+
| `WorkspaceMemCellExtractor` | Placeholder stub for Jira / Email / Confluence (raises `NotImplementedError`) |
|
|
42
|
+
|
|
43
|
+
The class-style facades (`BoundaryDetector` for user-scenario chat, `AgentBoundaryDetector` for agent trajectories with tool calls) live in [`everalgo-user-memory`](../everalgo-user-memory/) and [`everalgo-agent-memory`](../everalgo-agent-memory/) respectively.
|
|
44
|
+
|
|
45
|
+
## Quick start
|
|
46
|
+
|
|
47
|
+
```python
|
|
48
|
+
import asyncio
|
|
49
|
+
import json
|
|
50
|
+
|
|
51
|
+
from everalgo.boundary import detect_boundaries
|
|
52
|
+
from everalgo.llm.types import ChatMessage as LLMChatMessage, ChatResponse
|
|
53
|
+
from everalgo.testing.fake_llm import FakeLLMClient
|
|
54
|
+
from everalgo.types import ChatMessage
|
|
55
|
+
|
|
56
|
+
_BOUNDARY_JSON = json.dumps(
|
|
57
|
+
{"reasoning": "single topic", "boundaries": [], "should_wait": False}
|
|
58
|
+
)
|
|
59
|
+
|
|
60
|
+
async def main() -> None:
|
|
61
|
+
fake = FakeLLMClient(responses=[ChatResponse(content=_BOUNDARY_JSON, model="fake")])
|
|
62
|
+
messages = [
|
|
63
|
+
ChatMessage(id="m1", role="user", content="Let's talk about deployment.", timestamp=1_700_000_000_000, sender_id="u_alice"),
|
|
64
|
+
ChatMessage(id="m2", role="assistant", content="Sure — what's the target env?", timestamp=1_700_000_001_000, sender_id="assistant"),
|
|
65
|
+
ChatMessage(id="m3", role="user", content="K8s. Switching topic: lunch?", timestamp=1_700_000_002_000, sender_id="u_alice"),
|
|
66
|
+
]
|
|
67
|
+
|
|
68
|
+
# Streaming: hold `tail` between calls; pass prior tail + new messages each time.
|
|
69
|
+
result = await detect_boundaries(messages, llm=fake)
|
|
70
|
+
cells, tail = result # NamedTuple unpacking
|
|
71
|
+
|
|
72
|
+
# End-of-session: tail is forced into the last cell.
|
|
73
|
+
result = await detect_boundaries(messages, llm=fake, is_final=True)
|
|
74
|
+
assert result.tail == []
|
|
75
|
+
for mc in result.cells:
|
|
76
|
+
print(mc.timestamp, len(mc.items))
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
asyncio.run(main())
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
## The streaming state machine
|
|
83
|
+
|
|
84
|
+
`detect_boundaries` deliberately holds back trailing messages as `tail` — the LLM cannot know whether a conversation continues beyond the last seen message. The caller maintains state:
|
|
85
|
+
|
|
86
|
+
```python
|
|
87
|
+
tail: list[ChatMessage] = []
|
|
88
|
+
|
|
89
|
+
for batch in incoming_batches:
|
|
90
|
+
result = await detect_boundaries(tail + batch, llm=client)
|
|
91
|
+
await persist(result.cells)
|
|
92
|
+
tail = result.tail
|
|
93
|
+
|
|
94
|
+
# Session ends — flush everything.
|
|
95
|
+
final = await detect_boundaries(tail, llm=client, is_final=True)
|
|
96
|
+
await persist(final.cells)
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
## Tokenizer utilities
|
|
100
|
+
|
|
101
|
+
`everalgo._tokenize` (in `everalgo-core`) exposes two module-private utilities used by boundary algorithms; not part of the public surface:
|
|
102
|
+
|
|
103
|
+
- `count_tokens(text: str) → int` — token count under OpenAI `o200k_base` encoding via [`tiktoken`](https://github.com/openai/tiktoken).
|
|
104
|
+
- `force_split(text: str, *, max_tokens: int) → list[str]` — last-resort token-bounded chunking; no semantic awareness.
|
|
105
|
+
|
|
106
|
+
## Stubs
|
|
107
|
+
|
|
108
|
+
`WorkspaceMemCellExtractor` (Jira / Email / Confluence) is a placeholder in `v0.x` — all methods raise `NotImplementedError`. Implementation lands in a future minor bump when the `RawData` contract is finalised.
|
|
109
|
+
|
|
110
|
+
## Related distributions
|
|
111
|
+
|
|
112
|
+
- [`everalgo-user-memory`](../everalgo-user-memory/) — `BoundaryDetector` class facade for chat scenarios
|
|
113
|
+
- [`everalgo-agent-memory`](../everalgo-agent-memory/) — `AgentBoundaryDetector` class facade for agent trajectories
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
# everalgo-boundary
|
|
2
|
+
|
|
3
|
+
Chat boundary detection for EverAlgo — segments a flat list of `ChatMessage` objects into coherent `MemCell` slices using an LLM-based batch algorithm.
|
|
4
|
+
|
|
5
|
+
See the umbrella project: [EverAlgo monorepo](../../README.md) and the architecture document at [`docs/concepts/architecture.md`](../../docs/concepts/architecture.md).
|
|
6
|
+
|
|
7
|
+
## Install
|
|
8
|
+
|
|
9
|
+
```bash
|
|
10
|
+
pip install everalgo-boundary
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
For the user-scenario class facade, install [`everalgo-user-memory`](../everalgo-user-memory/) instead — it re-exports `BoundaryDetector` which wraps this package.
|
|
14
|
+
|
|
15
|
+
## What this distribution provides
|
|
16
|
+
|
|
17
|
+
| Symbol | Role |
|
|
18
|
+
|---|---|
|
|
19
|
+
| `detect_boundaries` | Low-level async function: `(list[ChatMessage], *, llm, is_final, ...) → DetectionResult` |
|
|
20
|
+
| `DetectionResult` | `NamedTuple(cells: list[MemCell], tail: list[ChatMessage])` |
|
|
21
|
+
| `WorkspaceMemCellExtractor` | Placeholder stub for Jira / Email / Confluence (raises `NotImplementedError`) |
|
|
22
|
+
|
|
23
|
+
The class-style facades (`BoundaryDetector` for user-scenario chat, `AgentBoundaryDetector` for agent trajectories with tool calls) live in [`everalgo-user-memory`](../everalgo-user-memory/) and [`everalgo-agent-memory`](../everalgo-agent-memory/) respectively.
|
|
24
|
+
|
|
25
|
+
## Quick start
|
|
26
|
+
|
|
27
|
+
```python
|
|
28
|
+
import asyncio
|
|
29
|
+
import json
|
|
30
|
+
|
|
31
|
+
from everalgo.boundary import detect_boundaries
|
|
32
|
+
from everalgo.llm.types import ChatMessage as LLMChatMessage, ChatResponse
|
|
33
|
+
from everalgo.testing.fake_llm import FakeLLMClient
|
|
34
|
+
from everalgo.types import ChatMessage
|
|
35
|
+
|
|
36
|
+
_BOUNDARY_JSON = json.dumps(
|
|
37
|
+
{"reasoning": "single topic", "boundaries": [], "should_wait": False}
|
|
38
|
+
)
|
|
39
|
+
|
|
40
|
+
async def main() -> None:
|
|
41
|
+
fake = FakeLLMClient(responses=[ChatResponse(content=_BOUNDARY_JSON, model="fake")])
|
|
42
|
+
messages = [
|
|
43
|
+
ChatMessage(id="m1", role="user", content="Let's talk about deployment.", timestamp=1_700_000_000_000, sender_id="u_alice"),
|
|
44
|
+
ChatMessage(id="m2", role="assistant", content="Sure — what's the target env?", timestamp=1_700_000_001_000, sender_id="assistant"),
|
|
45
|
+
ChatMessage(id="m3", role="user", content="K8s. Switching topic: lunch?", timestamp=1_700_000_002_000, sender_id="u_alice"),
|
|
46
|
+
]
|
|
47
|
+
|
|
48
|
+
# Streaming: hold `tail` between calls; pass prior tail + new messages each time.
|
|
49
|
+
result = await detect_boundaries(messages, llm=fake)
|
|
50
|
+
cells, tail = result # NamedTuple unpacking
|
|
51
|
+
|
|
52
|
+
# End-of-session: tail is forced into the last cell.
|
|
53
|
+
result = await detect_boundaries(messages, llm=fake, is_final=True)
|
|
54
|
+
assert result.tail == []
|
|
55
|
+
for mc in result.cells:
|
|
56
|
+
print(mc.timestamp, len(mc.items))
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
asyncio.run(main())
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
## The streaming state machine
|
|
63
|
+
|
|
64
|
+
`detect_boundaries` deliberately holds back trailing messages as `tail` — the LLM cannot know whether a conversation continues beyond the last seen message. The caller maintains state:
|
|
65
|
+
|
|
66
|
+
```python
|
|
67
|
+
tail: list[ChatMessage] = []
|
|
68
|
+
|
|
69
|
+
for batch in incoming_batches:
|
|
70
|
+
result = await detect_boundaries(tail + batch, llm=client)
|
|
71
|
+
await persist(result.cells)
|
|
72
|
+
tail = result.tail
|
|
73
|
+
|
|
74
|
+
# Session ends — flush everything.
|
|
75
|
+
final = await detect_boundaries(tail, llm=client, is_final=True)
|
|
76
|
+
await persist(final.cells)
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
## Tokenizer utilities
|
|
80
|
+
|
|
81
|
+
`everalgo._tokenize` (in `everalgo-core`) exposes two module-private utilities used by boundary algorithms; not part of the public surface:
|
|
82
|
+
|
|
83
|
+
- `count_tokens(text: str) → int` — token count under OpenAI `o200k_base` encoding via [`tiktoken`](https://github.com/openai/tiktoken).
|
|
84
|
+
- `force_split(text: str, *, max_tokens: int) → list[str]` — last-resort token-bounded chunking; no semantic awareness.
|
|
85
|
+
|
|
86
|
+
## Stubs
|
|
87
|
+
|
|
88
|
+
`WorkspaceMemCellExtractor` (Jira / Email / Confluence) is a placeholder in `v0.x` — all methods raise `NotImplementedError`. Implementation lands in a future minor bump when the `RawData` contract is finalised.
|
|
89
|
+
|
|
90
|
+
## Related distributions
|
|
91
|
+
|
|
92
|
+
- [`everalgo-user-memory`](../everalgo-user-memory/) — `BoundaryDetector` class facade for chat scenarios
|
|
93
|
+
- [`everalgo-agent-memory`](../everalgo-agent-memory/) — `AgentBoundaryDetector` class facade for agent trajectories
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "everalgo-boundary"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "EverAlgo boundary: MemCell extractors (chat / workspace / agent)."
|
|
9
|
+
requires-python = ">=3.12"
|
|
10
|
+
license = "MIT"
|
|
11
|
+
readme = "README.md"
|
|
12
|
+
authors = [{ name = "EverMind" }]
|
|
13
|
+
classifiers = [
|
|
14
|
+
"Development Status :: 3 - Alpha",
|
|
15
|
+
"License :: OSI Approved :: MIT License",
|
|
16
|
+
"Programming Language :: Python :: 3.12",
|
|
17
|
+
]
|
|
18
|
+
# Loose lower bound + cross-major upper bound, mirroring HuggingFace `transformers`
|
|
19
|
+
# pin on `huggingface-hub`. See AGENTS.md §6 (Branching & Commits) for version strategy.
|
|
20
|
+
dependencies = [
|
|
21
|
+
"asgiref>=3.0",
|
|
22
|
+
"everalgo-core>=0.1.0,<2.0.0",
|
|
23
|
+
]
|
|
24
|
+
|
|
25
|
+
[project.urls]
|
|
26
|
+
Homepage = "https://github.com/EverMind-AI/EverAlgo"
|
|
27
|
+
Repository = "https://github.com/EverMind-AI/EverAlgo"
|
|
28
|
+
Issues = "https://github.com/EverMind-AI/EverAlgo/issues"
|
|
29
|
+
Documentation = "https://github.com/EverMind-AI/EverAlgo/tree/main/packages/everalgo-boundary"
|
|
30
|
+
Changelog = "https://github.com/EverMind-AI/EverAlgo/blob/main/packages/everalgo-boundary/CHANGELOG.md"
|
|
31
|
+
|
|
32
|
+
[tool.hatch.build.targets.wheel]
|
|
33
|
+
packages = ["src/everalgo"]
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
"""Boundary detection — chat MemCell primitive.
|
|
2
|
+
|
|
3
|
+
Public surface:
|
|
4
|
+
- detect_boundaries — async function: split chat messages into MemCells
|
|
5
|
+
- DetectionResult — ``(cells, tail)`` NamedTuple returned by detect_boundaries
|
|
6
|
+
- WorkspaceMemCellExtractor — Jira / Email / Confluence stub (unchanged)
|
|
7
|
+
|
|
8
|
+
Facade classes (BoundaryDetector / AgentBoundaryDetector) live in everalgo-user-memory and
|
|
9
|
+
everalgo-agent-memory respectively; see Stage 3-4 of the refactor.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
import logging
|
|
13
|
+
|
|
14
|
+
from everalgo.boundary.chat import DetectionResult, detect_boundaries
|
|
15
|
+
from everalgo.boundary.workspace import WorkspaceMemCellExtractor
|
|
16
|
+
|
|
17
|
+
__all__ = [
|
|
18
|
+
"DetectionResult",
|
|
19
|
+
"WorkspaceMemCellExtractor",
|
|
20
|
+
"detect_boundaries",
|
|
21
|
+
]
|
|
22
|
+
|
|
23
|
+
# Library logging setup (ADR-013): NullHandler on each subpackage logger.
|
|
24
|
+
logging.getLogger(__name__).addHandler(logging.NullHandler())
|
|
@@ -0,0 +1,221 @@
|
|
|
1
|
+
"""Low-level chat boundary detection primitive — splits a flat list of ChatMessages into MemCell slices.
|
|
2
|
+
|
|
3
|
+
Facade classes for the two upstream scenarios live in ``everalgo-user-memory`` (``BoundaryDetector``)
|
|
4
|
+
and ``everalgo-agent-memory`` (``AgentBoundaryDetector``).
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from dataclasses import dataclass
|
|
10
|
+
from typing import TYPE_CHECKING, NamedTuple, cast
|
|
11
|
+
|
|
12
|
+
from everalgo._tokenize import count_tokens
|
|
13
|
+
from everalgo.boundary.prompts.en.chat import CHAT_BOUNDARY_DETECT_PROMPT_EN
|
|
14
|
+
from everalgo.llm.format import format_message_timestamp
|
|
15
|
+
from everalgo.llm.parse import parse_llm_json_object
|
|
16
|
+
from everalgo.llm.types import ChatMessage as LLMChatMessage
|
|
17
|
+
from everalgo.prompts import render_prompt
|
|
18
|
+
from everalgo.types import ChatMessage, ConversationItem, MemCell
|
|
19
|
+
|
|
20
|
+
if TYPE_CHECKING:
|
|
21
|
+
from everalgo.llm.protocols import LLMClient
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
DEFAULT_HARD_TOKEN_LIMIT = 65536
|
|
25
|
+
DEFAULT_HARD_MSG_LIMIT = 500
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@dataclass(frozen=True)
|
|
29
|
+
class _BatchBoundaryResult:
|
|
30
|
+
"""Parsed LLM batch response."""
|
|
31
|
+
|
|
32
|
+
boundaries: list[int]
|
|
33
|
+
should_wait: bool
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class DetectionResult(NamedTuple):
|
|
37
|
+
"""Return type of :func:`detect_boundaries`.
|
|
38
|
+
|
|
39
|
+
NamedTuple subclass of ``tuple`` — supports positional unpacking, named access, and index access::
|
|
40
|
+
|
|
41
|
+
cells, tail = await detect_boundaries(messages, llm=client)
|
|
42
|
+
result = await detect_boundaries(messages, llm=client)
|
|
43
|
+
result.cells # list[MemCell]
|
|
44
|
+
result.tail # list[ChatMessage]
|
|
45
|
+
result[0], result[1]
|
|
46
|
+
|
|
47
|
+
Attributes:
|
|
48
|
+
cells: Zero or more :class:`~everalgo.types.MemCell` instances the algorithm has confidently
|
|
49
|
+
closed. Caller persists immediately.
|
|
50
|
+
tail: Trailing messages the algorithm did NOT close — the LLM cannot judge whether the
|
|
51
|
+
conversation continues beyond the last seen message, so the trailing segment is left as a
|
|
52
|
+
tail for the caller's state machine. When ``is_final=True``, tail is forced into the final
|
|
53
|
+
MemCell and this field is guaranteed empty.
|
|
54
|
+
"""
|
|
55
|
+
|
|
56
|
+
cells: list[MemCell]
|
|
57
|
+
tail: list[ChatMessage]
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
async def detect_boundaries(
|
|
61
|
+
messages: list[ChatMessage],
|
|
62
|
+
*,
|
|
63
|
+
llm: LLMClient,
|
|
64
|
+
is_final: bool = False,
|
|
65
|
+
prompt: str | None = None,
|
|
66
|
+
hard_token_limit: int = DEFAULT_HARD_TOKEN_LIMIT,
|
|
67
|
+
hard_msg_limit: int = DEFAULT_HARD_MSG_LIMIT,
|
|
68
|
+
) -> DetectionResult:
|
|
69
|
+
"""Detect MemCell boundaries on a list of ChatMessages (chat-only path).
|
|
70
|
+
|
|
71
|
+
Args:
|
|
72
|
+
messages: Ordered chat messages, typically ``prior_tail + new_messages``.
|
|
73
|
+
llm: LLM client. Algorithm always requires an LLM; the caller must supply one.
|
|
74
|
+
is_final: When ``True``, forces the trailing segment into a cell so ``tail == []``.
|
|
75
|
+
prompt: Prompt override; ``None`` uses the bundled default.
|
|
76
|
+
hard_token_limit: Max tokens per cell before forced split.
|
|
77
|
+
hard_msg_limit: Max messages per cell before forced split.
|
|
78
|
+
|
|
79
|
+
Raises:
|
|
80
|
+
ValueError: If the LLM response cannot be parsed as a valid JSON object.
|
|
81
|
+
TypeError: If the ``boundaries`` field in the LLM response is not a list.
|
|
82
|
+
LLMError: Propagated from the underlying LLM client call.
|
|
83
|
+
"""
|
|
84
|
+
# Phase 1 — Input validation
|
|
85
|
+
if not messages:
|
|
86
|
+
return DetectionResult(cells=[], tail=[])
|
|
87
|
+
|
|
88
|
+
# Phase 2 — Default resolution
|
|
89
|
+
prompt_template = prompt or CHAT_BOUNDARY_DETECT_PROMPT_EN
|
|
90
|
+
|
|
91
|
+
# Phase 3 — Force-split loop
|
|
92
|
+
cells: list[MemCell] = []
|
|
93
|
+
remaining: list[ChatMessage] = list(messages)
|
|
94
|
+
while len(remaining) > 1:
|
|
95
|
+
total_tokens = count_tokens(_render_for_token_count(remaining))
|
|
96
|
+
total_msgs = len(remaining)
|
|
97
|
+
if total_tokens < hard_token_limit and total_msgs < hard_msg_limit:
|
|
98
|
+
break
|
|
99
|
+
split_at = _find_force_split_point(remaining, hard_token_limit, hard_msg_limit)
|
|
100
|
+
cells.append(_make_cell(remaining[:split_at]))
|
|
101
|
+
remaining = remaining[split_at:]
|
|
102
|
+
|
|
103
|
+
# Phase 4 — LLM batch detection (single call, multi-boundary).
|
|
104
|
+
boundaries: list[int] = []
|
|
105
|
+
if remaining:
|
|
106
|
+
batch = await _detect_boundaries(remaining, llm, prompt_template)
|
|
107
|
+
boundaries = batch.boundaries
|
|
108
|
+
|
|
109
|
+
# Phase 5 — Slice by boundaries + is_final closure.
|
|
110
|
+
prev = 0
|
|
111
|
+
for b in boundaries:
|
|
112
|
+
segment = remaining[prev:b]
|
|
113
|
+
if segment:
|
|
114
|
+
cells.append(_make_cell(segment))
|
|
115
|
+
prev = b
|
|
116
|
+
tail_msgs = remaining[prev:]
|
|
117
|
+
|
|
118
|
+
if is_final and tail_msgs:
|
|
119
|
+
cells.append(_make_cell(tail_msgs))
|
|
120
|
+
tail: list[ChatMessage] = []
|
|
121
|
+
else:
|
|
122
|
+
tail = tail_msgs
|
|
123
|
+
|
|
124
|
+
return DetectionResult(cells=cells, tail=tail)
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
# ---------- Module-level helpers (stateless, prefix `_`) ----------
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def _render_for_token_count(messages: list[ChatMessage]) -> str:
|
|
131
|
+
"""Render messages as ``"{sender}: {content}"`` lines for token counting.
|
|
132
|
+
|
|
133
|
+
Includes the sender prefix so the count reflects the actual prompt budget.
|
|
134
|
+
"""
|
|
135
|
+
lines: list[str] = []
|
|
136
|
+
for m in messages:
|
|
137
|
+
speaker = m.sender_name or m.sender_id
|
|
138
|
+
content = (
|
|
139
|
+
m.content
|
|
140
|
+
if isinstance(m.content, str)
|
|
141
|
+
else " ".join(block.text for block in m.content if hasattr(block, "text"))
|
|
142
|
+
)
|
|
143
|
+
lines.append(f"{speaker}: {content}")
|
|
144
|
+
return "\n".join(lines)
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def _format_messages_with_indices(messages: list[ChatMessage]) -> str:
|
|
148
|
+
"""Render messages as ``[N] [ISO+TZ] sender_name: content`` lines (1-based N). Empty-content messages are dropped."""
|
|
149
|
+
lines: list[str] = []
|
|
150
|
+
for i, msg in enumerate(messages, start=1):
|
|
151
|
+
content = (
|
|
152
|
+
msg.content
|
|
153
|
+
if isinstance(msg.content, str)
|
|
154
|
+
else " ".join(block.text for block in msg.content if hasattr(block, "text"))
|
|
155
|
+
)
|
|
156
|
+
if not content:
|
|
157
|
+
continue
|
|
158
|
+
sender_name = msg.sender_name or msg.sender_id
|
|
159
|
+
time_str = format_message_timestamp(msg.timestamp)
|
|
160
|
+
lines.append(f"[{i}] [{time_str}] {sender_name}: {content}")
|
|
161
|
+
return "\n".join(lines)
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def _find_force_split_point(messages: list[ChatMessage], hard_token_limit: int, hard_msg_limit: int) -> int:
|
|
165
|
+
"""Binary-halving search for a force-split index; floor 1 so even a single very-long message is accepted."""
|
|
166
|
+
if len(messages) <= 1:
|
|
167
|
+
return len(messages)
|
|
168
|
+
candidate = min(hard_msg_limit - 1, len(messages) - 1)
|
|
169
|
+
while candidate > 1 and count_tokens(_render_for_token_count(messages[:candidate])) >= hard_token_limit:
|
|
170
|
+
candidate = max(1, candidate // 2)
|
|
171
|
+
return candidate
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
async def _detect_boundaries(
|
|
175
|
+
messages: list[ChatMessage],
|
|
176
|
+
llm: LLMClient,
|
|
177
|
+
prompt_template: str,
|
|
178
|
+
) -> _BatchBoundaryResult:
|
|
179
|
+
"""Call the LLM once; return parsed boundaries.
|
|
180
|
+
|
|
181
|
+
Boundary indices validated to ``1 <= b < len(messages)`` and deduplicated.
|
|
182
|
+
|
|
183
|
+
Raises:
|
|
184
|
+
ValueError: If the LLM response cannot be parsed as a valid JSON object.
|
|
185
|
+
TypeError: If the ``boundaries`` field is not a list.
|
|
186
|
+
"""
|
|
187
|
+
messages_text = _format_messages_with_indices(messages)
|
|
188
|
+
rendered = render_prompt(prompt_template, None, messages=messages_text)
|
|
189
|
+
|
|
190
|
+
response = await llm.chat(
|
|
191
|
+
messages=[LLMChatMessage(role="user", content=rendered)],
|
|
192
|
+
response_format={"type": "json_object"},
|
|
193
|
+
)
|
|
194
|
+
result = _parse_batch_boundary_response(response.content)
|
|
195
|
+
valid = sorted({b for b in result.boundaries if 1 <= b < len(messages)})
|
|
196
|
+
return _BatchBoundaryResult(boundaries=valid, should_wait=result.should_wait)
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def _parse_batch_boundary_response(raw: str) -> _BatchBoundaryResult:
|
|
200
|
+
"""Parse LLM batch-boundary response.
|
|
201
|
+
|
|
202
|
+
Schema: ``{"boundaries": list[int], "should_wait": bool, ...}``.
|
|
203
|
+
Non-int boundary entries are silently skipped.
|
|
204
|
+
|
|
205
|
+
Raises:
|
|
206
|
+
ValueError: If the response cannot be parsed as a valid JSON object.
|
|
207
|
+
TypeError: If ``boundaries`` is not a list.
|
|
208
|
+
"""
|
|
209
|
+
data = parse_llm_json_object(raw)
|
|
210
|
+
|
|
211
|
+
raw_boundaries_val = data.get("boundaries", [])
|
|
212
|
+
if not isinstance(raw_boundaries_val, list):
|
|
213
|
+
raise TypeError("LLM boundary response 'boundaries' field is not a list")
|
|
214
|
+
boundaries: list[int] = [int(item) for item in cast("list[int | str]", raw_boundaries_val)]
|
|
215
|
+
|
|
216
|
+
return _BatchBoundaryResult(boundaries=boundaries, should_wait=bool(data.get("should_wait", False)))
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
def _make_cell(slice_msgs: list[ChatMessage]) -> MemCell:
|
|
220
|
+
"""Build a MemCell from a non-empty message slice."""
|
|
221
|
+
return MemCell(items=cast("list[ConversationItem]", slice_msgs), timestamp=slice_msgs[-1].timestamp)
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
"""Boundary extractor prompts (en + zh).
|
|
2
|
+
|
|
3
|
+
Each prompt is a module-level Python string constant per AGENTS.md §5 (Code Style) (no external .md / .yaml /
|
|
4
|
+
.toml prompt stores). Algorithm authors customize via per-call ``prompt=`` argument or by
|
|
5
|
+
monkey-patching the constant at startup.
|
|
6
|
+
"""
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""English boundary extractor prompts."""
|