everalgo-core 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- everalgo_core-0.1.0/.gitignore +58 -0
- everalgo_core-0.1.0/CHANGELOG.md +34 -0
- everalgo_core-0.1.0/LICENSE +21 -0
- everalgo_core-0.1.0/PKG-INFO +152 -0
- everalgo_core-0.1.0/README.md +127 -0
- everalgo_core-0.1.0/pyproject.toml +39 -0
- everalgo_core-0.1.0/src/everalgo/_tokenize.py +45 -0
- everalgo_core-0.1.0/src/everalgo/config.py +4 -0
- everalgo_core-0.1.0/src/everalgo/llm/__init__.py +64 -0
- everalgo_core-0.1.0/src/everalgo/llm/_filters.py +39 -0
- everalgo_core-0.1.0/src/everalgo/llm/config.py +21 -0
- everalgo_core-0.1.0/src/everalgo/llm/errors.py +5 -0
- everalgo_core-0.1.0/src/everalgo/llm/factory.py +15 -0
- everalgo_core-0.1.0/src/everalgo/llm/format.py +43 -0
- everalgo_core-0.1.0/src/everalgo/llm/parse.py +61 -0
- everalgo_core-0.1.0/src/everalgo/llm/protocols.py +28 -0
- everalgo_core-0.1.0/src/everalgo/llm/providers/__init__.py +7 -0
- everalgo_core-0.1.0/src/everalgo/llm/providers/openai_compat.py +91 -0
- everalgo_core-0.1.0/src/everalgo/llm/py.typed +0 -0
- everalgo_core-0.1.0/src/everalgo/llm/types.py +117 -0
- everalgo_core-0.1.0/src/everalgo/prompts/__init__.py +15 -0
- everalgo_core-0.1.0/src/everalgo/prompts/py.typed +0 -0
- everalgo_core-0.1.0/src/everalgo/prompts/render.py +40 -0
- everalgo_core-0.1.0/src/everalgo/prompts/validator.py +62 -0
- everalgo_core-0.1.0/src/everalgo/protocols.py +4 -0
- everalgo_core-0.1.0/src/everalgo/testing/__init__.py +36 -0
- everalgo_core-0.1.0/src/everalgo/testing/assertions.py +60 -0
- everalgo_core-0.1.0/src/everalgo/testing/fake_llm.py +135 -0
- everalgo_core-0.1.0/src/everalgo/testing/py.typed +0 -0
- everalgo_core-0.1.0/src/everalgo/types/__init__.py +80 -0
- everalgo_core-0.1.0/src/everalgo/types/_render.py +38 -0
- everalgo_core-0.1.0/src/everalgo/types/agent.py +97 -0
- everalgo_core-0.1.0/src/everalgo/types/chat.py +28 -0
- everalgo_core-0.1.0/src/everalgo/types/content.py +22 -0
- everalgo_core-0.1.0/src/everalgo/types/conversation.py +28 -0
- everalgo_core-0.1.0/src/everalgo/types/knowledge.py +13 -0
- everalgo_core-0.1.0/src/everalgo/types/memories.py +65 -0
- everalgo_core-0.1.0/src/everalgo/types/modality.py +322 -0
- everalgo_core-0.1.0/src/everalgo/types/parsed.py +46 -0
- everalgo_core-0.1.0/src/everalgo/types/py.typed +0 -0
- everalgo_core-0.1.0/src/everalgo/types/rank.py +98 -0
- everalgo_core-0.1.0/src/everalgo/types/raw.py +69 -0
- everalgo_core-0.1.0/tests/conftest.py +5 -0
- everalgo_core-0.1.0/tests/llm/providers/test_openai_compat.py +155 -0
- everalgo_core-0.1.0/tests/llm/test_config.py +66 -0
- everalgo_core-0.1.0/tests/llm/test_errors.py +30 -0
- everalgo_core-0.1.0/tests/llm/test_factory.py +43 -0
- everalgo_core-0.1.0/tests/llm/test_format.py +121 -0
- everalgo_core-0.1.0/tests/llm/test_multimodal_parts.py +93 -0
- everalgo_core-0.1.0/tests/llm/test_parse.py +57 -0
- everalgo_core-0.1.0/tests/llm/test_protocols.py +46 -0
- everalgo_core-0.1.0/tests/llm/test_public_api.py +75 -0
- everalgo_core-0.1.0/tests/llm/test_types.py +130 -0
- everalgo_core-0.1.0/tests/prompts/__init__.py +0 -0
- everalgo_core-0.1.0/tests/prompts/test_render.py +64 -0
- everalgo_core-0.1.0/tests/prompts/test_validator.py +79 -0
- everalgo_core-0.1.0/tests/test_tokenize.py +69 -0
- everalgo_core-0.1.0/tests/testing/test_assertions.py +248 -0
- everalgo_core-0.1.0/tests/testing/test_fake_llm.py +254 -0
- everalgo_core-0.1.0/tests/testing/test_testing_public_api.py +53 -0
- everalgo_core-0.1.0/tests/types/test_episode.py +66 -0
- everalgo_core-0.1.0/tests/types/test_modality.py +144 -0
- everalgo_core-0.1.0/tests/types/test_parsed.py +39 -0
- everalgo_core-0.1.0/tests/types/test_round_trip.py +47 -0
- everalgo_core-0.1.0/tests/types/test_stub_types.py +101 -0
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
# Byte-compiled / optimized / DLL files
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
*$py.class
|
|
5
|
+
|
|
6
|
+
# Distribution / packaging
|
|
7
|
+
build/
|
|
8
|
+
dist/
|
|
9
|
+
*.egg-info/
|
|
10
|
+
*.egg
|
|
11
|
+
wheels/
|
|
12
|
+
|
|
13
|
+
# Environments
|
|
14
|
+
.env
|
|
15
|
+
.env.local
|
|
16
|
+
.venv
|
|
17
|
+
env/
|
|
18
|
+
venv/
|
|
19
|
+
|
|
20
|
+
# uv
|
|
21
|
+
.uv/
|
|
22
|
+
|
|
23
|
+
# Test / coverage / cache
|
|
24
|
+
.pytest_cache/
|
|
25
|
+
.cache
|
|
26
|
+
.coverage
|
|
27
|
+
.coverage.*
|
|
28
|
+
coverage.xml
|
|
29
|
+
report.xml
|
|
30
|
+
htmlcov/
|
|
31
|
+
.tox/
|
|
32
|
+
.nox/
|
|
33
|
+
.hypothesis/
|
|
34
|
+
|
|
35
|
+
# Type checkers / linters
|
|
36
|
+
.mypy_cache/
|
|
37
|
+
.dmypy.json
|
|
38
|
+
.pyre/
|
|
39
|
+
.pytype/
|
|
40
|
+
.ruff_cache/
|
|
41
|
+
|
|
42
|
+
# Jupyter
|
|
43
|
+
.ipynb_checkpoints/
|
|
44
|
+
|
|
45
|
+
# IDE / OS
|
|
46
|
+
.vscode/
|
|
47
|
+
.idea/
|
|
48
|
+
*.swp
|
|
49
|
+
.DS_Store
|
|
50
|
+
|
|
51
|
+
# Logs
|
|
52
|
+
*.log
|
|
53
|
+
|
|
54
|
+
# Project local artifacts (placed under local/ per project convention)
|
|
55
|
+
local/
|
|
56
|
+
|
|
57
|
+
# Claude Code session-local state (scheduled tasks, agent caches, locks)
|
|
58
|
+
.claude/
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to this package are documented here. Format follows
|
|
4
|
+
[Keep a Changelog 1.1.0](https://keepachangelog.com/en/1.1.0/). Versioning
|
|
5
|
+
follows [Semantic Versioning 2.0](https://semver.org/spec/v2.0.0.html).
|
|
6
|
+
|
|
7
|
+
## [Unreleased]
|
|
8
|
+
|
|
9
|
+
### Added
|
|
10
|
+
|
|
11
|
+
- `MemCell` pydantic model: ordered `items: list[ConversationItem]` slice produced by boundary detection, plus `timestamp` (Unix epoch ms of the closing item).
|
|
12
|
+
- `ConversationItem` discriminated union (`ChatMessage | ToolCallRequest | ToolCallResult`) keyed on the `kind` field.
|
|
13
|
+
- `ChatMessage` with `kind="text"` discriminator, multimodal `content: str | list[ContentBlock]`, required `id`, `sender_id`, `sender_name`, and ms-epoch `timestamp`.
|
|
14
|
+
- `ContentBlock` / `TextContent` for multimodal message payloads.
|
|
15
|
+
- `Episode`, `Foresight`, `AtomicFact`, `Profile` user-memory output types with `ConfigDict(extra="allow")` so additional LLM-emitted keys are preserved without a schema bump.
|
|
16
|
+
- `AgentCase`, `AgentSkill` agent-memory output types; `ToolCall`, `ToolCallFunction`, `ToolCallRequest`, `ToolCallResult` agent-trajectory wire types.
|
|
17
|
+
- `Candidate`, `FactCandidate`, `ScoredItem`, `RankInput`, `RankOutput` ranking I/O types.
|
|
18
|
+
- `ParsedContent`, `RawData`, `RawFile`, `KnowledgeMemory` types reserved for the parser and knowledge distributions.
|
|
19
|
+
- `LLMClient` Protocol (async-first: `achat` / `achat_stream`; sync bridge via `asgiref.async_to_sync`).
|
|
20
|
+
- `ChatMessage` (LLM-layer), `ChatResponse`, `Usage` wire types for the LLM facade.
|
|
21
|
+
- `LLMConfig` pydantic model carrying model name, base URL, API key, and per-call timeout.
|
|
22
|
+
- `LLMError` base exception with provider-neutral subclass hierarchy.
|
|
23
|
+
- `build_client` factory that auto-detects provider from `LLMConfig.base_url`.
|
|
24
|
+
- `SensitiveHeadersFilter`: redacts authorization-style values from log `record.args` mappings; attached to the `everalgo.llm` logger by default (ADR-013).
|
|
25
|
+
- `render_prompt` helper: string-replace template rendering that tolerates literal `{` / `}` in template bodies without raising `KeyError`.
|
|
26
|
+
- `everalgo.testing.FakeLLMClient`: deterministic LLM replay via a call queue; raises `AssertionError` on unexpected calls.
|
|
27
|
+
- `everalgo.testing.CallRecord`: frozen record of a single fake LLM call (prompt, model, usage).
|
|
28
|
+
- `everalgo.testing.assert_episode_shape`, `assert_foresight_shape`, `assert_atomic_fact_shape`, `assert_profile_shape`: structural assertion helpers for memory output types.
|
|
29
|
+
|
|
30
|
+
### Changed
|
|
31
|
+
|
|
32
|
+
- LLM binding simplified to instance-only injection: `build_client` / constructor `llm=` parameter is the sole binding path. The prior 4-layer resolution (configure / use / current / resolve) was removed in favour of the pattern used by `openai-python`, `anthropic-sdk-python`, LangChain, and Instructor.
|
|
33
|
+
|
|
34
|
+
[Unreleased]: https://github.com/EverMind-AI/EverAlgo/compare/main...HEAD
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 EverMind
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: everalgo-core
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: EverAlgo foundation: types, LLM client + providers, prompt validators, testing helpers.
|
|
5
|
+
Project-URL: Homepage, https://github.com/EverMind-AI/EverAlgo
|
|
6
|
+
Project-URL: Repository, https://github.com/EverMind-AI/EverAlgo
|
|
7
|
+
Project-URL: Issues, https://github.com/EverMind-AI/EverAlgo/issues
|
|
8
|
+
Project-URL: Documentation, https://github.com/EverMind-AI/EverAlgo/tree/main/packages/everalgo-core
|
|
9
|
+
Project-URL: Changelog, https://github.com/EverMind-AI/EverAlgo/blob/main/packages/everalgo-core/CHANGELOG.md
|
|
10
|
+
Author: EverMind
|
|
11
|
+
License-Expression: MIT
|
|
12
|
+
License-File: LICENSE
|
|
13
|
+
Keywords: algorithm,everalgo,llm,memory
|
|
14
|
+
Classifier: Development Status :: 3 - Alpha
|
|
15
|
+
Classifier: Intended Audience :: Developers
|
|
16
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
17
|
+
Classifier: Programming Language :: Python :: 3
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
20
|
+
Requires-Python: >=3.12
|
|
21
|
+
Requires-Dist: openai>=1.0
|
|
22
|
+
Requires-Dist: pydantic>=2.7
|
|
23
|
+
Requires-Dist: tiktoken>=0.7
|
|
24
|
+
Description-Content-Type: text/markdown
|
|
25
|
+
|
|
26
|
+
# everalgo-core
|
|
27
|
+
|
|
28
|
+
Foundation distribution of EverAlgo — data types, LLM client + provider routing, prompt helpers, and testing utilities shared by every other `everalgo-*` package.
|
|
29
|
+
|
|
30
|
+
See the umbrella project: [EverAlgo monorepo](../../README.md) and the architecture document at [`docs/concepts/architecture.md`](../../docs/concepts/architecture.md).
|
|
31
|
+
|
|
32
|
+
## Install
|
|
33
|
+
|
|
34
|
+
```bash
|
|
35
|
+
pip install everalgo-core
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
## What this distribution provides
|
|
39
|
+
|
|
40
|
+
| Subpackage | What it contains |
|
|
41
|
+
|---|---|
|
|
42
|
+
| `everalgo.types` | All shared data types: `MemCell`, `ChatMessage`, `ConversationItem`, `Episode`, `Foresight`, `AtomicFact`, `Profile`, `AgentCase`, `AgentSkill`, `RankInput`, `RankOutput`, `Candidate`, `ScoredItem`, `ParsedContent`, `RawFile`, tool-call types |
|
|
43
|
+
| `everalgo.llm` | `LLMClient` Protocol, `LLMConfig`, `ChatMessage` / `ChatResponse` / `Usage`, `LLMError`, `build_client` factory |
|
|
44
|
+
| `everalgo.llm.format` | `format_message_timestamp`, `format_natural_language_time` — timestamp rendering for LLM prompts |
|
|
45
|
+
| `everalgo.llm.parse` | `parse_llm_json_object` — three-tier robust JSON-object extraction |
|
|
46
|
+
| `everalgo.llm.providers.openai_compat` | `OpenAICompatClient` — wraps `openai.AsyncOpenAI` |
|
|
47
|
+
| `everalgo.prompts` | `render_prompt` — shared template substitution helper |
|
|
48
|
+
| `everalgo.testing` | `FakeLLMClient`, `CallRecord`, and `assert_*_shape` structural assertion helpers |
|
|
49
|
+
|
|
50
|
+
## Quick start
|
|
51
|
+
|
|
52
|
+
```python
|
|
53
|
+
from everalgo.llm import LLMConfig, build_client
|
|
54
|
+
from everalgo.llm.types import ChatMessage
|
|
55
|
+
|
|
56
|
+
config = LLMConfig(model="gpt-4o-mini", api_key="sk-...", base_url="https://api.openai.com/v1")
|
|
57
|
+
client = build_client(config)
|
|
58
|
+
|
|
59
|
+
response = await client.chat(
|
|
60
|
+
messages=[ChatMessage(role="user", content="Hello")],
|
|
61
|
+
)
|
|
62
|
+
print(response.content)
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
### Using FakeLLMClient in tests
|
|
66
|
+
|
|
67
|
+
```python
|
|
68
|
+
import json
|
|
69
|
+
from everalgo.llm.types import ChatResponse
|
|
70
|
+
from everalgo.testing import FakeLLMClient
|
|
71
|
+
|
|
72
|
+
# Scripted mode — responses are returned in order
|
|
73
|
+
fake = FakeLLMClient(responses=[
|
|
74
|
+
ChatResponse(content=json.dumps({"title": "T", "content": "C"}), model="fake"),
|
|
75
|
+
])
|
|
76
|
+
|
|
77
|
+
# Handler mode — full control over each call
|
|
78
|
+
from everalgo.llm.types import ChatMessage as LLMMsg
|
|
79
|
+
def handler(messages: list[LLMMsg], **_) -> ChatResponse:
|
|
80
|
+
return ChatResponse(content="ok", model="fake")
|
|
81
|
+
|
|
82
|
+
fake = FakeLLMClient(handler=handler)
|
|
83
|
+
print(fake.call_count) # 0
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
## LLM utilities
|
|
87
|
+
|
|
88
|
+
`everalgo.llm.format` and `everalgo.llm.parse` provide prompt-rendering helpers used by boundary detection and every extractor.
|
|
89
|
+
|
|
90
|
+
### `everalgo.llm.format` — timestamp rendering
|
|
91
|
+
|
|
92
|
+
```python
|
|
93
|
+
from everalgo.llm.format import format_message_timestamp, format_natural_language_time
|
|
94
|
+
|
|
95
|
+
# ISO 8601 UTC anchor — used as inline message prefix in prompts
|
|
96
|
+
# e.g. "2023-11-14T22:13:20Z"
|
|
97
|
+
ts_str = format_message_timestamp(1_700_000_000_000)
|
|
98
|
+
|
|
99
|
+
# Human-readable label for LLM time-of-day reasoning
|
|
100
|
+
# Example output: "November 14, 2023 (Tuesday) at 10:13 PM UTC"
|
|
101
|
+
label = format_natural_language_time(1_700_000_000_000, lang="en")
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
### `everalgo.llm.parse` — robust JSON extraction
|
|
105
|
+
|
|
106
|
+
```python
|
|
107
|
+
from everalgo.llm.parse import parse_llm_json_object
|
|
108
|
+
|
|
109
|
+
# Three-tier fallback: fence block → direct loads → outermost braces
|
|
110
|
+
result = parse_llm_json_object('```json\n{"key": "value"}\n```')
|
|
111
|
+
# → {"key": "value"}
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
`parse_llm_json_object(raw: str) -> dict[str, Any]` tries each strategy in order and raises `ValueError` if all three fail.
|
|
115
|
+
|
|
116
|
+
### `everalgo._tokenize` — token counting
|
|
117
|
+
|
|
118
|
+
`count_tokens(text: str) → int` and `force_split(text: str, *, max_tokens: int) → list[str]` are module-private utilities (used internally by boundary algorithms; not part of the public `everalgo.*` surface). They use OpenAI `o200k_base` encoding via [`tiktoken`](https://github.com/openai/tiktoken).
|
|
119
|
+
|
|
120
|
+
## API surface
|
|
121
|
+
|
|
122
|
+
| Symbol | Module | Role |
|
|
123
|
+
|---|---|---|
|
|
124
|
+
| `MemCell` | `everalgo.types` | Boundary-detected conversation slice; `items: list[ConversationItem]`, `timestamp: int` |
|
|
125
|
+
| `ChatMessage` | `everalgo.types` | Chat wire type: `id`, `role`, `content`, `timestamp`, `sender_id`, `sender_name` |
|
|
126
|
+
| `ConversationItem` | `everalgo.types` | `ChatMessage \| ToolCallRequest \| ToolCallResult` discriminated union |
|
|
127
|
+
| `Episode` / `Foresight` / `AtomicFact` / `Profile` | `everalgo.types` | User-side memory output types |
|
|
128
|
+
| `AgentCase` / `AgentSkill` | `everalgo.types` | Agent-side memory output types |
|
|
129
|
+
| `RankInput` / `RankOutput` / `Candidate` / `ScoredItem` | `everalgo.types` | Rank I/O contracts |
|
|
130
|
+
| `LLMClient` | `everalgo.llm` | Protocol — `chat(messages, *, model=None, ...) → ChatResponse` |
|
|
131
|
+
| `LLMConfig` | `everalgo.llm` | `model`, `api_key` (SecretStr), `base_url`, `temperature`, `max_tokens`, `timeout` |
|
|
132
|
+
| `build_client` | `everalgo.llm` | Factory: `LLMConfig → LLMClient` (returns `OpenAICompatClient`) |
|
|
133
|
+
| `LLMError` | `everalgo.llm` | Base exception raised by all providers |
|
|
134
|
+
| `OpenAICompatClient` | `everalgo.llm.providers.openai_compat` | Wraps `openai.AsyncOpenAI`; no retry logic |
|
|
135
|
+
| `format_message_timestamp` | `everalgo.llm.format` | ISO 8601 UTC string from a millisecond timestamp |
|
|
136
|
+
| `format_natural_language_time` | `everalgo.llm.format` | Human-readable timestamp label; `lang="en"` or `"zh"` |
|
|
137
|
+
| `parse_llm_json_object` | `everalgo.llm.parse` | Robust JSON-object parser with three-tier fallback |
|
|
138
|
+
| `render_prompt` | `everalgo.prompts` | Template substitution for `{placeholder}` patterns |
|
|
139
|
+
| `FakeLLMClient` | `everalgo.testing` | In-memory `LLMClient` double; `responses=[...]` or `handler=callable` |
|
|
140
|
+
| `CallRecord` | `everalgo.testing` | Single recorded `chat()` invocation (for test assertions) |
|
|
141
|
+
| `assert_episode_shape` | `everalgo.testing` | Structural assertion: required `Episode` fields are non-empty |
|
|
142
|
+
| `assert_foresight_shape` | `everalgo.testing` | Structural assertion for `Foresight` |
|
|
143
|
+
| `assert_atomic_fact_shape` | `everalgo.testing` | Structural assertion for `AtomicFact` |
|
|
144
|
+
| `assert_profile_shape` | `everalgo.testing` | Structural assertion for `Profile` |
|
|
145
|
+
|
|
146
|
+
## Related distributions
|
|
147
|
+
|
|
148
|
+
- [`everalgo-boundary`](../everalgo-boundary/) — uses `MemCell`, `ChatMessage`, `LLMClient`
|
|
149
|
+
- [`everalgo-clustering`](../everalgo-clustering/) — uses `MemCell`, `LLMClient`
|
|
150
|
+
- [`everalgo-rank`](../everalgo-rank/) — uses `RankInput`, `RankOutput`, `Candidate`, `ScoredItem`
|
|
151
|
+
- [`everalgo-user-memory`](../everalgo-user-memory/) — uses all user-side types + `LLMClient`
|
|
152
|
+
- [`everalgo-agent-memory`](../everalgo-agent-memory/) — uses agent-side types + `LLMClient`
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
# everalgo-core
|
|
2
|
+
|
|
3
|
+
Foundation distribution of EverAlgo — data types, LLM client + provider routing, prompt helpers, and testing utilities shared by every other `everalgo-*` package.
|
|
4
|
+
|
|
5
|
+
See the umbrella project: [EverAlgo monorepo](../../README.md) and the architecture document at [`docs/concepts/architecture.md`](../../docs/concepts/architecture.md).
|
|
6
|
+
|
|
7
|
+
## Install
|
|
8
|
+
|
|
9
|
+
```bash
|
|
10
|
+
pip install everalgo-core
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
## What this distribution provides
|
|
14
|
+
|
|
15
|
+
| Subpackage | What it contains |
|
|
16
|
+
|---|---|
|
|
17
|
+
| `everalgo.types` | All shared data types: `MemCell`, `ChatMessage`, `ConversationItem`, `Episode`, `Foresight`, `AtomicFact`, `Profile`, `AgentCase`, `AgentSkill`, `RankInput`, `RankOutput`, `Candidate`, `ScoredItem`, `ParsedContent`, `RawFile`, tool-call types |
|
|
18
|
+
| `everalgo.llm` | `LLMClient` Protocol, `LLMConfig`, `ChatMessage` / `ChatResponse` / `Usage`, `LLMError`, `build_client` factory |
|
|
19
|
+
| `everalgo.llm.format` | `format_message_timestamp`, `format_natural_language_time` — timestamp rendering for LLM prompts |
|
|
20
|
+
| `everalgo.llm.parse` | `parse_llm_json_object` — three-tier robust JSON-object extraction |
|
|
21
|
+
| `everalgo.llm.providers.openai_compat` | `OpenAICompatClient` — wraps `openai.AsyncOpenAI` |
|
|
22
|
+
| `everalgo.prompts` | `render_prompt` — shared template substitution helper |
|
|
23
|
+
| `everalgo.testing` | `FakeLLMClient`, `CallRecord`, and `assert_*_shape` structural assertion helpers |
|
|
24
|
+
|
|
25
|
+
## Quick start
|
|
26
|
+
|
|
27
|
+
```python
|
|
28
|
+
from everalgo.llm import LLMConfig, build_client
|
|
29
|
+
from everalgo.llm.types import ChatMessage
|
|
30
|
+
|
|
31
|
+
config = LLMConfig(model="gpt-4o-mini", api_key="sk-...", base_url="https://api.openai.com/v1")
|
|
32
|
+
client = build_client(config)
|
|
33
|
+
|
|
34
|
+
response = await client.chat(
|
|
35
|
+
messages=[ChatMessage(role="user", content="Hello")],
|
|
36
|
+
)
|
|
37
|
+
print(response.content)
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
### Using FakeLLMClient in tests
|
|
41
|
+
|
|
42
|
+
```python
|
|
43
|
+
import json
|
|
44
|
+
from everalgo.llm.types import ChatResponse
|
|
45
|
+
from everalgo.testing import FakeLLMClient
|
|
46
|
+
|
|
47
|
+
# Scripted mode — responses are returned in order
|
|
48
|
+
fake = FakeLLMClient(responses=[
|
|
49
|
+
ChatResponse(content=json.dumps({"title": "T", "content": "C"}), model="fake"),
|
|
50
|
+
])
|
|
51
|
+
|
|
52
|
+
# Handler mode — full control over each call
|
|
53
|
+
from everalgo.llm.types import ChatMessage as LLMMsg
|
|
54
|
+
def handler(messages: list[LLMMsg], **_) -> ChatResponse:
|
|
55
|
+
return ChatResponse(content="ok", model="fake")
|
|
56
|
+
|
|
57
|
+
fake = FakeLLMClient(handler=handler)
|
|
58
|
+
print(fake.call_count) # 0
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
## LLM utilities
|
|
62
|
+
|
|
63
|
+
`everalgo.llm.format` and `everalgo.llm.parse` provide prompt-rendering helpers used by boundary detection and every extractor.
|
|
64
|
+
|
|
65
|
+
### `everalgo.llm.format` — timestamp rendering
|
|
66
|
+
|
|
67
|
+
```python
|
|
68
|
+
from everalgo.llm.format import format_message_timestamp, format_natural_language_time
|
|
69
|
+
|
|
70
|
+
# ISO 8601 UTC anchor — used as inline message prefix in prompts
|
|
71
|
+
# e.g. "2023-11-14T22:13:20Z"
|
|
72
|
+
ts_str = format_message_timestamp(1_700_000_000_000)
|
|
73
|
+
|
|
74
|
+
# Human-readable label for LLM time-of-day reasoning
|
|
75
|
+
# Example output: "November 14, 2023 (Tuesday) at 10:13 PM UTC"
|
|
76
|
+
label = format_natural_language_time(1_700_000_000_000, lang="en")
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
### `everalgo.llm.parse` — robust JSON extraction
|
|
80
|
+
|
|
81
|
+
```python
|
|
82
|
+
from everalgo.llm.parse import parse_llm_json_object
|
|
83
|
+
|
|
84
|
+
# Three-tier fallback: fence block → direct loads → outermost braces
|
|
85
|
+
result = parse_llm_json_object('```json\n{"key": "value"}\n```')
|
|
86
|
+
# → {"key": "value"}
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
`parse_llm_json_object(raw: str) -> dict[str, Any]` tries each strategy in order and raises `ValueError` if all three fail.
|
|
90
|
+
|
|
91
|
+
### `everalgo._tokenize` — token counting
|
|
92
|
+
|
|
93
|
+
`count_tokens(text: str) → int` and `force_split(text: str, *, max_tokens: int) → list[str]` are module-private utilities (used internally by boundary algorithms; not part of the public `everalgo.*` surface). They use OpenAI `o200k_base` encoding via [`tiktoken`](https://github.com/openai/tiktoken).
|
|
94
|
+
|
|
95
|
+
## API surface
|
|
96
|
+
|
|
97
|
+
| Symbol | Module | Role |
|
|
98
|
+
|---|---|---|
|
|
99
|
+
| `MemCell` | `everalgo.types` | Boundary-detected conversation slice; `items: list[ConversationItem]`, `timestamp: int` |
|
|
100
|
+
| `ChatMessage` | `everalgo.types` | Chat wire type: `id`, `role`, `content`, `timestamp`, `sender_id`, `sender_name` |
|
|
101
|
+
| `ConversationItem` | `everalgo.types` | `ChatMessage \| ToolCallRequest \| ToolCallResult` discriminated union |
|
|
102
|
+
| `Episode` / `Foresight` / `AtomicFact` / `Profile` | `everalgo.types` | User-side memory output types |
|
|
103
|
+
| `AgentCase` / `AgentSkill` | `everalgo.types` | Agent-side memory output types |
|
|
104
|
+
| `RankInput` / `RankOutput` / `Candidate` / `ScoredItem` | `everalgo.types` | Rank I/O contracts |
|
|
105
|
+
| `LLMClient` | `everalgo.llm` | Protocol — `chat(messages, *, model=None, ...) → ChatResponse` |
|
|
106
|
+
| `LLMConfig` | `everalgo.llm` | `model`, `api_key` (SecretStr), `base_url`, `temperature`, `max_tokens`, `timeout` |
|
|
107
|
+
| `build_client` | `everalgo.llm` | Factory: `LLMConfig → LLMClient` (returns `OpenAICompatClient`) |
|
|
108
|
+
| `LLMError` | `everalgo.llm` | Base exception raised by all providers |
|
|
109
|
+
| `OpenAICompatClient` | `everalgo.llm.providers.openai_compat` | Wraps `openai.AsyncOpenAI`; no retry logic |
|
|
110
|
+
| `format_message_timestamp` | `everalgo.llm.format` | ISO 8601 UTC string from a millisecond timestamp |
|
|
111
|
+
| `format_natural_language_time` | `everalgo.llm.format` | Human-readable timestamp label; `lang="en"` or `"zh"` |
|
|
112
|
+
| `parse_llm_json_object` | `everalgo.llm.parse` | Robust JSON-object parser with three-tier fallback |
|
|
113
|
+
| `render_prompt` | `everalgo.prompts` | Template substitution for `{placeholder}` patterns |
|
|
114
|
+
| `FakeLLMClient` | `everalgo.testing` | In-memory `LLMClient` double; `responses=[...]` or `handler=callable` |
|
|
115
|
+
| `CallRecord` | `everalgo.testing` | Single recorded `chat()` invocation (for test assertions) |
|
|
116
|
+
| `assert_episode_shape` | `everalgo.testing` | Structural assertion: required `Episode` fields are non-empty |
|
|
117
|
+
| `assert_foresight_shape` | `everalgo.testing` | Structural assertion for `Foresight` |
|
|
118
|
+
| `assert_atomic_fact_shape` | `everalgo.testing` | Structural assertion for `AtomicFact` |
|
|
119
|
+
| `assert_profile_shape` | `everalgo.testing` | Structural assertion for `Profile` |
|
|
120
|
+
|
|
121
|
+
## Related distributions
|
|
122
|
+
|
|
123
|
+
- [`everalgo-boundary`](../everalgo-boundary/) — uses `MemCell`, `ChatMessage`, `LLMClient`
|
|
124
|
+
- [`everalgo-clustering`](../everalgo-clustering/) — uses `MemCell`, `LLMClient`
|
|
125
|
+
- [`everalgo-rank`](../everalgo-rank/) — uses `RankInput`, `RankOutput`, `Candidate`, `ScoredItem`
|
|
126
|
+
- [`everalgo-user-memory`](../everalgo-user-memory/) — uses all user-side types + `LLMClient`
|
|
127
|
+
- [`everalgo-agent-memory`](../everalgo-agent-memory/) — uses agent-side types + `LLMClient`
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "everalgo-core"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "EverAlgo foundation: types, LLM client + providers, prompt validators, testing helpers."
|
|
9
|
+
requires-python = ">=3.12"
|
|
10
|
+
license = "MIT"
|
|
11
|
+
readme = "README.md"
|
|
12
|
+
authors = [{ name = "EverMind" }]
|
|
13
|
+
keywords = ["llm", "memory", "algorithm", "everalgo"]
|
|
14
|
+
classifiers = [
|
|
15
|
+
"Development Status :: 3 - Alpha",
|
|
16
|
+
"Intended Audience :: Developers",
|
|
17
|
+
"License :: OSI Approved :: MIT License",
|
|
18
|
+
"Programming Language :: Python :: 3",
|
|
19
|
+
"Programming Language :: Python :: 3.12",
|
|
20
|
+
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
|
21
|
+
]
|
|
22
|
+
dependencies = [
|
|
23
|
+
"openai>=1.0",
|
|
24
|
+
"pydantic>=2.7",
|
|
25
|
+
"tiktoken>=0.7",
|
|
26
|
+
]
|
|
27
|
+
|
|
28
|
+
[project.urls]
|
|
29
|
+
Homepage = "https://github.com/EverMind-AI/EverAlgo"
|
|
30
|
+
Repository = "https://github.com/EverMind-AI/EverAlgo"
|
|
31
|
+
Issues = "https://github.com/EverMind-AI/EverAlgo/issues"
|
|
32
|
+
Documentation = "https://github.com/EverMind-AI/EverAlgo/tree/main/packages/everalgo-core"
|
|
33
|
+
Changelog = "https://github.com/EverMind-AI/EverAlgo/blob/main/packages/everalgo-core/CHANGELOG.md"
|
|
34
|
+
|
|
35
|
+
# `everalgo` here is a PEP 420 namespace; hatchling packages the directory
|
|
36
|
+
# tree under `src/everalgo/` belonging to this distribution only.
|
|
37
|
+
# Reference: https://github.com/pydantic/pydantic-ai/blob/main/pydantic_ai_slim/pyproject.toml
|
|
38
|
+
[tool.hatch.build.targets.wheel]
|
|
39
|
+
packages = ["src/everalgo"]
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
"""Shared tokenizer utilities using OpenAI ``o200k_base`` encoding (GPT-4o / o-series).
|
|
2
|
+
|
|
3
|
+
Module-private — internal shared implementation; not part of the public ``everalgo.*`` API.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import tiktoken
|
|
9
|
+
|
|
10
|
+
_ENCODING_NAME = "o200k_base"
|
|
11
|
+
_tokenizer_cache: tiktoken.Encoding | None = None
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def _get_tokenizer() -> tiktoken.Encoding:
|
|
15
|
+
"""Return the shared ``o200k_base`` encoding, initialising on first call."""
|
|
16
|
+
global _tokenizer_cache
|
|
17
|
+
if _tokenizer_cache is None:
|
|
18
|
+
_tokenizer_cache = tiktoken.get_encoding(_ENCODING_NAME)
|
|
19
|
+
return _tokenizer_cache
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def count_tokens(text: str) -> int:
|
|
23
|
+
"""Count tokens in ``text`` under ``o200k_base``. Empty string returns 0."""
|
|
24
|
+
if not text:
|
|
25
|
+
return 0
|
|
26
|
+
return len(_get_tokenizer().encode(text))
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def force_split(text: str, *, max_tokens: int) -> list[str]:
|
|
30
|
+
"""Split ``text`` into chunks of at most ``max_tokens`` tokens (no semantic awareness).
|
|
31
|
+
|
|
32
|
+
Returns ``[]`` for empty input, ``[text]`` when the whole string already fits.
|
|
33
|
+
|
|
34
|
+
Raises:
|
|
35
|
+
ValueError: If ``max_tokens <= 0``.
|
|
36
|
+
"""
|
|
37
|
+
if max_tokens <= 0:
|
|
38
|
+
raise ValueError(f"max_tokens must be positive, got {max_tokens}")
|
|
39
|
+
if not text:
|
|
40
|
+
return []
|
|
41
|
+
tokenizer = _get_tokenizer()
|
|
42
|
+
token_ids = tokenizer.encode(text)
|
|
43
|
+
if len(token_ids) <= max_tokens:
|
|
44
|
+
return [text]
|
|
45
|
+
return [tokenizer.decode(token_ids[i : i + max_tokens]) for i in range(0, len(token_ids), max_tokens)]
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
"""LLM facade — chat-style abstraction over OpenAI-compatible providers.
|
|
2
|
+
|
|
3
|
+
Public surface, grouped by role:
|
|
4
|
+
|
|
5
|
+
- protocol: LLMClient
|
|
6
|
+
- data: ChatMessage, ChatResponse, ContentPart, ImageUrlInner, ImageUrlPart, TextPart, Usage, LLMConfig
|
|
7
|
+
- helpers: image_url_part_from_bytes
|
|
8
|
+
- errors: LLMError
|
|
9
|
+
- factory: build_client
|
|
10
|
+
- format/parse: Lang, format_message_timestamp, format_natural_language_time, parse_llm_json_object
|
|
11
|
+
|
|
12
|
+
LLM binding is instance-only: every operator accepts ``llm: LLMClient`` at construction time.
|
|
13
|
+
There is no global default, no scoped context-var override, and no per-call resolution layer.
|
|
14
|
+
This matches the industry convention used by openai-python, anthropic-sdk-python, LangChain,
|
|
15
|
+
Instructor, and scikit-learn (5/7 dominant SDKs bind the client at construction time only).
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import logging
|
|
21
|
+
|
|
22
|
+
from everalgo.llm._filters import SensitiveHeadersFilter
|
|
23
|
+
from everalgo.llm.config import LLMConfig
|
|
24
|
+
from everalgo.llm.errors import LLMError
|
|
25
|
+
from everalgo.llm.factory import build_client
|
|
26
|
+
from everalgo.llm.format import Lang, format_message_timestamp, format_natural_language_time
|
|
27
|
+
from everalgo.llm.parse import parse_llm_json_object
|
|
28
|
+
from everalgo.llm.protocols import LLMClient
|
|
29
|
+
from everalgo.llm.types import (
|
|
30
|
+
ChatMessage,
|
|
31
|
+
ChatResponse,
|
|
32
|
+
ContentPart,
|
|
33
|
+
ImageUrlInner,
|
|
34
|
+
ImageUrlPart,
|
|
35
|
+
TextPart,
|
|
36
|
+
Usage,
|
|
37
|
+
image_url_part_from_bytes,
|
|
38
|
+
)
|
|
39
|
+
|
|
40
|
+
# Library logging setup (ADR-013). NullHandler suppresses "No handlers" noise
|
|
41
|
+
# under Python's library-logging contract; SensitiveHeadersFilter redacts
|
|
42
|
+
# authorization-style values from `record.args` mappings, default-on.
|
|
43
|
+
_logger = logging.getLogger(__name__)
|
|
44
|
+
_logger.addHandler(logging.NullHandler())
|
|
45
|
+
_logger.addFilter(SensitiveHeadersFilter())
|
|
46
|
+
|
|
47
|
+
__all__ = [
|
|
48
|
+
"ChatMessage",
|
|
49
|
+
"ChatResponse",
|
|
50
|
+
"ContentPart",
|
|
51
|
+
"ImageUrlInner",
|
|
52
|
+
"ImageUrlPart",
|
|
53
|
+
"LLMClient",
|
|
54
|
+
"LLMConfig",
|
|
55
|
+
"LLMError",
|
|
56
|
+
"Lang",
|
|
57
|
+
"TextPart",
|
|
58
|
+
"Usage",
|
|
59
|
+
"build_client",
|
|
60
|
+
"format_message_timestamp",
|
|
61
|
+
"format_natural_language_time",
|
|
62
|
+
"image_url_part_from_bytes",
|
|
63
|
+
"parse_llm_json_object",
|
|
64
|
+
] # fmt: skip
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
"""Logging filters for ``everalgo.llm`` — redacts sensitive header values."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import logging
|
|
6
|
+
import re
|
|
7
|
+
from typing import Final
|
|
8
|
+
|
|
9
|
+
__all__ = ["SensitiveHeadersFilter"]
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
_REDACTED: Final[str] = "<redacted>"
|
|
13
|
+
|
|
14
|
+
# Matches header / parameter names that conventionally carry credentials.
|
|
15
|
+
# Case-insensitive; covers Authorization, Api-Key / api_key, X-Api-Key,
|
|
16
|
+
# Bearer tokens. Keep narrow — false positives leak data; false negatives
|
|
17
|
+
# leak credentials.
|
|
18
|
+
_SENSITIVE_KEY_PATTERN: Final[re.Pattern[str]] = re.compile(
|
|
19
|
+
r"(authorization|api[-_]?key|x-api-key|bearer)",
|
|
20
|
+
re.IGNORECASE,
|
|
21
|
+
)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class SensitiveHeadersFilter(logging.Filter):
|
|
25
|
+
"""Redact sensitive header values in ``LogRecord.args`` mappings.
|
|
26
|
+
|
|
27
|
+
Default-attached to ``everalgo.llm``; ``filter()`` always returns ``True`` — it mutates payload, never
|
|
28
|
+
drops records. Only mapping-shaped ``record.args`` are scanned; tuple / ``None`` args pass through.
|
|
29
|
+
Does not scan request/response bodies (body PII is out of scope for a header filter).
|
|
30
|
+
"""
|
|
31
|
+
|
|
32
|
+
def filter(self, record: logging.LogRecord) -> bool:
|
|
33
|
+
"""Redact sensitive entries from ``record.args`` in place; always returns ``True``."""
|
|
34
|
+
if isinstance(record.args, dict):
|
|
35
|
+
record.args = {
|
|
36
|
+
key: (_REDACTED if _SENSITIVE_KEY_PATTERN.search(str(key)) else value)
|
|
37
|
+
for key, value in record.args.items()
|
|
38
|
+
}
|
|
39
|
+
return True
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
"""LLM client configuration."""
|
|
2
|
+
|
|
3
|
+
from typing import Any
|
|
4
|
+
|
|
5
|
+
from pydantic import BaseModel, Field, SecretStr
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class LLMConfig(BaseModel):
|
|
9
|
+
"""OpenAI-compatible LLM client configuration.
|
|
10
|
+
|
|
11
|
+
``api_key`` uses ``SecretStr`` so repr / serialization never leaks the raw key; call
|
|
12
|
+
``config.api_key.get_secret_value()`` explicitly when passing it to an SDK.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
model: str
|
|
16
|
+
api_key: SecretStr
|
|
17
|
+
base_url: str
|
|
18
|
+
temperature: float = 0.0
|
|
19
|
+
max_tokens: int | None = None
|
|
20
|
+
timeout: float = 60.0
|
|
21
|
+
extra: dict[str, Any] = Field(default_factory=dict)
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
"""Factory for building an LLM client from configuration."""
|
|
2
|
+
|
|
3
|
+
from everalgo.llm.config import LLMConfig
|
|
4
|
+
from everalgo.llm.protocols import LLMClient
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def build_client(config: LLMConfig) -> LLMClient:
|
|
8
|
+
"""Build an OpenAI-compatible LLM client.
|
|
9
|
+
|
|
10
|
+
``OpenAICompatClient`` is imported lazily so that ``import everalgo.llm`` remains cheap for callers
|
|
11
|
+
that only need the Protocol / Config / Error types. Do not hoist this to a top-level import.
|
|
12
|
+
"""
|
|
13
|
+
from everalgo.llm.providers.openai_compat import OpenAICompatClient
|
|
14
|
+
|
|
15
|
+
return OpenAICompatClient(config)
|