langchain-curt 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- langchain_curt-0.1.0/LICENSE +21 -0
- langchain_curt-0.1.0/PKG-INFO +86 -0
- langchain_curt-0.1.0/README.md +43 -0
- langchain_curt-0.1.0/pyproject.toml +40 -0
- langchain_curt-0.1.0/src/langchain_curt/__init__.py +19 -0
- langchain_curt-0.1.0/src/langchain_curt/curt_chat.py +293 -0
- langchain_curt-0.1.0/tests/test_curt_chat.py +288 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025-2026 Jordan Newell
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: langchain-curt
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: LangChain chat loader for .curt conversation files (.curt v1 — open single-file AI conversation format)
|
|
5
|
+
Project-URL: Homepage, https://github.com/JordanNewell/langchain-curt
|
|
6
|
+
Project-URL: Repository, https://github.com/JordanNewell/langchain-curt
|
|
7
|
+
Project-URL: Documentation, https://github.com/JordanNewell/dot_curt
|
|
8
|
+
Author: Jordan Newell
|
|
9
|
+
License: MIT License
|
|
10
|
+
|
|
11
|
+
Copyright (c) 2025-2026 Jordan Newell
|
|
12
|
+
|
|
13
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
14
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
15
|
+
in the Software without restriction, including without limitation the rights
|
|
16
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
17
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
18
|
+
furnished to do so, subject to the following conditions:
|
|
19
|
+
|
|
20
|
+
The above copyright notice and this permission notice shall be included in all
|
|
21
|
+
copies or substantial portions of the Software.
|
|
22
|
+
|
|
23
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
24
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
25
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
26
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
27
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
28
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
29
|
+
SOFTWARE.
|
|
30
|
+
License-File: LICENSE
|
|
31
|
+
Classifier: Development Status :: 4 - Beta
|
|
32
|
+
Classifier: Intended Audience :: Developers
|
|
33
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
34
|
+
Classifier: Programming Language :: Python :: 3
|
|
35
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
36
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
37
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
38
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
39
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
40
|
+
Requires-Python: >=3.10
|
|
41
|
+
Requires-Dist: langchain-core<2.0,>=0.3
|
|
42
|
+
Description-Content-Type: text/markdown
|
|
43
|
+
|
|
44
|
+
# langchain-curt
|
|
45
|
+
|
|
46
|
+
[LangChain](https://github.com/langchain-ai/langchain) chat loader for
|
|
47
|
+
[`.curt`](https://github.com/JordanNewell/dot_curt) — the open (MIT),
|
|
48
|
+
single-file AI conversation format. One timestamped, attributed conversation
|
|
49
|
+
per file; unknown fields are data and survive round-trips.
|
|
50
|
+
|
|
51
|
+
```python
|
|
52
|
+
from langchain_curt import CurtChatLoader
|
|
53
|
+
|
|
54
|
+
loader = CurtChatLoader("./conversations/*.curt")
|
|
55
|
+
sessions = loader.lazy_load() # one ChatSession per .curt file
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
- **Role mapping:** `user` → `HumanMessage`, `assistant` → `AIMessage`,
|
|
59
|
+
`system` → `SystemMessage`. `tool` messages are skipped and counted
|
|
60
|
+
(`loader.stats["tool_messages_skipped"]`) — `ToolMessage` requires a
|
|
61
|
+
`tool_call_id` that `.curt` v1 does not carry.
|
|
62
|
+
- **Metadata:** per-message timestamps (epoch ms → ISO 8601 UTC), model, and
|
|
63
|
+
all unknown/extension fields under `additional_kwargs["curt"]`;
|
|
64
|
+
conversation-level metadata in `loader.conversation_metadata`.
|
|
65
|
+
- **Validation:** spec-compliant whole-file rejection (bad magic, wrong
|
|
66
|
+
version, missing fields → rejected, never half-loaded). `on_error="skip"`
|
|
67
|
+
imports valid files and reports invalid ones in `loader.errors`.
|
|
68
|
+
- **Dependencies:** `langchain-core` only. The `.curt` parser is stdlib-only.
|
|
69
|
+
|
|
70
|
+
Converters into `.curt` exist for Claude Code transcripts
|
|
71
|
+
([curt-convert](https://github.com/JordanNewell/curt-convert)), Open WebUI
|
|
72
|
+
exports ([curt-openwebui](https://github.com/JordanNewell/curt-openwebui)),
|
|
73
|
+
and the WildChat-1M research corpus
|
|
74
|
+
([curt-wildchat](https://github.com/JordanNewell/curt-wildchat)).
|
|
75
|
+
|
|
76
|
+
Spec: https://github.com/JordanNewell/dot_curt · npm reference impl:
|
|
77
|
+
[`curt-format`](https://www.npmjs.com/package/curt-format)
|
|
78
|
+
|
|
79
|
+
## Development
|
|
80
|
+
|
|
81
|
+
```sh
|
|
82
|
+
pip install -e . pytest
|
|
83
|
+
pytest
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
MIT License.
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
# langchain-curt
|
|
2
|
+
|
|
3
|
+
[LangChain](https://github.com/langchain-ai/langchain) chat loader for
|
|
4
|
+
[`.curt`](https://github.com/JordanNewell/dot_curt) — the open (MIT),
|
|
5
|
+
single-file AI conversation format. One timestamped, attributed conversation
|
|
6
|
+
per file; unknown fields are data and survive round-trips.
|
|
7
|
+
|
|
8
|
+
```python
|
|
9
|
+
from langchain_curt import CurtChatLoader
|
|
10
|
+
|
|
11
|
+
loader = CurtChatLoader("./conversations/*.curt")
|
|
12
|
+
sessions = loader.lazy_load() # one ChatSession per .curt file
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
- **Role mapping:** `user` → `HumanMessage`, `assistant` → `AIMessage`,
|
|
16
|
+
`system` → `SystemMessage`. `tool` messages are skipped and counted
|
|
17
|
+
(`loader.stats["tool_messages_skipped"]`) — `ToolMessage` requires a
|
|
18
|
+
`tool_call_id` that `.curt` v1 does not carry.
|
|
19
|
+
- **Metadata:** per-message timestamps (epoch ms → ISO 8601 UTC), model, and
|
|
20
|
+
all unknown/extension fields under `additional_kwargs["curt"]`;
|
|
21
|
+
conversation-level metadata in `loader.conversation_metadata`.
|
|
22
|
+
- **Validation:** spec-compliant whole-file rejection (bad magic, wrong
|
|
23
|
+
version, missing fields → rejected, never half-loaded). `on_error="skip"`
|
|
24
|
+
imports valid files and reports invalid ones in `loader.errors`.
|
|
25
|
+
- **Dependencies:** `langchain-core` only. The `.curt` parser is stdlib-only.
|
|
26
|
+
|
|
27
|
+
Converters into `.curt` exist for Claude Code transcripts
|
|
28
|
+
([curt-convert](https://github.com/JordanNewell/curt-convert)), Open WebUI
|
|
29
|
+
exports ([curt-openwebui](https://github.com/JordanNewell/curt-openwebui)),
|
|
30
|
+
and the WildChat-1M research corpus
|
|
31
|
+
([curt-wildchat](https://github.com/JordanNewell/curt-wildchat)).
|
|
32
|
+
|
|
33
|
+
Spec: https://github.com/JordanNewell/dot_curt · npm reference impl:
|
|
34
|
+
[`curt-format`](https://www.npmjs.com/package/curt-format)
|
|
35
|
+
|
|
36
|
+
## Development
|
|
37
|
+
|
|
38
|
+
```sh
|
|
39
|
+
pip install -e . pytest
|
|
40
|
+
pytest
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
MIT License.
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "langchain-curt"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "LangChain chat loader for .curt conversation files (.curt v1 — open single-file AI conversation format)"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = { file = "LICENSE" }
|
|
11
|
+
authors = [{ name = "Jordan Newell" }]
|
|
12
|
+
requires-python = ">=3.10"
|
|
13
|
+
dependencies = ["langchain-core>=0.3,<2.0"]
|
|
14
|
+
classifiers = [
|
|
15
|
+
"Development Status :: 4 - Beta",
|
|
16
|
+
"Intended Audience :: Developers",
|
|
17
|
+
"License :: OSI Approved :: MIT License",
|
|
18
|
+
"Programming Language :: Python :: 3",
|
|
19
|
+
"Programming Language :: Python :: 3.10",
|
|
20
|
+
"Programming Language :: Python :: 3.11",
|
|
21
|
+
"Programming Language :: Python :: 3.12",
|
|
22
|
+
"Programming Language :: Python :: 3.13",
|
|
23
|
+
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
|
24
|
+
]
|
|
25
|
+
|
|
26
|
+
[project.urls]
|
|
27
|
+
Homepage = "https://github.com/JordanNewell/langchain-curt"
|
|
28
|
+
Repository = "https://github.com/JordanNewell/langchain-curt"
|
|
29
|
+
Documentation = "https://github.com/JordanNewell/dot_curt"
|
|
30
|
+
|
|
31
|
+
[tool.hatch.build.targets.wheel]
|
|
32
|
+
packages = ["src/langchain_curt"]
|
|
33
|
+
|
|
34
|
+
[tool.pytest.ini_options]
|
|
35
|
+
addopts = "-ra"
|
|
36
|
+
testpaths = ["tests"]
|
|
37
|
+
|
|
38
|
+
[tool.ruff]
|
|
39
|
+
line-length = 88
|
|
40
|
+
target-version = "py310"
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
"""langchain-curt: LangChain chat loader for `.curt` conversation files."""
|
|
2
|
+
|
|
3
|
+
from langchain_curt.curt_chat import (
|
|
4
|
+
CurtChatLoader,
|
|
5
|
+
curt_ms_to_iso8601,
|
|
6
|
+
is_curt_conversation,
|
|
7
|
+
is_curt_message,
|
|
8
|
+
parse_curt,
|
|
9
|
+
)
|
|
10
|
+
|
|
11
|
+
__version__ = "0.1.0"
|
|
12
|
+
|
|
13
|
+
__all__ = [
|
|
14
|
+
"CurtChatLoader",
|
|
15
|
+
"curt_ms_to_iso8601",
|
|
16
|
+
"is_curt_conversation",
|
|
17
|
+
"is_curt_message",
|
|
18
|
+
"parse_curt",
|
|
19
|
+
]
|
|
@@ -0,0 +1,293 @@
|
|
|
1
|
+
"""LangChain chat loader for `.curt` v1 conversation files.
|
|
2
|
+
|
|
3
|
+
`.curt` is an open (MIT), single-file conversation format: one timestamped,
|
|
4
|
+
attributed conversation per file with unknown fields preserved on round-trip.
|
|
5
|
+
See https://github.com/JordanNewell/dot_curt (SPEC.md) — this module is a
|
|
6
|
+
reader, not a reimplementation; the reference implementation and SPEC there
|
|
7
|
+
are authoritative.
|
|
8
|
+
|
|
9
|
+
Dependencies: langchain-core only (BaseChatLoader / ChatSession / messages).
|
|
10
|
+
The parser below is stdlib-only and importable without langchain installed
|
|
11
|
+
(the langchain imports are guarded).
|
|
12
|
+
|
|
13
|
+
Spec: `.curt` v1 — https://github.com/JordanNewell/dot_curt (SPEC.md,
|
|
14
|
+
"Validation rules (version 1)"). Whole-file rejection on any validation
|
|
15
|
+
failure; unknown fields are data and must never cause a rejection.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import glob as _glob
|
|
21
|
+
import json
|
|
22
|
+
import math
|
|
23
|
+
import os
|
|
24
|
+
from collections.abc import Iterator
|
|
25
|
+
from datetime import datetime, timezone
|
|
26
|
+
from pathlib import Path
|
|
27
|
+
from typing import Any
|
|
28
|
+
|
|
29
|
+
try: # pragma: no cover - exercised only when langchain-core is installed
|
|
30
|
+
from langchain_core.chat_loaders import BaseChatLoader
|
|
31
|
+
from langchain_core.chat_sessions import ChatSession
|
|
32
|
+
from langchain_core.messages import (
|
|
33
|
+
AIMessage,
|
|
34
|
+
BaseMessage,
|
|
35
|
+
HumanMessage,
|
|
36
|
+
SystemMessage,
|
|
37
|
+
)
|
|
38
|
+
|
|
39
|
+
_LANGCHAIN_AVAILABLE = True
|
|
40
|
+
except ImportError: # pragma: no cover - parser-only standalone use
|
|
41
|
+
_LANGCHAIN_AVAILABLE = False
|
|
42
|
+
|
|
43
|
+
# ---------------------------------------------------------------------------
|
|
44
|
+
# .curt v1 parser — stdlib only, no langchain imports.
|
|
45
|
+
# Mirrors the reference implementation (dot_curt src/index.ts: parseCurt,
|
|
46
|
+
# isCurtConversation, isCurtMessage).
|
|
47
|
+
# ---------------------------------------------------------------------------
|
|
48
|
+
|
|
49
|
+
CURT_MAGIC = "curtis-conversation"
|
|
50
|
+
CURT_VERSION = 1
|
|
51
|
+
|
|
52
|
+
_CORE_MESSAGE_FIELDS = frozenset({"id", "role", "content", "timestamp", "model"})
|
|
53
|
+
_CORE_CONVERSATION_FIELDS = frozenset(
|
|
54
|
+
{"id", "title", "createdAt", "updatedAt", "messages"}
|
|
55
|
+
)
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _is_str(value: Any) -> bool:
|
|
59
|
+
return isinstance(value, str)
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _is_number(value: Any) -> bool:
|
|
63
|
+
# JSON numbers only: ``True`` is an ``int`` in Python but not a JSON number.
|
|
64
|
+
return isinstance(value, (int, float)) and not isinstance(value, bool)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def curt_ms_to_iso8601(ms: Any) -> str | None:
|
|
68
|
+
"""Convert epoch milliseconds to an ISO 8601 UTC string.
|
|
69
|
+
|
|
70
|
+
Returns None for non-finite values (the spec's bulk convention treats
|
|
71
|
+
those as "no meaningful timestamp"); the raw number is kept in
|
|
72
|
+
``additional_kwargs`` instead.
|
|
73
|
+
"""
|
|
74
|
+
if not _is_number(ms) or not math.isfinite(ms):
|
|
75
|
+
return None
|
|
76
|
+
return (
|
|
77
|
+
datetime.fromtimestamp(ms / 1000, tz=timezone.utc)
|
|
78
|
+
.isoformat()
|
|
79
|
+
.replace("+00:00", "Z")
|
|
80
|
+
)
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def is_curt_message(value: Any) -> bool:
|
|
84
|
+
"""Spec-valid message predicate (SPEC.md, "Each message in `messages`")."""
|
|
85
|
+
if not isinstance(value, dict):
|
|
86
|
+
return False
|
|
87
|
+
return (
|
|
88
|
+
_is_str(value.get("id"))
|
|
89
|
+
and value.get("role") in ("user", "assistant", "system", "tool")
|
|
90
|
+
and _is_str(value.get("content"))
|
|
91
|
+
and _is_number(value.get("timestamp"))
|
|
92
|
+
)
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def is_curt_conversation(value: Any) -> bool:
|
|
96
|
+
"""Spec-valid conversation predicate (SPEC.md, "Validation rules")."""
|
|
97
|
+
if not isinstance(value, dict):
|
|
98
|
+
return False
|
|
99
|
+
messages = value.get("messages")
|
|
100
|
+
return (
|
|
101
|
+
_is_str(value.get("id"))
|
|
102
|
+
and _is_str(value.get("title"))
|
|
103
|
+
and _is_number(value.get("createdAt"))
|
|
104
|
+
and _is_number(value.get("updatedAt"))
|
|
105
|
+
and isinstance(messages, list)
|
|
106
|
+
and all(is_curt_message(m) for m in messages)
|
|
107
|
+
)
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def parse_curt(raw: str) -> dict | None:
|
|
111
|
+
"""Parse a `.curt` file body. Returns the conversation dict or None.
|
|
112
|
+
|
|
113
|
+
Whole-file rejection: any validation failure returns None, never a
|
|
114
|
+
best-effort parse. Unknown fields on the top level, the conversation,
|
|
115
|
+
and messages are preserved verbatim (SPEC.md, "Unknown fields"). A
|
|
116
|
+
single leading UTF-8 BOM is ignored. ``version`` must be exactly the
|
|
117
|
+
number 1 — the string ``"1"`` and ``true`` are rejections.
|
|
118
|
+
"""
|
|
119
|
+
raw = raw.removeprefix("")
|
|
120
|
+
try:
|
|
121
|
+
data = json.loads(raw)
|
|
122
|
+
except (ValueError, TypeError):
|
|
123
|
+
return None
|
|
124
|
+
if not isinstance(data, dict):
|
|
125
|
+
return None
|
|
126
|
+
# ``type() is int`` so that ``True`` (== 1 in Python) is rejected.
|
|
127
|
+
if data.get("curt") != CURT_MAGIC or type(data.get("version")) is not int:
|
|
128
|
+
return None
|
|
129
|
+
if data["version"] != CURT_VERSION:
|
|
130
|
+
return None
|
|
131
|
+
conversation = data.get("conversation")
|
|
132
|
+
return conversation if is_curt_conversation(conversation) else None
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def _extra_fields(obj: dict, core: frozenset) -> dict:
|
|
136
|
+
"""Unknown fields are data: preserved verbatim, never interpreted."""
|
|
137
|
+
return {k: v for k, v in obj.items() if k not in core}
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
# ---------------------------------------------------------------------------
|
|
141
|
+
# Loader — requires langchain-core.
|
|
142
|
+
# ---------------------------------------------------------------------------
|
|
143
|
+
|
|
144
|
+
if _LANGCHAIN_AVAILABLE:
|
|
145
|
+
|
|
146
|
+
class CurtChatLoader(BaseChatLoader):
|
|
147
|
+
"""Load `.curt` v1 conversation files into LangChain chat sessions.
|
|
148
|
+
|
|
149
|
+
Spec: https://github.com/JordanNewell/dot_curt (SPEC.md, v1).
|
|
150
|
+
Each file yields one :class:`~langchain_core.chat_sessions.ChatSession`.
|
|
151
|
+
|
|
152
|
+
Args:
|
|
153
|
+
paths: A single path or glob pattern (``str`` or ``Path``), or a
|
|
154
|
+
list of them. Glob patterns are resolved and sorted for
|
|
155
|
+
deterministic ordering.
|
|
156
|
+
on_error: ``"raise"`` (default) rejects the whole batch with a
|
|
157
|
+
``ValueError`` on the first invalid file, matching the spec's
|
|
158
|
+
whole-file-rejection rule. ``"skip"`` matches the spec's bulk
|
|
159
|
+
convention instead: invalid files are reported (in
|
|
160
|
+
:attr:`errors`, never silently dropped) and valid ones load.
|
|
161
|
+
|
|
162
|
+
Role mapping: ``user`` -> HumanMessage, ``assistant`` -> AIMessage,
|
|
163
|
+
``system`` -> SystemMessage. ``tool`` messages are **skipped and
|
|
164
|
+
counted** (see :attr:`stats`): ToolMessage requires a
|
|
165
|
+
``tool_call_id``, which `.curt` v1 does not carry (message ``id`` is
|
|
166
|
+
a conversation-local identifier, not tool-call linkage), so a lossy
|
|
167
|
+
conversion would misrepresent agent traces.
|
|
168
|
+
|
|
169
|
+
Metadata: per-message timestamp (converted from epoch ms to ISO 8601
|
|
170
|
+
UTC), ``model``, and any unknown/extension fields are preserved under
|
|
171
|
+
``additional_kwargs["curt"]``. Conversation-level metadata (``id``,
|
|
172
|
+
``title``, ``createdAt``/``updatedAt`` as ISO 8601, ``generator``,
|
|
173
|
+
unknown fields, per-file counts) is collected in
|
|
174
|
+
:attr:`conversation_metadata` because ChatSession carries no
|
|
175
|
+
metadata slot of its own.
|
|
176
|
+
|
|
177
|
+
Attributes:
|
|
178
|
+
stats: Running counters (``files_loaded``, ``files_failed``,
|
|
179
|
+
``messages_loaded``, ``tool_messages_skipped``).
|
|
180
|
+
errors: ``(path, reason)`` tuples for files rejected under
|
|
181
|
+
``on_error="skip"``.
|
|
182
|
+
conversation_metadata: One metadata dict per loaded file, in
|
|
183
|
+
load order.
|
|
184
|
+
"""
|
|
185
|
+
|
|
186
|
+
def __init__(
|
|
187
|
+
self,
|
|
188
|
+
paths: str | Path | list[str | Path],
|
|
189
|
+
*,
|
|
190
|
+
on_error: str = "raise",
|
|
191
|
+
) -> None:
|
|
192
|
+
if on_error not in ("raise", "skip"):
|
|
193
|
+
msg = f"on_error must be 'raise' or 'skip', got {on_error!r}"
|
|
194
|
+
raise ValueError(msg)
|
|
195
|
+
self._patterns: list[str | Path] = (
|
|
196
|
+
list(paths) if isinstance(paths, (list, tuple)) else [paths]
|
|
197
|
+
)
|
|
198
|
+
self.on_error = on_error
|
|
199
|
+
self.stats = {
|
|
200
|
+
"files_loaded": 0,
|
|
201
|
+
"files_failed": 0,
|
|
202
|
+
"messages_loaded": 0,
|
|
203
|
+
"tool_messages_skipped": 0,
|
|
204
|
+
}
|
|
205
|
+
self.errors: list[Any] = []
|
|
206
|
+
self.conversation_metadata: list[dict] = []
|
|
207
|
+
|
|
208
|
+
def _resolve_files(self) -> list[Path]:
|
|
209
|
+
files: list[Path] = []
|
|
210
|
+
for pattern in self._patterns:
|
|
211
|
+
matches = sorted(_glob.glob(os.fspath(pattern)))
|
|
212
|
+
if not matches and not any(c in os.fspath(pattern) for c in "*?["):
|
|
213
|
+
# A literal path that does not exist is an error, even
|
|
214
|
+
# under on_error="skip": unlike a corrupt file, the user
|
|
215
|
+
# almost certainly mistyped it.
|
|
216
|
+
raise FileNotFoundError(os.fspath(pattern))
|
|
217
|
+
files.extend(Path(m) for m in matches)
|
|
218
|
+
if not files:
|
|
219
|
+
msg = f"No .curt files matched: {self._patterns!r}"
|
|
220
|
+
raise ValueError(msg)
|
|
221
|
+
return files
|
|
222
|
+
|
|
223
|
+
def _load_file(self, path: Path) -> ChatSession:
|
|
224
|
+
conversation = parse_curt(path.read_text(encoding="utf-8"))
|
|
225
|
+
if conversation is None:
|
|
226
|
+
msg = (
|
|
227
|
+
f"{path}: not a valid .curt v1 file "
|
|
228
|
+
"(bad JSON, wrong magic/version, or failed validation; "
|
|
229
|
+
"see https://github.com/JordanNewell/dot_curt)"
|
|
230
|
+
)
|
|
231
|
+
raise ValueError(msg)
|
|
232
|
+
|
|
233
|
+
messages: list[BaseMessage] = []
|
|
234
|
+
tool_skipped = 0
|
|
235
|
+
for m in conversation["messages"]:
|
|
236
|
+
role = m["role"]
|
|
237
|
+
if role == "tool":
|
|
238
|
+
tool_skipped += 1
|
|
239
|
+
continue
|
|
240
|
+
curt_kwargs: dict = {
|
|
241
|
+
"message_id": m["id"],
|
|
242
|
+
"conversation_id": conversation["id"],
|
|
243
|
+
"conversation_title": conversation["title"],
|
|
244
|
+
"timestamp": curt_ms_to_iso8601(m["timestamp"]),
|
|
245
|
+
"unknown_fields": _extra_fields(m, _CORE_MESSAGE_FIELDS),
|
|
246
|
+
}
|
|
247
|
+
if curt_kwargs["timestamp"] is None:
|
|
248
|
+
curt_kwargs["timestamp_raw_ms"] = m["timestamp"]
|
|
249
|
+
if "model" in m:
|
|
250
|
+
curt_kwargs["model"] = m["model"]
|
|
251
|
+
cls = {
|
|
252
|
+
"user": HumanMessage,
|
|
253
|
+
"assistant": AIMessage,
|
|
254
|
+
"system": SystemMessage,
|
|
255
|
+
}[role]
|
|
256
|
+
messages.append(
|
|
257
|
+
cls(content=m["content"], additional_kwargs={"curt": curt_kwargs})
|
|
258
|
+
)
|
|
259
|
+
|
|
260
|
+
created = curt_ms_to_iso8601(conversation["createdAt"])
|
|
261
|
+
updated = curt_ms_to_iso8601(conversation["updatedAt"])
|
|
262
|
+
meta = {
|
|
263
|
+
"source": str(path),
|
|
264
|
+
"id": conversation["id"],
|
|
265
|
+
"title": conversation["title"],
|
|
266
|
+
"created_at": created,
|
|
267
|
+
"updated_at": updated,
|
|
268
|
+
"generator": None, # readers must ignore `generator` (spec)
|
|
269
|
+
"unknown_fields": _extra_fields(
|
|
270
|
+
conversation, _CORE_CONVERSATION_FIELDS
|
|
271
|
+
),
|
|
272
|
+
"tool_messages_skipped": tool_skipped,
|
|
273
|
+
}
|
|
274
|
+
for key in ("created_at", "updated_at"):
|
|
275
|
+
if meta[key] is None:
|
|
276
|
+
meta[key + "_raw_ms"] = conversation[
|
|
277
|
+
"createdAt" if key == "created_at" else "updatedAt"
|
|
278
|
+
]
|
|
279
|
+
self.conversation_metadata.append(meta)
|
|
280
|
+
self.stats["files_loaded"] += 1
|
|
281
|
+
self.stats["messages_loaded"] += len(messages)
|
|
282
|
+
self.stats["tool_messages_skipped"] += tool_skipped
|
|
283
|
+
return ChatSession(messages=messages)
|
|
284
|
+
|
|
285
|
+
def lazy_load(self) -> Iterator[ChatSession]:
|
|
286
|
+
for path in self._resolve_files():
|
|
287
|
+
try:
|
|
288
|
+
yield self._load_file(path)
|
|
289
|
+
except (ValueError, OSError) as e:
|
|
290
|
+
if self.on_error == "raise":
|
|
291
|
+
raise
|
|
292
|
+
self.stats["files_failed"] += 1
|
|
293
|
+
self.errors.append((str(path), str(e)))
|
|
@@ -0,0 +1,288 @@
|
|
|
1
|
+
"""Tests for CurtChatLoader.
|
|
2
|
+
|
|
3
|
+
All fixtures are synthetic. Never load real conversation content in tests.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import json
|
|
9
|
+
|
|
10
|
+
import pytest
|
|
11
|
+
|
|
12
|
+
from langchain_curt.curt_chat import (
|
|
13
|
+
CurtChatLoader,
|
|
14
|
+
curt_ms_to_iso8601,
|
|
15
|
+
is_curt_conversation,
|
|
16
|
+
is_curt_message,
|
|
17
|
+
parse_curt,
|
|
18
|
+
)
|
|
19
|
+
|
|
20
|
+
MAGIC = "curtis-conversation"
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def curt_body(
|
|
24
|
+
conv_id="conv_001",
|
|
25
|
+
title="Test Conversation",
|
|
26
|
+
messages=None,
|
|
27
|
+
**top_extra,
|
|
28
|
+
):
|
|
29
|
+
"""Build a minimal valid .curt v1 body. All content is synthetic."""
|
|
30
|
+
if messages is None:
|
|
31
|
+
messages = [
|
|
32
|
+
{
|
|
33
|
+
"id": "m1",
|
|
34
|
+
"role": "user",
|
|
35
|
+
"content": "Hello",
|
|
36
|
+
"timestamp": 1700000000000,
|
|
37
|
+
},
|
|
38
|
+
{
|
|
39
|
+
"id": "m2",
|
|
40
|
+
"role": "assistant",
|
|
41
|
+
"content": "Hi there",
|
|
42
|
+
"timestamp": 1700000001000,
|
|
43
|
+
"model": "test-model",
|
|
44
|
+
},
|
|
45
|
+
]
|
|
46
|
+
body = {
|
|
47
|
+
"curt": MAGIC,
|
|
48
|
+
"version": 1,
|
|
49
|
+
"generator": "synthetic-fixture@0.0.1",
|
|
50
|
+
"conversation": {
|
|
51
|
+
"id": conv_id,
|
|
52
|
+
"title": title,
|
|
53
|
+
"createdAt": 1700000000000,
|
|
54
|
+
"updatedAt": 1700000001000,
|
|
55
|
+
"messages": messages,
|
|
56
|
+
},
|
|
57
|
+
}
|
|
58
|
+
body.update(top_extra)
|
|
59
|
+
return body
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def write_curt(path, body):
|
|
63
|
+
path.write_text(json.dumps(body, indent="\t"), encoding="utf-8")
|
|
64
|
+
return path
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def load_one(loader):
|
|
68
|
+
sessions = list(loader.lazy_load())
|
|
69
|
+
return sessions
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
class TestHappyPath:
|
|
73
|
+
def test_single_file(self, tmp_path):
|
|
74
|
+
f = write_curt(tmp_path / "a.curt", curt_body())
|
|
75
|
+
loader = CurtChatLoader(f)
|
|
76
|
+
sessions = load_one(loader)
|
|
77
|
+
assert len(sessions) == 1
|
|
78
|
+
msgs = sessions[0]["messages"]
|
|
79
|
+
assert [type(m).__name__ for m in msgs] == ["HumanMessage", "AIMessage"]
|
|
80
|
+
assert msgs[0].content == "Hello"
|
|
81
|
+
assert msgs[1].content == "Hi there"
|
|
82
|
+
assert msgs[1].additional_kwargs["curt"]["model"] == "test-model"
|
|
83
|
+
# Timestamps: epoch ms -> ISO 8601 UTC.
|
|
84
|
+
assert msgs[0].additional_kwargs["curt"]["timestamp"] == (
|
|
85
|
+
"2023-11-14T22:13:20Z"
|
|
86
|
+
)
|
|
87
|
+
assert loader.stats["files_loaded"] == 1
|
|
88
|
+
assert loader.stats["messages_loaded"] == 2
|
|
89
|
+
assert loader.errors == []
|
|
90
|
+
|
|
91
|
+
def test_conversation_metadata_collected(self, tmp_path):
|
|
92
|
+
f = write_curt(tmp_path / "a.curt", curt_body())
|
|
93
|
+
loader = CurtChatLoader(f)
|
|
94
|
+
load_one(loader)
|
|
95
|
+
(meta,) = loader.conversation_metadata
|
|
96
|
+
assert meta["id"] == "conv_001"
|
|
97
|
+
assert meta["title"] == "Test Conversation"
|
|
98
|
+
assert meta["created_at"] == "2023-11-14T22:13:20Z"
|
|
99
|
+
assert meta["source"].endswith("a.curt")
|
|
100
|
+
|
|
101
|
+
def test_system_role(self, tmp_path):
|
|
102
|
+
msgs = [
|
|
103
|
+
{"id": "m0", "role": "system", "content": "You are terse.", "timestamp": 0},
|
|
104
|
+
{"id": "m1", "role": "user", "content": "Hi", "timestamp": 1},
|
|
105
|
+
]
|
|
106
|
+
f = write_curt(tmp_path / "a.curt", curt_body(messages=msgs))
|
|
107
|
+
(session,) = load_one(CurtChatLoader(f))
|
|
108
|
+
assert type(session["messages"][0]).__name__ == "SystemMessage"
|
|
109
|
+
|
|
110
|
+
def test_timestamp_zero_and_fractional(self):
|
|
111
|
+
assert curt_ms_to_iso8601(0) == "1970-01-01T00:00:00Z"
|
|
112
|
+
# Spec: validators accept any JSON number; never reject fractional.
|
|
113
|
+
assert curt_ms_to_iso8601(1700000000123.5) == "2023-11-14T22:13:20.123500Z"
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
class TestMultiFile:
|
|
117
|
+
def test_glob(self, tmp_path):
|
|
118
|
+
for i in (2, 1):
|
|
119
|
+
write_curt(tmp_path / f"chat{i}.curt", curt_body(conv_id=f"conv_{i:03d}"))
|
|
120
|
+
loader = CurtChatLoader(str(tmp_path / "*.curt"))
|
|
121
|
+
sessions = load_one(loader)
|
|
122
|
+
assert len(sessions) == 2
|
|
123
|
+
# Sorted for deterministic ordering.
|
|
124
|
+
assert loader.conversation_metadata[0]["id"] == "conv_001"
|
|
125
|
+
|
|
126
|
+
def test_path_list(self, tmp_path):
|
|
127
|
+
fa = write_curt(tmp_path / "a.curt", curt_body(conv_id="a"))
|
|
128
|
+
fb = write_curt(tmp_path / "b.curt", curt_body(conv_id="b"))
|
|
129
|
+
sessions = load_one(CurtChatLoader([fa, fb]))
|
|
130
|
+
assert [
|
|
131
|
+
m["messages"][0].additional_kwargs["curt"]["conversation_id"]
|
|
132
|
+
for m in sessions
|
|
133
|
+
] == ["a", "b"]
|
|
134
|
+
|
|
135
|
+
def test_no_matches_raises(self, tmp_path):
|
|
136
|
+
with pytest.raises(ValueError, match="No .curt files matched"):
|
|
137
|
+
load_one(CurtChatLoader(str(tmp_path / "*.curt")))
|
|
138
|
+
|
|
139
|
+
def test_missing_literal_path_raises(self, tmp_path):
|
|
140
|
+
with pytest.raises(FileNotFoundError):
|
|
141
|
+
load_one(CurtChatLoader(tmp_path / "nope.curt"))
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
class TestRejections:
|
|
145
|
+
"""Whole-file rejection per spec — a corrupted file is never half-loaded."""
|
|
146
|
+
|
|
147
|
+
def test_bad_magic_string(self, tmp_path):
|
|
148
|
+
write_curt(tmp_path / "a.curt", curt_body(curt="wrong-magic"))
|
|
149
|
+
with pytest.raises(ValueError, match="not a valid .curt v1 file"):
|
|
150
|
+
load_one(CurtChatLoader(tmp_path / "a.curt"))
|
|
151
|
+
|
|
152
|
+
def test_version_as_string(self, tmp_path):
|
|
153
|
+
write_curt(tmp_path / "a.curt", curt_body(version="1"))
|
|
154
|
+
with pytest.raises(ValueError, match="not a valid .curt v1 file"):
|
|
155
|
+
load_one(CurtChatLoader(tmp_path / "a.curt"))
|
|
156
|
+
|
|
157
|
+
def test_version_true_rejected(self, tmp_path):
|
|
158
|
+
# Python gotcha the parser must guard: True == 1.
|
|
159
|
+
write_curt(tmp_path / "a.curt", curt_body(version=True))
|
|
160
|
+
assert parse_curt((tmp_path / "a.curt").read_text(encoding="utf-8")) is None
|
|
161
|
+
|
|
162
|
+
def test_unknown_role_rejects_whole_file(self, tmp_path):
|
|
163
|
+
# Spec: role must be user/assistant/system/tool; anything else fails
|
|
164
|
+
# message validation, and failure is whole-file — the valid messages
|
|
165
|
+
# in the same file must NOT load.
|
|
166
|
+
msgs = [
|
|
167
|
+
{"id": "m1", "role": "user", "content": "ok", "timestamp": 1},
|
|
168
|
+
{"id": "m2", "role": "moderator", "content": "nope", "timestamp": 2},
|
|
169
|
+
]
|
|
170
|
+
write_curt(tmp_path / "a.curt", curt_body(messages=msgs))
|
|
171
|
+
with pytest.raises(ValueError, match="not a valid .curt v1 file"):
|
|
172
|
+
load_one(CurtChatLoader(tmp_path / "a.curt"))
|
|
173
|
+
|
|
174
|
+
def test_missing_required_message_field(self, tmp_path):
|
|
175
|
+
msgs = [{"id": "m1", "role": "user", "content": "no timestamp"}]
|
|
176
|
+
write_curt(tmp_path / "a.curt", curt_body(messages=msgs))
|
|
177
|
+
with pytest.raises(ValueError):
|
|
178
|
+
load_one(CurtChatLoader(tmp_path / "a.curt"))
|
|
179
|
+
|
|
180
|
+
def test_malformed_json(self, tmp_path):
|
|
181
|
+
(tmp_path / "a.curt").write_text("{not json", encoding="utf-8")
|
|
182
|
+
with pytest.raises(ValueError, match="not a valid .curt v1 file"):
|
|
183
|
+
load_one(CurtChatLoader(tmp_path / "a.curt"))
|
|
184
|
+
|
|
185
|
+
def test_non_object_body(self, tmp_path):
|
|
186
|
+
(tmp_path / "a.curt").write_text('["a"]', encoding="utf-8")
|
|
187
|
+
assert parse_curt('["a"]') is None
|
|
188
|
+
|
|
189
|
+
def test_skip_mode_reports(self, tmp_path):
|
|
190
|
+
write_curt(tmp_path / "good.curt", curt_body())
|
|
191
|
+
write_curt(tmp_path / "bad.curt", curt_body(curt="nope"))
|
|
192
|
+
loader = CurtChatLoader(str(tmp_path / "*.curt"), on_error="skip")
|
|
193
|
+
sessions = load_one(loader)
|
|
194
|
+
assert len(sessions) == 1
|
|
195
|
+
assert loader.stats["files_loaded"] == 1
|
|
196
|
+
assert loader.stats["files_failed"] == 1
|
|
197
|
+
assert len(loader.errors) == 1
|
|
198
|
+
assert "bad.curt" in loader.errors[0][0]
|
|
199
|
+
|
|
200
|
+
def test_bad_on_error_arg(self):
|
|
201
|
+
with pytest.raises(ValueError, match="on_error"):
|
|
202
|
+
CurtChatLoader("x", on_error="explode")
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
class TestToolRole:
|
|
206
|
+
def test_tool_messages_skipped_and_counted(self, tmp_path):
|
|
207
|
+
msgs = [
|
|
208
|
+
{"id": "m1", "role": "user", "content": "q", "timestamp": 1},
|
|
209
|
+
{
|
|
210
|
+
"id": "m2",
|
|
211
|
+
"role": "tool",
|
|
212
|
+
"content": '{"result": 42}',
|
|
213
|
+
"timestamp": 2,
|
|
214
|
+
},
|
|
215
|
+
{"id": "m3", "role": "assistant", "content": "a", "timestamp": 3},
|
|
216
|
+
]
|
|
217
|
+
write_curt(tmp_path / "a.curt", curt_body(messages=msgs))
|
|
218
|
+
loader = CurtChatLoader(tmp_path / "a.curt")
|
|
219
|
+
(session,) = load_one(loader)
|
|
220
|
+
# Documented loss: tool -> skipped (ToolMessage needs tool_call_id).
|
|
221
|
+
assert [type(m).__name__ for m in session["messages"]] == [
|
|
222
|
+
"HumanMessage",
|
|
223
|
+
"AIMessage",
|
|
224
|
+
]
|
|
225
|
+
assert loader.stats["tool_messages_skipped"] == 1
|
|
226
|
+
assert loader.conversation_metadata[0]["tool_messages_skipped"] == 1
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
class TestSpecCompliance:
|
|
230
|
+
def test_bom_tolerated(self, tmp_path):
|
|
231
|
+
f = tmp_path / "a.curt"
|
|
232
|
+
f.write_text("" + json.dumps(curt_body()), encoding="utf-8")
|
|
233
|
+
sessions = load_one(CurtChatLoader(f))
|
|
234
|
+
assert len(sessions) == 1
|
|
235
|
+
|
|
236
|
+
def test_unknown_fields_preserved(self, tmp_path):
|
|
237
|
+
msgs = [
|
|
238
|
+
{
|
|
239
|
+
"id": "m1",
|
|
240
|
+
"role": "user",
|
|
241
|
+
"content": "hi",
|
|
242
|
+
"timestamp": 1,
|
|
243
|
+
"tokens": 5,
|
|
244
|
+
"images-inline": {"v": 1},
|
|
245
|
+
}
|
|
246
|
+
]
|
|
247
|
+
body = curt_body(messages=msgs, extensions=["images-inline"])
|
|
248
|
+
body["conversation"]["folder_hint"] = "keep-me"
|
|
249
|
+
f = write_curt(tmp_path / "a.curt", body)
|
|
250
|
+
loader = CurtChatLoader(f)
|
|
251
|
+
(session,) = load_one(loader)
|
|
252
|
+
kwargs = session["messages"][0].additional_kwargs["curt"]
|
|
253
|
+
# Unknown fields are data: preserved verbatim, never a rejection.
|
|
254
|
+
assert kwargs["unknown_fields"] == {
|
|
255
|
+
"tokens": 5,
|
|
256
|
+
"images-inline": {"v": 1},
|
|
257
|
+
}
|
|
258
|
+
meta = loader.conversation_metadata[0]
|
|
259
|
+
assert meta["unknown_fields"] == {"folder_hint": "keep-me"}
|
|
260
|
+
# `generator` must be ignored by readers (spec), never validated on.
|
|
261
|
+
assert meta["generator"] is None
|
|
262
|
+
|
|
263
|
+
def test_empty_title_allowed(self, tmp_path):
|
|
264
|
+
f = write_curt(tmp_path / "a.curt", curt_body(title=""))
|
|
265
|
+
(session,) = load_one(CurtChatLoader(f))
|
|
266
|
+
assert (
|
|
267
|
+
session["messages"][0].additional_kwargs["curt"]["conversation_title"] == ""
|
|
268
|
+
)
|
|
269
|
+
|
|
270
|
+
def test_empty_messages_accepted(self, tmp_path):
|
|
271
|
+
# Spec: readers may accept empty message arrays.
|
|
272
|
+
f = write_curt(tmp_path / "a.curt", curt_body(messages=[]))
|
|
273
|
+
(session,) = load_one(CurtChatLoader(f))
|
|
274
|
+
assert session["messages"] == []
|
|
275
|
+
|
|
276
|
+
def test_predicates(self):
|
|
277
|
+
assert is_curt_message(
|
|
278
|
+
{"id": "m", "role": "user", "content": "c", "timestamp": 0}
|
|
279
|
+
)
|
|
280
|
+
assert not is_curt_message(
|
|
281
|
+
{"id": "m", "role": "user", "content": "c", "timestamp": True}
|
|
282
|
+
)
|
|
283
|
+
assert not is_curt_conversation(
|
|
284
|
+
{"id": "c", "title": "t", "createdAt": 0, "updatedAt": 0}
|
|
285
|
+
)
|
|
286
|
+
assert is_curt_conversation(
|
|
287
|
+
{"id": "c", "title": "t", "createdAt": 0, "updatedAt": 0, "messages": []}
|
|
288
|
+
)
|