llmalchemy 1.4.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- llmalchemy/__init__.py +6 -0
- llmalchemy/agent.py +249 -0
- llmalchemy/code.py +149 -0
- llmalchemy/context.py +90 -0
- llmalchemy/database.py +62 -0
- llmalchemy/prompt.jinja +22 -0
- llmalchemy/tools.py +78 -0
- llmalchemy-1.4.1.dist-info/METADATA +308 -0
- llmalchemy-1.4.1.dist-info/RECORD +10 -0
- llmalchemy-1.4.1.dist-info/WHEEL +4 -0
llmalchemy/__init__.py
ADDED
llmalchemy/agent.py
ADDED
|
@@ -0,0 +1,249 @@
|
|
|
1
|
+
"""Contains the agentic loop and related utils."""
|
|
2
|
+
|
|
3
|
+
import weakref
|
|
4
|
+
from collections.abc import Generator, Iterator
|
|
5
|
+
from dataclasses import dataclass, field
|
|
6
|
+
from enum import StrEnum
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from typing import Any, cast
|
|
9
|
+
|
|
10
|
+
from lmdk import Message, UserMessage, complete
|
|
11
|
+
from pydantic import BaseModel, Field, create_model
|
|
12
|
+
from sqlalchemy import create_engine
|
|
13
|
+
from sqlalchemy.orm import DeclarativeBase, Session
|
|
14
|
+
|
|
15
|
+
from .code import execute, validate
|
|
16
|
+
from .context import render
|
|
17
|
+
from .tools import Tool, make_disclose_fn
|
|
18
|
+
|
|
19
|
+
MAX_LOOPS = 20
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
@dataclass
|
|
23
|
+
class State:
|
|
24
|
+
"""Contains the different objects whose state is modified through the agentic loop.
|
|
25
|
+
|
|
26
|
+
Attributes:
|
|
27
|
+
session: the sqlalchemy database connection
|
|
28
|
+
messages: the conversation history
|
|
29
|
+
namespace: symbols of the code environment that the agent uses
|
|
30
|
+
"""
|
|
31
|
+
|
|
32
|
+
session: Session | None = None
|
|
33
|
+
messages: list[Message] = field(default_factory=list)
|
|
34
|
+
namespace: dict = field(default_factory=dict)
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class Output(BaseModel):
|
|
38
|
+
"""Pydantic model to force the LM structured output."""
|
|
39
|
+
|
|
40
|
+
message: str = Field(description="The response shown to the user.")
|
|
41
|
+
code: str = Field(default="", description="The optional code to run.")
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
class Signal(StrEnum):
|
|
45
|
+
"""Signals emitted by the agentic loop to indicate current stage."""
|
|
46
|
+
|
|
47
|
+
COMPLETION = "COMPLETION"
|
|
48
|
+
VALIDATION = "VALIDATION"
|
|
49
|
+
EXECUTION = "EXECUTION"
|
|
50
|
+
EXCEEDED = "EXCEEDED"
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
@dataclass(frozen=True)
|
|
54
|
+
class Event:
|
|
55
|
+
"""Base class for all events yielded by the agentic loop."""
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
@dataclass(frozen=True)
|
|
59
|
+
class SignalEvent(Event):
|
|
60
|
+
"""A control-flow signal indicating the current stage."""
|
|
61
|
+
|
|
62
|
+
signal: Signal
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
@dataclass(frozen=True)
|
|
66
|
+
class MessageEvent(Event):
|
|
67
|
+
"""A message appended to the conversation history."""
|
|
68
|
+
|
|
69
|
+
message: Message
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
@dataclass(frozen=True)
|
|
73
|
+
class SystemInstructionEvent(Event):
|
|
74
|
+
"""The system instruction sent to the model."""
|
|
75
|
+
|
|
76
|
+
content: str
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _complete(
|
|
80
|
+
state: State,
|
|
81
|
+
model: str,
|
|
82
|
+
system_instruction: str,
|
|
83
|
+
output_schema: type[Output],
|
|
84
|
+
) -> Generator[Event, None, Output]:
|
|
85
|
+
"""Single LM call: append the response, yield signals and the message, return parsed output."""
|
|
86
|
+
yield SignalEvent(Signal.COMPLETION)
|
|
87
|
+
response = complete(
|
|
88
|
+
model=model,
|
|
89
|
+
prompt=state.messages,
|
|
90
|
+
system_instruction=system_instruction,
|
|
91
|
+
output_schema=output_schema,
|
|
92
|
+
)
|
|
93
|
+
state.messages.append(response.message)
|
|
94
|
+
yield MessageEvent(response.message)
|
|
95
|
+
assert isinstance(response.output, Output)
|
|
96
|
+
return response.output
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def _build_output_schema(output_extensions: type[BaseModel] | None) -> type[Output]:
|
|
100
|
+
"""Build the structured-output schema used by the agent loop.
|
|
101
|
+
|
|
102
|
+
When ``output_extensions`` is ``None``, the plain :class:`Output` model is
|
|
103
|
+
returned. Otherwise, a dynamic subclass is created whose fields are the
|
|
104
|
+
extension fields followed by ``message`` and ``code`` (in that order).
|
|
105
|
+
|
|
106
|
+
Ordering matters: fields emitted earlier in the structured output act as a
|
|
107
|
+
scratchpad for later fields (this is how chain-of-thought-in-schema works).
|
|
108
|
+
Typical uses are reasoning slots (e.g. ``thoughts: str`` or ARQ-style
|
|
109
|
+
``user_intent`` / ``info_needed`` / ``info_missing``) and product fields
|
|
110
|
+
(``confidence``, ``citations``, ``suggested_followups``, …). The agent loop
|
|
111
|
+
only reads ``.message`` and ``.code``; extra fields ride along on the
|
|
112
|
+
yielded message object for the caller to consume.
|
|
113
|
+
"""
|
|
114
|
+
if output_extensions is None:
|
|
115
|
+
return Output
|
|
116
|
+
|
|
117
|
+
extension_fields = {
|
|
118
|
+
name: (f.annotation, f) for name, f in output_extensions.model_fields.items()
|
|
119
|
+
}
|
|
120
|
+
base_fields = {name: (f.annotation, f) for name, f in Output.model_fields.items()}
|
|
121
|
+
# ``create_model``'s overloads don't accept ``**kwargs`` unpacking, so we
|
|
122
|
+
# cast to ``Any`` to silence the type checker without a per-call pragma.
|
|
123
|
+
return cast(Any, create_model)("Output", **extension_fields, **base_fields)
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def _init_session(state: State, base: type[DeclarativeBase]) -> None:
|
|
127
|
+
"""Initialize the SQLAlchemy session when missing (first call)."""
|
|
128
|
+
if state.session is None:
|
|
129
|
+
engine = create_engine("sqlite://")
|
|
130
|
+
base.metadata.create_all(engine)
|
|
131
|
+
state.session = Session(engine)
|
|
132
|
+
# Dispose the engine (and close its pooled sqlite3.Connection) when
|
|
133
|
+
# the session is garbage-collected, to avoid ResourceWarnings.
|
|
134
|
+
weakref.finalize(state.session, engine.dispose)
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def _init_namespace(
|
|
138
|
+
state: State,
|
|
139
|
+
base: type[DeclarativeBase],
|
|
140
|
+
tools: list[Tool],
|
|
141
|
+
) -> dict[str, str]:
|
|
142
|
+
"""Populate or refresh the agent code execution namespace.
|
|
143
|
+
|
|
144
|
+
On the first call the namespace is empty, so all symbols are injected:
|
|
145
|
+
``session``, ORM model classes, tool functions, and ``disclose``.
|
|
146
|
+
On follow-up calls only ``session`` is refreshed because the database
|
|
147
|
+
may have changed between calls (user side).
|
|
148
|
+
|
|
149
|
+
Returns:
|
|
150
|
+
A ``{name: description}`` dict of every injected **infrastructure**
|
|
151
|
+
symbol. Tool symbols are excluded — their source of truth is the
|
|
152
|
+
``Tool`` object itself, rendered separately by ``_render_tools_summary``.
|
|
153
|
+
"""
|
|
154
|
+
descriptions: dict[str, str] = {}
|
|
155
|
+
|
|
156
|
+
state.namespace["session"] = state.session
|
|
157
|
+
descriptions["session"] = "a `sqlalchemy.orm.Session` connected to the database."
|
|
158
|
+
|
|
159
|
+
orm_classes = base.__subclasses__()
|
|
160
|
+
for cls in orm_classes:
|
|
161
|
+
state.namespace[cls.__name__] = cls
|
|
162
|
+
if orm_classes:
|
|
163
|
+
names = ", ".join(cls.__name__ for cls in orm_classes)
|
|
164
|
+
descriptions[names] = "ORM model classes (see schema above)."
|
|
165
|
+
|
|
166
|
+
for t in tools:
|
|
167
|
+
state.namespace[t.name] = t.fn
|
|
168
|
+
|
|
169
|
+
if tools:
|
|
170
|
+
state.namespace["disclose"] = make_disclose_fn(tools)
|
|
171
|
+
descriptions["disclose"] = (
|
|
172
|
+
"`disclose(name: str) -> str` — prints the full signature"
|
|
173
|
+
" and docstring of a tool. Call it before using a tool you haven't seen yet."
|
|
174
|
+
)
|
|
175
|
+
|
|
176
|
+
return descriptions
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def run(
|
|
180
|
+
state: State,
|
|
181
|
+
base: type[DeclarativeBase],
|
|
182
|
+
model: str,
|
|
183
|
+
tools: list[Tool] | None = None,
|
|
184
|
+
allowed_imports: list[str] | None = None,
|
|
185
|
+
prompt_template: str | Path | None = None,
|
|
186
|
+
output_extensions: type[BaseModel] | None = None,
|
|
187
|
+
thinking: bool = False,
|
|
188
|
+
) -> Iterator[Event]:
|
|
189
|
+
"""Execute the agentic loop.
|
|
190
|
+
|
|
191
|
+
An intermediate turn is one where the assistant message requests to run ``.code``.
|
|
192
|
+
The loop ends as soon as the assistant responds without code. Turn is returned to user.
|
|
193
|
+
|
|
194
|
+
Args:
|
|
195
|
+
state: Conversation, database state and python namespace (mutated in place).
|
|
196
|
+
base: SQLAlchemy declarative base that defines the db schema.
|
|
197
|
+
model: Model identifier forwarded to ``complete()``.
|
|
198
|
+
tools: User-provided tools the agent can call in generated code.
|
|
199
|
+
allowed_imports: Any vanilla module or third-party package that the agent can use.
|
|
200
|
+
output_extensions: Optional Pydantic model to force in the LM structured output.
|
|
201
|
+
thinking: Level of thinking for provider-native reasoning tokens. Not implemented yet.
|
|
202
|
+
prompt_template: Custom jinja system prompt. Should contain placeholders for:
|
|
203
|
+
- ``SCHEMA``: used to show agent the source code of ORM classes
|
|
204
|
+
- ``SYMBOLS``: used to show ageent all pre-loaded namespace symbols.
|
|
205
|
+
- ``TOOLS``: usedf to show the agent tool names + short descriptions.
|
|
206
|
+
|
|
207
|
+
Yields:
|
|
208
|
+
``Event``: system instruction, loop signals, and conversation messages.
|
|
209
|
+
"""
|
|
210
|
+
if thinking:
|
|
211
|
+
raise NotImplementedError("Native provider thinking is not yet wired through lmdk.")
|
|
212
|
+
|
|
213
|
+
# Initialize everything
|
|
214
|
+
tools = tools or []
|
|
215
|
+
allowed_imports = allowed_imports or []
|
|
216
|
+
output_schema = _build_output_schema(output_extensions)
|
|
217
|
+
_init_session(state, base)
|
|
218
|
+
descriptions = _init_namespace(state, base, tools)
|
|
219
|
+
system_instruction = render(base, tools, descriptions, prompt_template)
|
|
220
|
+
yield SystemInstructionEvent(system_instruction)
|
|
221
|
+
|
|
222
|
+
# First call to the model
|
|
223
|
+
output = yield from _complete(state, model, system_instruction, output_schema)
|
|
224
|
+
code = output.code
|
|
225
|
+
|
|
226
|
+
# Loop until model is over with the task
|
|
227
|
+
loops = 0
|
|
228
|
+
while code:
|
|
229
|
+
if loops >= MAX_LOOPS:
|
|
230
|
+
yield SignalEvent(Signal.EXCEEDED)
|
|
231
|
+
break
|
|
232
|
+
loops += 1
|
|
233
|
+
|
|
234
|
+
yield SignalEvent(Signal.VALIDATION)
|
|
235
|
+
if reason := validate(source=code, allowed_imports=allowed_imports):
|
|
236
|
+
message = UserMessage(f"Code rejected: {reason}")
|
|
237
|
+
state.messages.append(message)
|
|
238
|
+
yield MessageEvent(message)
|
|
239
|
+
output = yield from _complete(state, model, system_instruction, output_schema)
|
|
240
|
+
code = output.code
|
|
241
|
+
continue
|
|
242
|
+
|
|
243
|
+
yield SignalEvent(Signal.EXECUTION)
|
|
244
|
+
result = execute(source=code, namespace=state.namespace)
|
|
245
|
+
message = UserMessage(f"Execution result:\n{result}")
|
|
246
|
+
state.messages.append(message)
|
|
247
|
+
yield MessageEvent(message)
|
|
248
|
+
output = yield from _complete(state, model, system_instruction, output_schema)
|
|
249
|
+
code = output.code
|
llmalchemy/code.py
ADDED
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
"""Contains the logic for the sandboxed python env on which to run the agent requested code.
|
|
2
|
+
|
|
3
|
+
HuggingFace has a very nice reference for further ideas:
|
|
4
|
+
https://github.com/huggingface/smolagents/blob/main/src/smolagents/local_python_executor.py
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
import ast
|
|
8
|
+
import contextlib
|
|
9
|
+
import io
|
|
10
|
+
import traceback
|
|
11
|
+
|
|
12
|
+
FORBIDDEN_BUILTINS = frozenset(
|
|
13
|
+
{
|
|
14
|
+
"exec",
|
|
15
|
+
"eval",
|
|
16
|
+
"compile",
|
|
17
|
+
"__import__",
|
|
18
|
+
"open",
|
|
19
|
+
"exit",
|
|
20
|
+
"quit",
|
|
21
|
+
"breakpoint",
|
|
22
|
+
"input",
|
|
23
|
+
"getattr",
|
|
24
|
+
"setattr",
|
|
25
|
+
"delattr",
|
|
26
|
+
"globals",
|
|
27
|
+
"locals",
|
|
28
|
+
"vars",
|
|
29
|
+
"dir",
|
|
30
|
+
}
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
FORBIDDEN_MODULE_NAMES = frozenset(
|
|
34
|
+
{
|
|
35
|
+
"os",
|
|
36
|
+
"subprocess",
|
|
37
|
+
"sys",
|
|
38
|
+
"shutil",
|
|
39
|
+
"pathlib",
|
|
40
|
+
"importlib",
|
|
41
|
+
}
|
|
42
|
+
)
|
|
43
|
+
|
|
44
|
+
FORBIDDEN_ATTRIBUTES = frozenset(
|
|
45
|
+
{
|
|
46
|
+
"__globals__",
|
|
47
|
+
"__builtins__",
|
|
48
|
+
"__subclasses__",
|
|
49
|
+
"__bases__",
|
|
50
|
+
"__code__",
|
|
51
|
+
"__import__",
|
|
52
|
+
"__dict__",
|
|
53
|
+
}
|
|
54
|
+
)
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _check_import(node: ast.Import, allowed_imports: list[str]) -> str:
|
|
58
|
+
"""Block imports whose root module is not in the whitelist."""
|
|
59
|
+
for alias in node.names:
|
|
60
|
+
root = alias.name.split(".")[0]
|
|
61
|
+
if root not in allowed_imports:
|
|
62
|
+
return f"Forbidden import: {alias.name}"
|
|
63
|
+
return ""
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _check_import_from(node: ast.ImportFrom, allowed_imports: list[str]) -> str:
|
|
67
|
+
"""Block star imports unconditionally; block modules not in the whitelist."""
|
|
68
|
+
for alias in node.names:
|
|
69
|
+
if alias.name == "*":
|
|
70
|
+
return f"Forbidden star import: from {node.module} import *"
|
|
71
|
+
if node.module:
|
|
72
|
+
root = node.module.split(".")[0]
|
|
73
|
+
if root not in allowed_imports:
|
|
74
|
+
return f"Forbidden import: {node.module}"
|
|
75
|
+
return ""
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _check_name(node: ast.Name) -> str:
|
|
79
|
+
"""Block dangerous builtin calls/references and dangerous module names."""
|
|
80
|
+
if node.id in FORBIDDEN_BUILTINS:
|
|
81
|
+
return f"Forbidden builtin: {node.id}"
|
|
82
|
+
if node.id in FORBIDDEN_MODULE_NAMES:
|
|
83
|
+
return f"Forbidden name: {node.id}"
|
|
84
|
+
return ""
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def _check_attribute(node: ast.Attribute) -> str:
|
|
88
|
+
"""Block dangerous dunder attribute access."""
|
|
89
|
+
if node.attr in FORBIDDEN_ATTRIBUTES:
|
|
90
|
+
return f"Forbidden attribute access: {node.attr}"
|
|
91
|
+
return ""
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def validate(source: str, allowed_imports: list[str]) -> str:
|
|
95
|
+
"""Use AST parsing against a set of rules to decide if the code is safe.
|
|
96
|
+
|
|
97
|
+
Args:
|
|
98
|
+
source: Python source code produced by the model.
|
|
99
|
+
allowed_imports: Whitelist of importable module names.
|
|
100
|
+
|
|
101
|
+
Returns:
|
|
102
|
+
``""`` on success, or a human-readable reason string on rejection.
|
|
103
|
+
"""
|
|
104
|
+
try:
|
|
105
|
+
tree = ast.parse(source)
|
|
106
|
+
except SyntaxError as e:
|
|
107
|
+
return f"Syntax error: {e.msg}"
|
|
108
|
+
|
|
109
|
+
reason = ""
|
|
110
|
+
for node in ast.walk(tree):
|
|
111
|
+
if isinstance(node, ast.Import):
|
|
112
|
+
reason = _check_import(node, allowed_imports)
|
|
113
|
+
elif isinstance(node, ast.ImportFrom):
|
|
114
|
+
reason = _check_import_from(node, allowed_imports)
|
|
115
|
+
elif isinstance(node, ast.Name):
|
|
116
|
+
reason = _check_name(node)
|
|
117
|
+
elif isinstance(node, ast.Attribute):
|
|
118
|
+
reason = _check_attribute(node)
|
|
119
|
+
# Exit if any checker populated `reason` -> code is invalid
|
|
120
|
+
if reason:
|
|
121
|
+
break
|
|
122
|
+
|
|
123
|
+
return reason
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def execute(source: str, namespace: dict) -> str:
|
|
127
|
+
"""Execute *source* code requested by the Agent inside *namespace*.
|
|
128
|
+
|
|
129
|
+
Runs synchronously in the calling thread. There is no timeout guard:
|
|
130
|
+
the model could theoretically generate an infinite loop that we cannot
|
|
131
|
+
catch at the AST validation level.
|
|
132
|
+
|
|
133
|
+
Args:
|
|
134
|
+
source: Python/SQLAlchemy source code produced by the model.
|
|
135
|
+
namespace: Dict of python symbols available during execution.
|
|
136
|
+
**Mutated in-place** — new bindings created by the code
|
|
137
|
+
persist in *namespace* after this call returns.
|
|
138
|
+
|
|
139
|
+
Returns:
|
|
140
|
+
A string with stdout output, a traceback, or a status message.
|
|
141
|
+
"""
|
|
142
|
+
buf = io.StringIO()
|
|
143
|
+
try:
|
|
144
|
+
compiled = compile(source, "<agent>", "exec")
|
|
145
|
+
with contextlib.redirect_stdout(buf):
|
|
146
|
+
exec(compiled, namespace)
|
|
147
|
+
return buf.getvalue() or "Code executed successfully but produced no stdout."
|
|
148
|
+
except Exception:
|
|
149
|
+
return traceback.format_exc()
|
llmalchemy/context.py
ADDED
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
"""Contains the utils to engineer the context passed to the agent."""
|
|
2
|
+
|
|
3
|
+
import inspect
|
|
4
|
+
import re
|
|
5
|
+
import warnings
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
from lmdk import render_template
|
|
9
|
+
from sqlalchemy.orm import DeclarativeBase
|
|
10
|
+
|
|
11
|
+
from .tools import Tool
|
|
12
|
+
|
|
13
|
+
_TEMPLATE_PATH = Path(__file__).parent / "prompt.jinja"
|
|
14
|
+
|
|
15
|
+
_REQUIRED_MARKERS = ("SCHEMA", "SYMBOLS", "TOOLS")
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class LLMAlchemyPromptWarning(UserWarning):
|
|
19
|
+
"""Warning for prompt templates missing one of the documented Jinja variables."""
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _render_schema_source(base: type[DeclarativeBase]) -> str:
|
|
23
|
+
"""Return the source code of ORM classes for the LM prompt.
|
|
24
|
+
|
|
25
|
+
Extracts source code via ``inspect.getsource`` for every mapped class
|
|
26
|
+
registered under *base*. The sources are concatenated separated by
|
|
27
|
+
blank lines.
|
|
28
|
+
"""
|
|
29
|
+
sources = [inspect.getsource(cls) for cls in base.__subclasses__()]
|
|
30
|
+
return "\n\n".join(sources)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _render_symbols(descriptions: dict[str, str]) -> str:
|
|
34
|
+
"""Format the symbol descriptions dict as a Markdown bullet list."""
|
|
35
|
+
return "\n".join(f"- `{name}`: {desc}" for name, desc in descriptions.items())
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _render_tools_summary(tools: list[Tool]) -> str:
|
|
39
|
+
"""Render name + one-liner for each tool (empty string if no tools)."""
|
|
40
|
+
if not tools:
|
|
41
|
+
return ""
|
|
42
|
+
return "\n".join(f"- `{t.name}`: {t.short_description}" for t in tools)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _check_markers(source: str) -> None:
|
|
46
|
+
"""Warn for each required Jinja variable missing from the raw template source."""
|
|
47
|
+
for marker in _REQUIRED_MARKERS:
|
|
48
|
+
if not re.search(r"\{\{\s*" + marker + r"\s*\}\}", source):
|
|
49
|
+
warnings.warn(
|
|
50
|
+
f"Prompt template is missing the `{{{{ {marker} }}}}` variable; "
|
|
51
|
+
"the agent may behave unpredictably.",
|
|
52
|
+
LLMAlchemyPromptWarning,
|
|
53
|
+
stacklevel=2,
|
|
54
|
+
)
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def render(
|
|
58
|
+
base: type[DeclarativeBase],
|
|
59
|
+
tools: list[Tool],
|
|
60
|
+
descriptions: dict[str, str],
|
|
61
|
+
template: str | Path | None = None,
|
|
62
|
+
) -> str:
|
|
63
|
+
"""Build the system instruction for the LM with all context parts.
|
|
64
|
+
|
|
65
|
+
Args:
|
|
66
|
+
base: The declarative base describing the schema.
|
|
67
|
+
tools: User-provided tools registered for this run.
|
|
68
|
+
descriptions: ``{name: description}`` of every namespace symbol,
|
|
69
|
+
as returned by ``_init_namespace`` in ``agent.py``.
|
|
70
|
+
template: A Jinja template source string, a ``Path`` to a template
|
|
71
|
+
file, or ``None`` to use the shipped default.
|
|
72
|
+
"""
|
|
73
|
+
if template is None:
|
|
74
|
+
path: Path = _TEMPLATE_PATH
|
|
75
|
+
source = path.read_text()
|
|
76
|
+
_check_markers(source)
|
|
77
|
+
return render_template(
|
|
78
|
+
template=source,
|
|
79
|
+
SCHEMA=_render_schema_source(base=base),
|
|
80
|
+
SYMBOLS=_render_symbols(descriptions),
|
|
81
|
+
TOOLS=_render_tools_summary(tools),
|
|
82
|
+
)
|
|
83
|
+
source = template.read_text() if isinstance(template, Path) else template
|
|
84
|
+
_check_markers(source)
|
|
85
|
+
return render_template(
|
|
86
|
+
template=source,
|
|
87
|
+
SCHEMA=_render_schema_source(base=base),
|
|
88
|
+
SYMBOLS=_render_symbols(descriptions),
|
|
89
|
+
TOOLS=_render_tools_summary(tools),
|
|
90
|
+
)
|
llmalchemy/database.py
ADDED
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
"""Contains the utilities that access or modify the database."""
|
|
2
|
+
|
|
3
|
+
import weakref
|
|
4
|
+
|
|
5
|
+
from sqlalchemy import create_engine
|
|
6
|
+
from sqlalchemy.orm import DeclarativeBase, Session
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def deserialize(data: dict[str, list[dict]], base: type[DeclarativeBase]) -> Session:
|
|
10
|
+
"""Unpack a JSON-serialised database state into an SQLAlchemy session.
|
|
11
|
+
|
|
12
|
+
Creates an in-memory SQLite database, issues ``Base.metadata.create_all``,
|
|
13
|
+
and populates every table from *data*.
|
|
14
|
+
|
|
15
|
+
Args:
|
|
16
|
+
data: Mapping of ``{table_name: [row_dict, ...]}``.
|
|
17
|
+
base: The declarative base whose metadata describes the schema.
|
|
18
|
+
|
|
19
|
+
Returns:
|
|
20
|
+
A ready-to-use SQLAlchemy ``Session`` bound to the in-memory database.
|
|
21
|
+
"""
|
|
22
|
+
engine = create_engine("sqlite://")
|
|
23
|
+
base.metadata.create_all(engine)
|
|
24
|
+
session = Session(engine)
|
|
25
|
+
# Ensure the underlying sqlite3.Connection is closed when the session
|
|
26
|
+
# becomes unreachable, avoiding Python 3.13 ResourceWarnings.
|
|
27
|
+
weakref.finalize(session, engine.dispose)
|
|
28
|
+
|
|
29
|
+
# Build a lookup from table name to mapped class
|
|
30
|
+
cls_by_table: dict[str, type] = {}
|
|
31
|
+
for cls in base.__subclasses__():
|
|
32
|
+
table_name = cls.__tablename__ if hasattr(cls, "__tablename__") else cls.__table__.name
|
|
33
|
+
cls_by_table[table_name] = cls
|
|
34
|
+
|
|
35
|
+
for table_name, rows in data.items():
|
|
36
|
+
cls = cls_by_table.get(table_name)
|
|
37
|
+
if cls is None:
|
|
38
|
+
continue
|
|
39
|
+
for row in rows:
|
|
40
|
+
session.add(cls(**row))
|
|
41
|
+
|
|
42
|
+
session.commit()
|
|
43
|
+
return session
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def serialize(session: Session, base: type[DeclarativeBase]) -> dict[str, list[dict]]:
|
|
47
|
+
"""Freeze the current database state into a JSON-serialisable dict.
|
|
48
|
+
|
|
49
|
+
Args:
|
|
50
|
+
session: The active session to read from.
|
|
51
|
+
base: The declarative base whose metadata describes the schema.
|
|
52
|
+
|
|
53
|
+
Returns:
|
|
54
|
+
``{table_name: [row_dict, ...]}`` for every table in the schema.
|
|
55
|
+
"""
|
|
56
|
+
result: dict[str, list[dict]] = {}
|
|
57
|
+
for cls in base.__subclasses__():
|
|
58
|
+
table_name = cls.__tablename__ if hasattr(cls, "__tablename__") else cls.__table__.name
|
|
59
|
+
columns = [c.key for c in cls.__table__.columns]
|
|
60
|
+
rows = session.query(cls).all()
|
|
61
|
+
result[table_name] = [{col: getattr(row, col) for col in columns} for row in rows]
|
|
62
|
+
return result
|
llmalchemy/prompt.jinja
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
You are an expert assistant with access to a relational database containing the user's project.
|
|
2
|
+
You can interact with the database using Python code and SQLAlchemy.
|
|
3
|
+
|
|
4
|
+
# Guidelines
|
|
5
|
+
- **Code Execution**: Run Python code to query or modify the database. Use `print()` to inspect results.
|
|
6
|
+
- **Persistence**: The Python namespace is persistent. Imports and variables carry over between cells.
|
|
7
|
+
- **Efficiency**: Keep `print()` statements concise. Avoid redundant imports or session initialization.
|
|
8
|
+
- **Iteration**: Use the execution output (sent as an automatic user message) to decide if you need to run more code or provide a final answer.
|
|
9
|
+
{% if TOOLS %}- **Tools**: Several tools are available in your namespace. Use `print(disclose(name))` to discover a tool's full signature and docstring before calling it.{% endif %}
|
|
10
|
+
|
|
11
|
+
# Database Schema
|
|
12
|
+
```python
|
|
13
|
+
{{ SCHEMA }}
|
|
14
|
+
```
|
|
15
|
+
|
|
16
|
+
# Available Symbols
|
|
17
|
+
The following symbols are pre-loaded in your namespace:
|
|
18
|
+
{{ SYMBOLS }}
|
|
19
|
+
{% if TOOLS %}
|
|
20
|
+
# Available Tools
|
|
21
|
+
{{ TOOLS }}
|
|
22
|
+
{% endif %}
|
llmalchemy/tools.py
ADDED
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
"""Logic to expose functions to the model."""
|
|
2
|
+
|
|
3
|
+
import inspect
|
|
4
|
+
from collections.abc import Callable
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
from typing import Any, Protocol
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class HasMetadata(Protocol):
|
|
10
|
+
"""Protocol for objects that have __name__ and __doc__."""
|
|
11
|
+
|
|
12
|
+
__name__: str
|
|
13
|
+
__doc__: str | None
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@dataclass(frozen=True)
|
|
17
|
+
class Tool:
|
|
18
|
+
"""Metadata wrapper around a user-provided function.
|
|
19
|
+
|
|
20
|
+
Attributes:
|
|
21
|
+
fn: The original callable.
|
|
22
|
+
name: Function name (used as the namespace key).
|
|
23
|
+
short_description: First line of the docstring.
|
|
24
|
+
full_description: Full signature and docstring for discovery.
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
fn: Callable[..., Any]
|
|
28
|
+
name: str
|
|
29
|
+
short_description: str
|
|
30
|
+
full_description: str
|
|
31
|
+
|
|
32
|
+
@classmethod
|
|
33
|
+
def from_function(cls, fn: Any) -> "Tool":
|
|
34
|
+
"""Build a ``Tool`` from a plain function."""
|
|
35
|
+
name = getattr(fn, "__name__", "unknown")
|
|
36
|
+
doc = inspect.cleandoc(getattr(fn, "__doc__", "") or "")
|
|
37
|
+
short = doc.split("\n", 1)[0] if doc else ""
|
|
38
|
+
sig = inspect.signature(fn)
|
|
39
|
+
full = f'def {name}{sig}:\n """{doc}"""' if doc else f"def {name}{sig}:"
|
|
40
|
+
return cls(fn=fn, name=name, short_description=short, full_description=full)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def tool(fn: Any) -> Tool:
|
|
44
|
+
"""Decorator that turns a function into a ``Tool``."""
|
|
45
|
+
return Tool.from_function(fn)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def make_disclose_fn(tools: list[Tool]) -> Callable:
|
|
49
|
+
"""Build the ``disclose`` closure injected into the agent namespace.
|
|
50
|
+
|
|
51
|
+
This is a *closure factory*: it builds ``lookup`` once from *tools*,
|
|
52
|
+
then returns an inner function that captures ``lookup``. Each call to
|
|
53
|
+
``make_disclose_fn`` produces a fresh ``disclose`` bound to exactly the
|
|
54
|
+
tools registered for that ``run()`` invocation.
|
|
55
|
+
|
|
56
|
+
Args:
|
|
57
|
+
tools: The list of tools registered for this run.
|
|
58
|
+
|
|
59
|
+
Returns:
|
|
60
|
+
A callable ``(name: str) -> str``.
|
|
61
|
+
"""
|
|
62
|
+
lookup = {t.name: t.full_description for t in tools}
|
|
63
|
+
|
|
64
|
+
def disclose(name: str) -> str:
|
|
65
|
+
"""Reveal the full signature and docstring of a tool.
|
|
66
|
+
|
|
67
|
+
Args:
|
|
68
|
+
name: The tool name to look up.
|
|
69
|
+
|
|
70
|
+
Returns:
|
|
71
|
+
The full description string, or an error message if not found.
|
|
72
|
+
"""
|
|
73
|
+
if name in lookup:
|
|
74
|
+
return lookup[name]
|
|
75
|
+
available = ", ".join(lookup) or "(none)"
|
|
76
|
+
return f"Unknown tool: {name!r}. Available tools: {available}"
|
|
77
|
+
|
|
78
|
+
return disclose
|
|
@@ -0,0 +1,308 @@
|
|
|
1
|
+
Metadata-Version: 2.3
|
|
2
|
+
Name: llmalchemy
|
|
3
|
+
Version: 1.4.1
|
|
4
|
+
Summary: A full-access agent pattern.
|
|
5
|
+
Author: Ignacio Llorca
|
|
6
|
+
Author-email: Ignacio Llorca <nllorca@proton.me>
|
|
7
|
+
License: MIT
|
|
8
|
+
Requires-Dist: lmdk
|
|
9
|
+
Requires-Dist: sqlalchemy
|
|
10
|
+
Requires-Python: >=3.13
|
|
11
|
+
Description-Content-Type: text/markdown
|
|
12
|
+
|
|
13
|
+

|
|
14
|
+
|
|
15
|
+
Every **new operation** you want to allow a user to perform on your application's data model means **new logic** (backend) and a **new screen** (frontend). Then, the user has to learn how to perform that particular operation.
|
|
16
|
+
|
|
17
|
+
`llmalchemy` removes that layer. Hand an LM your ORM schema and a sandboxed Python environment. **The user requests any operation through natural language**. The agent composes its own queries and transformations as code in a single turn.
|
|
18
|
+
|
|
19
|
+
**You do not need to define a thousand GUIs for each possible action, nor a thousand tools for the LM to leverage** (GET X, POST Y, etc.): the abstraction to freely manipulate the data model already exists: it is SQL. ORM and python add an infinite number of possibilities on top.
|
|
20
|
+
|
|
21
|
+

|
|
22
|
+
|
|
23
|
+
**Read the complete motivation / reasoning in the [whitepaper](#whitepaper-why-llmalchemy)**
|
|
24
|
+
|
|
25
|
+
## Installation
|
|
26
|
+
|
|
27
|
+
`uv add llmalchemy`
|
|
28
|
+
|
|
29
|
+
## Usage
|
|
30
|
+
|
|
31
|
+
The only thing you need to define upfront is your database schema through an sqlalchemy declarative base:
|
|
32
|
+
```python
|
|
33
|
+
from llmalchemy.agent import State, run
|
|
34
|
+
from lmdk import UserMessage
|
|
35
|
+
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
|
|
36
|
+
|
|
37
|
+
class Base(DeclarativeBase): ...
|
|
38
|
+
|
|
39
|
+
class Author(Base):
|
|
40
|
+
"""An author who can write many books."""
|
|
41
|
+
|
|
42
|
+
__tablename__ = "authors"
|
|
43
|
+
|
|
44
|
+
id: Mapped[int] = mapped_column(primary_key=True, autoincrement=True)
|
|
45
|
+
name: Mapped[str] = mapped_column(String(120))
|
|
46
|
+
|
|
47
|
+
books: Mapped[list["Book"]] = relationship(back_populates="author")
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
class Book(Base):
|
|
51
|
+
"""A book belonging to a single author."""
|
|
52
|
+
|
|
53
|
+
__tablename__ = "books"
|
|
54
|
+
|
|
55
|
+
id: Mapped[int] = mapped_column(primary_key=True, autoincrement=True)
|
|
56
|
+
title: Mapped[str] = mapped_column(String(200))
|
|
57
|
+
author_id: Mapped[int] = mapped_column(ForeignKey("authors.id"))
|
|
58
|
+
|
|
59
|
+
author: Mapped["Author"] = relationship(back_populates="books")
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
model = "vertex:gemini-3-flash-preview"
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
For the minimal run, append the first message to the conversation and simply iterate over the `run` call.
|
|
66
|
+
You will receive `Events` with the messages and code results performed by the agent, together with precise signals indicating the agents loop state (waiting for the LM completion, executing code, etc.):
|
|
67
|
+
|
|
68
|
+
```python
|
|
69
|
+
state = State()
|
|
70
|
+
state.messages.append(UserMessage("Add authors Tolkien and Dhalia de la Cerda, and two books for each."))
|
|
71
|
+
|
|
72
|
+
for event in run(state=state, base=Base, model=model):
|
|
73
|
+
print(event)
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
Example output (abbreviated):
|
|
77
|
+
```text
|
|
78
|
+
SystemInstructionEvent(content='You are a coding agent... SCHEMA: class Author...')
|
|
79
|
+
SignalEvent(signal=<Signal.COMPLETION: 'COMPLETION'>)
|
|
80
|
+
MessageEvent(message=AssistantMessage(message="I'll add the authors and their books.", code="a1 = Author(name='J.R.R. Tolkien')\na2 = Author(name='Dhalia de la Cerda')\nsession.add_all([a1, a2])\nsession.flush()\nsession.add_all([\n Book(title='The Hobbit', author_id=a1.id),\n Book(title='The Lord of the Rings', author_id=a1.id),\n Book(title='Desde los zulos', author_id=a2.id),\n Book(title='Perras de reserva', author_id=a2.id),\n])\nsession.commit()\nprint('ok')"))
|
|
81
|
+
SignalEvent(signal=<Signal.VALIDATION: 'VALIDATION'>)
|
|
82
|
+
SignalEvent(signal=<Signal.EXECUTION: 'EXECUTION'>)
|
|
83
|
+
MessageEvent(message=UserMessage(content='Execution result:\nok\n'))
|
|
84
|
+
SignalEvent(signal=<Signal.COMPLETION: 'COMPLETION'>)
|
|
85
|
+
MessageEvent(message=AssistantMessage(message='Done — added Tolkien and Dhalia de la Cerda with two books each.', code=''))
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
The `state` persists across calls, just append a new `UserMessage` and call `run()` again to continue the conversation.
|
|
89
|
+
|
|
90
|
+
<details>
|
|
91
|
+
<summary>Custom tools</summary>
|
|
92
|
+
You can define any pre-built function that you want the agent to have access to.
|
|
93
|
+
These can modify the database or do something completely different, like performing math operations or calling another API.
|
|
94
|
+
|
|
95
|
+
```python
|
|
96
|
+
from sqlalchemy.orm import Session
|
|
97
|
+
from llmalchemy.tools import tool
|
|
98
|
+
|
|
99
|
+
@tool
|
|
100
|
+
def get_author_catalog(author: str, session: Session) -> list[str]:
|
|
101
|
+
"""List all book titles for the given author."""
|
|
102
|
+
obj = session.query(Author).filter(Author.name == author).first()
|
|
103
|
+
return [b.title for b in obj.books] if obj else []
|
|
104
|
+
|
|
105
|
+
state.messages.append(UserMessage("what's the catalog for Dhalia de la Cerda?"))
|
|
106
|
+
for event in run(state=state, base=Base, model=model, tools=[get_author_catalog]):
|
|
107
|
+
print(event)
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
Example output (abbreviated):
|
|
111
|
+
```text
|
|
112
|
+
SignalEvent(signal=<Signal.COMPLETION: 'COMPLETION'>)
|
|
113
|
+
MessageEvent(message=AssistantMessage(message='Let me check the tool signature first.', code="disclose('get_author_catalog')"))
|
|
114
|
+
SignalEvent(signal=<Signal.EXECUTION: 'EXECUTION'>)
|
|
115
|
+
MessageEvent(message=UserMessage(content="Execution result:\nget_author_catalog(author: str, session: Session) -> list[str]\nList all book titles for the given author.\n"))
|
|
116
|
+
SignalEvent(signal=<Signal.COMPLETION: 'COMPLETION'>)
|
|
117
|
+
MessageEvent(message=AssistantMessage(message='', code="print(get_author_catalog('Dhalia de la Cerda', session))"))
|
|
118
|
+
SignalEvent(signal=<Signal.EXECUTION: 'EXECUTION'>)
|
|
119
|
+
MessageEvent(message=UserMessage(content="Execution result:\n['Desde los zulos', 'Perras de reserva']\n"))
|
|
120
|
+
SignalEvent(signal=<Signal.COMPLETION: 'COMPLETION'>)
|
|
121
|
+
MessageEvent(message=AssistantMessage(message="Dhalia de la Cerda's catalog: 'Desde los zulos' and 'Perras de reserva'.", code=''))
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
Only the tool name and first docstring line are shown to the agent up-front.
|
|
125
|
+
The agent calls `disclose("get_author_catalog")` to inspect the full signature on demand.
|
|
126
|
+
</details>
|
|
127
|
+
|
|
128
|
+
<details>
|
|
129
|
+
<summary>Allowed imports</summary>
|
|
130
|
+
For safety, no imports are allowed inside agent-generated code.
|
|
131
|
+
Whitelist any stdlib or third-party module the agent may need.
|
|
132
|
+
|
|
133
|
+
```python
|
|
134
|
+
state.messages.append(UserMessage("what day is today?"))
|
|
135
|
+
for event in run( state=state, base=Base, model=model, allowed_imports=["datetime"]):
|
|
136
|
+
print(event)
|
|
137
|
+
```
|
|
138
|
+
|
|
139
|
+
Example output (abbreviated):
|
|
140
|
+
```text
|
|
141
|
+
SignalEvent(signal=<Signal.COMPLETION: 'COMPLETION'>)
|
|
142
|
+
MessageEvent(message=AssistantMessage(message='', code='import datetime\nprint(datetime.date.today().isoformat())'))
|
|
143
|
+
SignalEvent(signal=<Signal.VALIDATION: 'VALIDATION'>)
|
|
144
|
+
SignalEvent(signal=<Signal.EXECUTION: 'EXECUTION'>)
|
|
145
|
+
MessageEvent(message=UserMessage(content='Execution result:\n2026-04-21\n'))
|
|
146
|
+
SignalEvent(signal=<Signal.COMPLETION: 'COMPLETION'>)
|
|
147
|
+
MessageEvent(message=AssistantMessage(message='Today is 2026-04-21.', code=''))
|
|
148
|
+
```
|
|
149
|
+
|
|
150
|
+
If the agent tries to import something not whitelisted, validation fails and it gets a second chance:
|
|
151
|
+
```text
|
|
152
|
+
MessageEvent(message=UserMessage(content="Code rejected: import of 'os' is not allowed"))
|
|
153
|
+
```
|
|
154
|
+
</details>
|
|
155
|
+
|
|
156
|
+
<details>
|
|
157
|
+
<summary>Custom system prompt</summary>
|
|
158
|
+
You can pass a Jinja template to override the default one (see `src/llmalchemy/prompt.jinja`).
|
|
159
|
+
It is recommended that the template contains vars {{ SCHEMA }}, {{ SYMBOLS }} and {{ TOOLS }}.
|
|
160
|
+
|
|
161
|
+
```python
|
|
162
|
+
prompt = """Write python code to answer user requests. You have access to {{ SCHEMA }}, {{ SYMBOLS }} and {{ TOOLS }}"""
|
|
163
|
+
for event in run(
|
|
164
|
+
state=state,
|
|
165
|
+
base=Base,
|
|
166
|
+
model=model,
|
|
167
|
+
prompt_template="path/to/prompt.jinja",
|
|
168
|
+
):
|
|
169
|
+
print(event)
|
|
170
|
+
```
|
|
171
|
+
|
|
172
|
+
The emitted events are the same as in the minimal example; only the `SystemInstructionEvent` content changes to reflect your custom template:
|
|
173
|
+
```text
|
|
174
|
+
SystemInstructionEvent(content='Write python code to answer user requests. You have access to <schema...>, <symbols...> and <tools...>')
|
|
175
|
+
```
|
|
176
|
+
</details>
|
|
177
|
+
|
|
178
|
+
<details>
|
|
179
|
+
<summary>Output extensions</summary>
|
|
180
|
+
By default, the agent responds with a `message` for the user and optional `code` to perform actions.
|
|
181
|
+
You can specify any additional fields for the LM to fill.
|
|
182
|
+
|
|
183
|
+
```python
|
|
184
|
+
from pydantic import BaseModel, Field
|
|
185
|
+
|
|
186
|
+
class Reasoning(BaseModel):
|
|
187
|
+
thoughts: str = Field(description="Scratchpad before answering.")
|
|
188
|
+
confidence: float = Field(description="0..1 confidence score.")
|
|
189
|
+
|
|
190
|
+
for event in run(
|
|
191
|
+
state=state,
|
|
192
|
+
base=Base,
|
|
193
|
+
model=model,
|
|
194
|
+
output_extensions=Reasoning,
|
|
195
|
+
):
|
|
196
|
+
print(event)
|
|
197
|
+
# Extra fields ride along on the yielded AssistantMessage for you to consume:
|
|
198
|
+
#
|
|
199
|
+
# MessageEvent(message=AssistantMessage(
|
|
200
|
+
# thoughts='The user asked X; I should query Y then aggregate by Z.',
|
|
201
|
+
# confidence=0.82,
|
|
202
|
+
# message='Here are the results ...',
|
|
203
|
+
# code='...',
|
|
204
|
+
# ))
|
|
205
|
+
```
|
|
206
|
+
</details>
|
|
207
|
+
|
|
208
|
+
## How it works
|
|
209
|
+
In short: **user input → LM generates SQLAlchemy code → validate → execute → return results to LM → repeat until done**
|
|
210
|
+
|
|
211
|
+
```mermaid
|
|
212
|
+
flowchart TD
|
|
213
|
+
Start([run]) --> Init[Initialize session,<br/>namespace, system prompt]
|
|
214
|
+
Init --> Complete[LM completion<br/>→ message + code]
|
|
215
|
+
Complete --> HasCode{code?}
|
|
216
|
+
HasCode -- no --> End([return to user])
|
|
217
|
+
HasCode -- yes --> MaxLoops{loops ≥ MAX?}
|
|
218
|
+
MaxLoops -- yes --> Exceeded[signal: EXCEEDED] --> End
|
|
219
|
+
MaxLoops -- no --> Validate[validate code]
|
|
220
|
+
Validate --> Valid{valid?}
|
|
221
|
+
Valid -- no --> Reject[append 'Code rejected'<br/>user message]
|
|
222
|
+
Reject --> Complete
|
|
223
|
+
Valid -- yes --> Execute[execute code<br/>in namespace]
|
|
224
|
+
Execute --> Result[append 'Execution result'<br/>user message]
|
|
225
|
+
Result --> Complete
|
|
226
|
+
```
|
|
227
|
+
|
|
228
|
+
1. **Schema as context** (`context.py`): The source code of your ORM classes is extracted via `inspect.getsource` and rendered into a Jinja system prompt alongside available symbols and tools.
|
|
229
|
+
|
|
230
|
+
2. **Agentic loop** (`agent.py`): The LM produces structured output — a message and optional Python code. If code is present, it's validated, executed, and the result is fed back as a user message. The loop continues until the LM responds without code or hits `MAX_LOOPS`.
|
|
231
|
+
|
|
232
|
+
3. **Sandboxed execution** (`code.py`): An AST pass blocks dangerous builtins (`exec`, `eval`, `open`…), forbidden modules (`os`, `subprocess`…), dangerous dunder access, and enforces an import whitelist. Safe code runs in a persistent namespace with stdout captured.
|
|
233
|
+
|
|
234
|
+
4. **Optional tools** (`tools.py`): Developers can register custom functions with the `@tool` decorator. Only tool names and one-liners appear in the prompt — the LM calls `disclose(name)` to see full signatures on demand, keeping the context window lean.
|
|
235
|
+
|
|
236
|
+
## Whitepaper (why `llmalchemy`)
|
|
237
|
+
|
|
238
|
+
### Tool calling hits a wall
|
|
239
|
+
|
|
240
|
+
The standard agentic pattern is function calling: define tools, let the LM pick one, read the result, repeat. This is clean and safe, but it doesn't scale.
|
|
241
|
+
|
|
242
|
+
Real applications have complex data models. To give the LM meaningful access you end up writing dozens of tools, each one crowding the context window. Every compound operation (filter, then aggregate, then compare) either needs its own dedicated tool or forces the agent to chain calls across multiple LM completions — slow and expensive. And you become the bottleneck: every new user need is a new tool to design, implement, test and document.
|
|
243
|
+
|
|
244
|
+
### Code as the action space
|
|
245
|
+
|
|
246
|
+
Research shows that letting LMs write and execute code instead of picking from a discrete set of tools produces stronger agents[[1](https://arxiv.org/abs/2402.01030)][[2](https://arxiv.org/pdf/2401.00812)][[3](https://arxiv.org/pdf/2411.01747)][[4](https://platform.claude.com/docs/en/agents-and-tools/tool-use/programmatic-tool-calling)][[5](https://blog.cloudflare.com/code-mode/)]. Code gives the model composition (chain operations in one turn), control flow (loops, conditionals, error handling), and self-extension (define helper functions that persist in the namespace). A single code block can do what would otherwise take a long chain of tool calls.
|
|
247
|
+
|
|
248
|
+
But code execution alone doesn't solve the data access problem. The LM still needs some interface to read and modify application state. You're back to writing wrapper functions — unless the right abstraction already exists.
|
|
249
|
+
|
|
250
|
+
### Relational algebra is the interface you don't have to build
|
|
251
|
+
|
|
252
|
+
And it does. Relational algebra is a solved discipline: decades of refinement behind optimal ways of organizing and querying structured data. SQL databases implement it. ORMs like SQLAlchemy wrap it in the same Python the LM is already writing.
|
|
253
|
+
|
|
254
|
+
By placing an ORM session and the model classes in the agent's execution namespace, `llmalchemy` gives the LM full, structured access to the data without a single hand-crafted tool. The developer defines the schema once — which they'd do anyway. The LM handles everything else.
|
|
255
|
+
|
|
256
|
+
### What this means for applications
|
|
257
|
+
|
|
258
|
+
Traditional software requires two layers of work on top of the data model:
|
|
259
|
+
|
|
260
|
+
1. **Business logic** (backend) — functions and endpoints for every operation users might need.
|
|
261
|
+
2. **UI workflows** (frontend) — screens, forms and click sequences to expose those operations.
|
|
262
|
+
|
|
263
|
+
Both layers grow with the complexity of the data model and the operations pipelines that we want to allow for the user.
|
|
264
|
+
|
|
265
|
+
With `llmalchemy`, the developer defines the schema and optionally a handful of tools for things that genuinely require custom logic (sending emails, calling external APIs, very complex workflows). Everything else — every query, every data transformation, every "find all X where Y and then update Z" — the LM composes on the fly.
|
|
266
|
+
|
|
267
|
+
For users, this replaces navigating menus and filling forms with describing what they want. It removes the mismatch between what the user is thinking and the rigid paths a GUI offers.
|
|
268
|
+
|
|
269
|
+
### Beyond file systems
|
|
270
|
+
|
|
271
|
+
Today's coding agents (Opencode, Pi, Claude Code) prove that LMs can navigate file structures effectively with tools as simple as `bash`, `read`, `write` and `edit`. But file systems are structurally simple: trees of named nodes with blob contents.
|
|
272
|
+
|
|
273
|
+
Application data is a different beast. Dozens of entity types, foreign keys, many-to-many relationships, constraints, cascading dependencies. Relational data is orders of magnitude richer than a directory tree. The ORM gives the LM the right abstraction: it thinks in terms of entities, relationships and queries rather than raw files.
|
|
274
|
+
|
|
275
|
+
## Development
|
|
276
|
+
|
|
277
|
+
This package uses [`lmdk`](https://github.com/nachollorca/lmdk) to inference LLMs.
|
|
278
|
+
|
|
279
|
+
### Structure
|
|
280
|
+
```
|
|
281
|
+
src/llmalchemy/
|
|
282
|
+
├── agent.py # Entrypoint for the agentic loop
|
|
283
|
+
├── code.py # Sandboxed python env on which to run the agent requested code
|
|
284
|
+
├── context.py # Utils to engineer the context passed to the agent
|
|
285
|
+
├── database.py # Utils that access or modify the database
|
|
286
|
+
├── prompt.jinja # Default system instruction template
|
|
287
|
+
└── tools.py # Logic to expose functions to the model
|
|
288
|
+
```
|
|
289
|
+
|
|
290
|
+
### Tooling
|
|
291
|
+
We use `just` for development tasks. Use:
|
|
292
|
+
- `just sync`: Updates lockfile and syncs environment.
|
|
293
|
+
- `just format`: Lints and formats with `ruff`.
|
|
294
|
+
- `just check-types`: Static analysis with `ty`.
|
|
295
|
+
- `just check-complexity`: Cyclomatic complexity checks with `complexipy`.
|
|
296
|
+
- `just test`: Runs pytest with 90% coverage threshold.
|
|
297
|
+
|
|
298
|
+
See [`justfile`](justfile) for a complete list of dev commands.
|
|
299
|
+
|
|
300
|
+
### Contribute
|
|
301
|
+
1. **Hooks**: Install pre-commit hooks via `just install-hooks`. PRs will fail CI if linting/formatting is not applied.
|
|
302
|
+
2. **Issues**: Open an issue first using the default template.
|
|
303
|
+
3. **PRs**: Link your PR to the relevant issue using the PR template.
|
|
304
|
+
|
|
305
|
+
## License
|
|
306
|
+
MIT
|
|
307
|
+
|
|
308
|
+
_Made with [`mold`](https://github.com/nachollorca/mold) template_
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
llmalchemy/__init__.py,sha256=N3hmY3mE-9vX17gxbcB7If4loAW_PwrC2QI7o4MCdXU,107
|
|
2
|
+
llmalchemy/agent.py,sha256=zhha5LzYwCsENpvmCrtt_CxMmad72IOoTwRW7wDqMNs,9099
|
|
3
|
+
llmalchemy/code.py,sha256=2_qNfSdOI-1n-vQRzTb6Jd0BmY9GpTLzfwWha9Nm9YI,4343
|
|
4
|
+
llmalchemy/context.py,sha256=IswICwyvDi7gs2yVsiNqJK_lWyAKgu8FhYr7W1Iw9_Q,3112
|
|
5
|
+
llmalchemy/database.py,sha256=Zm-GZWL9j0Rj-RI0Ac1PtZOqljmn95nzy2Vic5Zd4GU,2261
|
|
6
|
+
llmalchemy/prompt.jinja,sha256=PkYErqZvocsQAr3GKyhDKL4yJOZ4k52mL8KtbAVlRMk,1011
|
|
7
|
+
llmalchemy/tools.py,sha256=jfw0R9mOODzmkyjHCfhtEzbAHsfuVSSxOeOuO4OTVK0,2410
|
|
8
|
+
llmalchemy-1.4.1.dist-info/WHEEL,sha256=q5IF0q2xCp3ktUFRCVWsQLjl2ChNlWXBJtnI1LCGdJ8,80
|
|
9
|
+
llmalchemy-1.4.1.dist-info/METADATA,sha256=3wodW6bZsJ5Vwz4LS-gykKPuuEfE16PI_GGdNmyeO_s,15429
|
|
10
|
+
llmalchemy-1.4.1.dist-info/RECORD,,
|