ai-pipeline-core 0.3.0__py3-none-any.whl → 0.3.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ai_pipeline_core/__init__.py +39 -2
- ai_pipeline_core/debug/__init__.py +26 -0
- ai_pipeline_core/debug/config.py +91 -0
- ai_pipeline_core/debug/content.py +706 -0
- ai_pipeline_core/debug/processor.py +99 -0
- ai_pipeline_core/debug/summary.py +236 -0
- ai_pipeline_core/debug/writer.py +913 -0
- ai_pipeline_core/documents/mime_type.py +28 -0
- ai_pipeline_core/flow/options.py +3 -3
- ai_pipeline_core/images/__init__.py +362 -0
- ai_pipeline_core/images/_processing.py +157 -0
- ai_pipeline_core/llm/ai_messages.py +41 -7
- ai_pipeline_core/llm/client.py +78 -17
- ai_pipeline_core/llm/model_response.py +5 -5
- ai_pipeline_core/llm/model_types.py +10 -12
- ai_pipeline_core/logging/logging_mixin.py +2 -2
- ai_pipeline_core/prompt_builder/prompt_builder.py +3 -3
- ai_pipeline_core/tracing.py +53 -1
- ai_pipeline_core/utils/deploy.py +214 -6
- {ai_pipeline_core-0.3.0.dist-info → ai_pipeline_core-0.3.4.dist-info}/METADATA +74 -8
- {ai_pipeline_core-0.3.0.dist-info → ai_pipeline_core-0.3.4.dist-info}/RECORD +23 -15
- {ai_pipeline_core-0.3.0.dist-info → ai_pipeline_core-0.3.4.dist-info}/WHEEL +0 -0
- {ai_pipeline_core-0.3.0.dist-info → ai_pipeline_core-0.3.4.dist-info}/licenses/LICENSE +0 -0
ai_pipeline_core/utils/deploy.py
CHANGED
|
@@ -18,10 +18,13 @@ Usage:
|
|
|
18
18
|
|
|
19
19
|
import argparse
|
|
20
20
|
import asyncio
|
|
21
|
+
import json
|
|
21
22
|
import subprocess
|
|
22
23
|
import sys
|
|
24
|
+
import tempfile
|
|
23
25
|
import tomllib
|
|
24
26
|
import traceback
|
|
27
|
+
from datetime import datetime, timezone
|
|
25
28
|
from pathlib import Path
|
|
26
29
|
from typing import Any, Optional
|
|
27
30
|
|
|
@@ -70,6 +73,8 @@ class Deployer:
|
|
|
70
73
|
with open(pyproject_path, "rb") as f:
|
|
71
74
|
data = tomllib.load(f)
|
|
72
75
|
|
|
76
|
+
self._pyproject_data = data
|
|
77
|
+
|
|
73
78
|
project = data.get("project", {})
|
|
74
79
|
name = project.get("name")
|
|
75
80
|
version = project.get("version")
|
|
@@ -160,6 +165,192 @@ class Deployer:
|
|
|
160
165
|
self._success(f"Built {tarball_path.name} ({tarball_path.stat().st_size // 1024} KB)")
|
|
161
166
|
return tarball_path
|
|
162
167
|
|
|
168
|
+
# -- Agent build/upload support --
|
|
169
|
+
|
|
170
|
+
def _load_agent_config(self) -> dict[str, dict[str, Any]]:
|
|
171
|
+
"""Load [tool.deploy.agents] from pyproject.toml.
|
|
172
|
+
|
|
173
|
+
Returns:
|
|
174
|
+
Dict mapping agent name to config (path, extra_vendor).
|
|
175
|
+
Empty dict if no agents configured.
|
|
176
|
+
"""
|
|
177
|
+
return self._pyproject_data.get("tool", {}).get("deploy", {}).get("agents", {})
|
|
178
|
+
|
|
179
|
+
def _get_cli_agents_source(self) -> str | None:
|
|
180
|
+
"""Get cli_agents_source path from [tool.deploy]."""
|
|
181
|
+
return self._pyproject_data.get("tool", {}).get("deploy", {}).get("cli_agents_source")
|
|
182
|
+
|
|
183
|
+
def _build_wheel_from_source(self, source_dir: Path) -> Path:
|
|
184
|
+
"""Build a wheel from a source directory.
|
|
185
|
+
|
|
186
|
+
Args:
|
|
187
|
+
source_dir: Directory containing pyproject.toml
|
|
188
|
+
|
|
189
|
+
Returns:
|
|
190
|
+
Path to built .whl file in a temp dist directory
|
|
191
|
+
"""
|
|
192
|
+
if not (source_dir / "pyproject.toml").exists():
|
|
193
|
+
self._die(f"No pyproject.toml in {source_dir}")
|
|
194
|
+
|
|
195
|
+
with tempfile.TemporaryDirectory() as tmpdir:
|
|
196
|
+
tmp_dist = Path(tmpdir) / "dist"
|
|
197
|
+
result = subprocess.run(
|
|
198
|
+
[sys.executable, "-m", "build", "--wheel", "--outdir", str(tmp_dist)],
|
|
199
|
+
cwd=source_dir,
|
|
200
|
+
capture_output=True,
|
|
201
|
+
text=True,
|
|
202
|
+
)
|
|
203
|
+
if result.returncode != 0:
|
|
204
|
+
self._die(f"Wheel build failed for {source_dir.name}:\n{result.stderr}")
|
|
205
|
+
|
|
206
|
+
wheels = list(tmp_dist.glob("*.whl"))
|
|
207
|
+
if not wheels:
|
|
208
|
+
self._die(f"No wheel produced for {source_dir.name}")
|
|
209
|
+
|
|
210
|
+
# Copy to persistent dist/ under source_dir
|
|
211
|
+
dist_dir = source_dir / "dist"
|
|
212
|
+
dist_dir.mkdir(exist_ok=True)
|
|
213
|
+
output = dist_dir / wheels[0].name
|
|
214
|
+
output.write_bytes(wheels[0].read_bytes())
|
|
215
|
+
return output
|
|
216
|
+
|
|
217
|
+
def _build_agents(self) -> dict[str, dict[str, Any]]:
|
|
218
|
+
"""Build agent wheels and manifests for all configured agents.
|
|
219
|
+
|
|
220
|
+
Returns:
|
|
221
|
+
Dict mapping agent name to build info:
|
|
222
|
+
{name: {"manifest_json": str, "files": {filename: Path}}}
|
|
223
|
+
Empty dict if no agents configured.
|
|
224
|
+
"""
|
|
225
|
+
agent_config = self._load_agent_config()
|
|
226
|
+
if not agent_config:
|
|
227
|
+
return {}
|
|
228
|
+
|
|
229
|
+
cli_agents_source = self._get_cli_agents_source()
|
|
230
|
+
if not cli_agents_source:
|
|
231
|
+
self._die(
|
|
232
|
+
"Agents configured in [tool.deploy.agents] but "
|
|
233
|
+
"[tool.deploy].cli_agents_source is not set.\n"
|
|
234
|
+
"Add to pyproject.toml:\n"
|
|
235
|
+
' [tool.deploy]\n cli_agents_source = "vendor/cli-agents"'
|
|
236
|
+
)
|
|
237
|
+
|
|
238
|
+
self._info(f"Building {len(agent_config)} agent(s): {', '.join(agent_config)}")
|
|
239
|
+
|
|
240
|
+
# Build cli-agents wheel once (shared across all agents)
|
|
241
|
+
cli_agents_dir = Path(cli_agents_source).resolve()
|
|
242
|
+
if not (cli_agents_dir / "pyproject.toml").exists():
|
|
243
|
+
self._die(f"cli-agents source not found at {cli_agents_dir}")
|
|
244
|
+
|
|
245
|
+
cli_agents_wheel = self._build_wheel_from_source(cli_agents_dir)
|
|
246
|
+
self._success(f"Built cli-agents wheel: {cli_agents_wheel.name}")
|
|
247
|
+
|
|
248
|
+
builds: dict[str, dict[str, Any]] = {}
|
|
249
|
+
|
|
250
|
+
for agent_name, config in agent_config.items():
|
|
251
|
+
agent_path = Path(config["path"]).resolve()
|
|
252
|
+
if not (agent_path / "pyproject.toml").exists():
|
|
253
|
+
self._die(
|
|
254
|
+
f"Agent '{agent_name}' path not found: {agent_path}\n"
|
|
255
|
+
f"Check [tool.deploy.agents.{agent_name}].path in pyproject.toml"
|
|
256
|
+
)
|
|
257
|
+
|
|
258
|
+
# Read module_name from agent's pyproject.toml
|
|
259
|
+
with open(agent_path / "pyproject.toml", "rb") as f:
|
|
260
|
+
agent_pyproject = tomllib.load(f)
|
|
261
|
+
|
|
262
|
+
module_name = agent_pyproject.get("tool", {}).get("agent", {}).get("module")
|
|
263
|
+
if not module_name:
|
|
264
|
+
self._die(
|
|
265
|
+
f"Agent '{agent_name}' missing [tool.agent].module in "
|
|
266
|
+
f"{agent_path / 'pyproject.toml'}\n"
|
|
267
|
+
f'Add:\n [tool.agent]\n module = "agent_{agent_name}"'
|
|
268
|
+
)
|
|
269
|
+
|
|
270
|
+
# Build agent wheel
|
|
271
|
+
agent_wheel = self._build_wheel_from_source(agent_path)
|
|
272
|
+
self._success(f"Built agent wheel: {agent_wheel.name}")
|
|
273
|
+
|
|
274
|
+
# Collect all files for this agent bundle
|
|
275
|
+
files: dict[str, Path] = {
|
|
276
|
+
agent_wheel.name: agent_wheel,
|
|
277
|
+
cli_agents_wheel.name: cli_agents_wheel,
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
# Build extra_vendor packages from repo root
|
|
281
|
+
vendor_packages: list[str] = []
|
|
282
|
+
extra_built: set[str] = set()
|
|
283
|
+
for vendor_name in config.get("extra_vendor", []):
|
|
284
|
+
extra_source_dir = Path(vendor_name).resolve()
|
|
285
|
+
if not (extra_source_dir / "pyproject.toml").exists():
|
|
286
|
+
self._die(
|
|
287
|
+
f"Extra vendor '{vendor_name}' for agent '{agent_name}' "
|
|
288
|
+
f"not found at {extra_source_dir}\n"
|
|
289
|
+
f"Ensure the directory exists at repo root with pyproject.toml"
|
|
290
|
+
)
|
|
291
|
+
vendor_wheel = self._build_wheel_from_source(extra_source_dir)
|
|
292
|
+
files[vendor_wheel.name] = vendor_wheel
|
|
293
|
+
vendor_packages.append(vendor_wheel.name)
|
|
294
|
+
extra_built.add(extra_source_dir.name.replace("-", "_"))
|
|
295
|
+
self._success(f"Built vendor wheel: {vendor_wheel.name}")
|
|
296
|
+
|
|
297
|
+
# Collect existing vendor/*.whl and vendor/*.tar.gz from agent directory,
|
|
298
|
+
# skipping packages already built from extra_vendor
|
|
299
|
+
agent_vendor_dir = agent_path / "vendor"
|
|
300
|
+
if agent_vendor_dir.exists():
|
|
301
|
+
for pkg in list(agent_vendor_dir.glob("*.whl")) + list(
|
|
302
|
+
agent_vendor_dir.glob("*.tar.gz")
|
|
303
|
+
):
|
|
304
|
+
pkg_base = pkg.name.split("-")[0].replace("-", "_")
|
|
305
|
+
if pkg.name not in files and pkg_base not in extra_built:
|
|
306
|
+
files[pkg.name] = pkg
|
|
307
|
+
vendor_packages.append(pkg.name)
|
|
308
|
+
|
|
309
|
+
# Write manifest (plain JSON dict, compatible with AgentManifest schema)
|
|
310
|
+
manifest = {
|
|
311
|
+
"module_name": module_name,
|
|
312
|
+
"agent_wheel": agent_wheel.name,
|
|
313
|
+
"cli_agents_wheel": cli_agents_wheel.name,
|
|
314
|
+
"vendor_packages": vendor_packages,
|
|
315
|
+
"built_at": datetime.now(timezone.utc).isoformat(),
|
|
316
|
+
}
|
|
317
|
+
manifest_json = json.dumps(manifest, indent=2)
|
|
318
|
+
|
|
319
|
+
builds[agent_name] = {"manifest_json": manifest_json, "files": files}
|
|
320
|
+
self._success(f"Agent '{agent_name}' bundle ready ({module_name}, {len(files)} files)")
|
|
321
|
+
|
|
322
|
+
return builds
|
|
323
|
+
|
|
324
|
+
async def _upload_agents(self, agent_builds: dict[str, dict[str, Any]]):
|
|
325
|
+
"""Upload agent bundles to GCS.
|
|
326
|
+
|
|
327
|
+
Args:
|
|
328
|
+
agent_builds: Output from _build_agents()
|
|
329
|
+
"""
|
|
330
|
+
if not agent_builds:
|
|
331
|
+
return
|
|
332
|
+
|
|
333
|
+
flow_folder = self.config["folder"].split("/", 1)[1] if "/" in self.config["folder"] else ""
|
|
334
|
+
base_uri = f"gs://{self.config['bucket']}/flows"
|
|
335
|
+
base_storage = await Storage.from_uri(base_uri)
|
|
336
|
+
base_storage = base_storage.with_base(flow_folder)
|
|
337
|
+
|
|
338
|
+
for agent_name, build_info in agent_builds.items():
|
|
339
|
+
agent_storage = base_storage.with_base(f"agents/{agent_name}")
|
|
340
|
+
self._info(f"Uploading agent '{agent_name}' bundle to {agent_storage.url_for('')}")
|
|
341
|
+
|
|
342
|
+
# Upload manifest
|
|
343
|
+
await agent_storage.write_bytes(
|
|
344
|
+
"manifest.json",
|
|
345
|
+
build_info["manifest_json"].encode(),
|
|
346
|
+
)
|
|
347
|
+
|
|
348
|
+
# Upload wheels
|
|
349
|
+
for filename, filepath in build_info["files"].items():
|
|
350
|
+
await agent_storage.write_bytes(filename, filepath.read_bytes())
|
|
351
|
+
|
|
352
|
+
self._success(f"Agent '{agent_name}' uploaded ({len(build_info['files'])} files)")
|
|
353
|
+
|
|
163
354
|
async def _upload_package(self, tarball: Path):
|
|
164
355
|
"""Upload package tarball to Google Cloud Storage using Storage abstraction.
|
|
165
356
|
|
|
@@ -184,13 +375,17 @@ class Deployer:
|
|
|
184
375
|
|
|
185
376
|
self._success(f"Package uploaded to {self.config['folder']}/{tarball.name}")
|
|
186
377
|
|
|
187
|
-
async def _deploy_via_api(self):
|
|
378
|
+
async def _deploy_via_api(self, agent_builds: dict[str, dict[str, Any]] | None = None):
|
|
188
379
|
"""Create or update Prefect deployment using RunnerDeployment pattern.
|
|
189
380
|
|
|
190
381
|
This is the official Prefect approach that:
|
|
191
382
|
1. Automatically creates/updates the flow registration
|
|
192
383
|
2. Handles deployment create vs update logic
|
|
193
384
|
3. Properly formats all parameters for the API
|
|
385
|
+
|
|
386
|
+
Args:
|
|
387
|
+
agent_builds: Output from _build_agents(). If non-empty, sets
|
|
388
|
+
AGENT_BUNDLES_URI env var on the deployment.
|
|
194
389
|
"""
|
|
195
390
|
# Define entrypoint (assumes flow function has same name as package)
|
|
196
391
|
entrypoint = f"{self.config['package']}:{self.config['package']}"
|
|
@@ -244,6 +439,13 @@ class Deployer:
|
|
|
244
439
|
# This is the official Prefect pattern that handles all the complexity
|
|
245
440
|
self._info(f"Creating deployment for flow '{flow.name}'")
|
|
246
441
|
|
|
442
|
+
# Set AGENT_BUNDLES_URI env var if agents were built
|
|
443
|
+
job_variables: dict[str, Any] = {}
|
|
444
|
+
if agent_builds:
|
|
445
|
+
bundles_uri = f"gs://{self.config['bucket']}/{self.config['folder']}/agents"
|
|
446
|
+
job_variables["env"] = {"AGENT_BUNDLES_URI": bundles_uri}
|
|
447
|
+
self._info(f"Setting AGENT_BUNDLES_URI={bundles_uri}")
|
|
448
|
+
|
|
247
449
|
deployment = RunnerDeployment(
|
|
248
450
|
name=self.config["package"],
|
|
249
451
|
flow_name=flow.name,
|
|
@@ -256,7 +458,7 @@ class Deployer:
|
|
|
256
458
|
or f"Deployment for {self.config['package']} v{self.config['version']}",
|
|
257
459
|
storage=_PullStepStorage(pull_steps),
|
|
258
460
|
parameters={},
|
|
259
|
-
job_variables=
|
|
461
|
+
job_variables=job_variables,
|
|
260
462
|
paused=False,
|
|
261
463
|
)
|
|
262
464
|
|
|
@@ -296,14 +498,20 @@ class Deployer:
|
|
|
296
498
|
print("=" * 70)
|
|
297
499
|
print()
|
|
298
500
|
|
|
299
|
-
# Phase 1: Build
|
|
501
|
+
# Phase 1: Build flow package
|
|
300
502
|
tarball = self._build_package()
|
|
301
503
|
|
|
302
|
-
# Phase 2:
|
|
504
|
+
# Phase 2: Build agent bundles (if configured)
|
|
505
|
+
agent_builds = self._build_agents()
|
|
506
|
+
|
|
507
|
+
# Phase 3: Upload flow package
|
|
303
508
|
await self._upload_package(tarball)
|
|
304
509
|
|
|
305
|
-
# Phase
|
|
306
|
-
await self.
|
|
510
|
+
# Phase 4: Upload agent bundles
|
|
511
|
+
await self._upload_agents(agent_builds)
|
|
512
|
+
|
|
513
|
+
# Phase 5: Create/update Prefect deployment
|
|
514
|
+
await self._deploy_via_api(agent_builds)
|
|
307
515
|
|
|
308
516
|
print()
|
|
309
517
|
print("=" * 70)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: ai-pipeline-core
|
|
3
|
-
Version: 0.3.
|
|
3
|
+
Version: 0.3.4
|
|
4
4
|
Summary: Core utilities for AI-powered processing pipelines using prefect
|
|
5
5
|
Project-URL: Homepage, https://github.com/bbarwik/ai-pipeline-core
|
|
6
6
|
Project-URL: Repository, https://github.com/bbarwik/ai-pipeline-core
|
|
@@ -22,6 +22,7 @@ Requires-Dist: httpx>=0.28.1
|
|
|
22
22
|
Requires-Dist: jinja2>=3.1.6
|
|
23
23
|
Requires-Dist: lmnr>=0.7.18
|
|
24
24
|
Requires-Dist: openai>=1.109.1
|
|
25
|
+
Requires-Dist: pillow>=10.0.0
|
|
25
26
|
Requires-Dist: prefect-gcp[cloud-storage]>=0.6.10
|
|
26
27
|
Requires-Dist: prefect>=3.4.21
|
|
27
28
|
Requires-Dist: pydantic-settings>=2.10.1
|
|
@@ -124,7 +125,7 @@ async def analyze_flow(
|
|
|
124
125
|
for doc in documents:
|
|
125
126
|
# Use AIMessages for LLM interaction
|
|
126
127
|
response = await llm.generate(
|
|
127
|
-
model="gpt-5",
|
|
128
|
+
model="gpt-5.1",
|
|
128
129
|
messages=AIMessages([doc])
|
|
129
130
|
)
|
|
130
131
|
|
|
@@ -151,7 +152,7 @@ class Analysis(BaseModel):
|
|
|
151
152
|
|
|
152
153
|
# Generate structured output
|
|
153
154
|
response = await llm.generate_structured(
|
|
154
|
-
model="gpt-5",
|
|
155
|
+
model="gpt-5.1",
|
|
155
156
|
response_format=Analysis,
|
|
156
157
|
messages="Analyze this product review: ..."
|
|
157
158
|
)
|
|
@@ -246,7 +247,7 @@ from ai_pipeline_core import llm, AIMessages, ModelOptions
|
|
|
246
247
|
|
|
247
248
|
# Simple generation
|
|
248
249
|
response = await llm.generate(
|
|
249
|
-
model="gpt-5",
|
|
250
|
+
model="gpt-5.1",
|
|
250
251
|
messages="Explain quantum computing"
|
|
251
252
|
)
|
|
252
253
|
print(response.content)
|
|
@@ -256,21 +257,21 @@ static_context = AIMessages([large_document])
|
|
|
256
257
|
|
|
257
258
|
# First call: caches context
|
|
258
259
|
r1 = await llm.generate(
|
|
259
|
-
model="gpt-5",
|
|
260
|
+
model="gpt-5.1",
|
|
260
261
|
context=static_context, # Cached for 120 seconds by default
|
|
261
262
|
messages="Summarize" # Dynamic query
|
|
262
263
|
)
|
|
263
264
|
|
|
264
265
|
# Second call: reuses cache
|
|
265
266
|
r2 = await llm.generate(
|
|
266
|
-
model="gpt-5",
|
|
267
|
+
model="gpt-5.1",
|
|
267
268
|
context=static_context, # Reused from cache!
|
|
268
269
|
messages="Key points?" # Different query
|
|
269
270
|
)
|
|
270
271
|
|
|
271
272
|
# Custom cache TTL
|
|
272
273
|
response = await llm.generate(
|
|
273
|
-
model="gpt-5",
|
|
274
|
+
model="gpt-5.1",
|
|
274
275
|
context=static_context,
|
|
275
276
|
messages="Analyze",
|
|
276
277
|
options=ModelOptions(cache_ttl="300s") # Cache for 5 minutes
|
|
@@ -278,7 +279,7 @@ response = await llm.generate(
|
|
|
278
279
|
|
|
279
280
|
# Disable caching for dynamic contexts
|
|
280
281
|
response = await llm.generate(
|
|
281
|
-
model="gpt-5",
|
|
282
|
+
model="gpt-5.1",
|
|
282
283
|
context=dynamic_context,
|
|
283
284
|
messages="Process",
|
|
284
285
|
options=ModelOptions(cache_ttl=None) # No caching
|
|
@@ -335,6 +336,68 @@ async def main_flow(
|
|
|
335
336
|
return DocumentList(results)
|
|
336
337
|
```
|
|
337
338
|
|
|
339
|
+
### Local Trace Debugging
|
|
340
|
+
|
|
341
|
+
Save all trace spans to the local filesystem for LLM-assisted debugging:
|
|
342
|
+
|
|
343
|
+
```bash
|
|
344
|
+
export TRACE_DEBUG_PATH=/path/to/debug/output
|
|
345
|
+
```
|
|
346
|
+
|
|
347
|
+
This creates a hierarchical directory structure that mirrors the execution flow with automatic deduplication:
|
|
348
|
+
|
|
349
|
+
```
|
|
350
|
+
20260128_152932_abc12345_my_flow/
|
|
351
|
+
├── _trace.yaml # Trace metadata
|
|
352
|
+
├── _index.yaml # Span ID → path mapping
|
|
353
|
+
├── _summary.md # Unified summary for human inspection and LLM debugging
|
|
354
|
+
├── artifacts/ # Deduplicated content storage
|
|
355
|
+
│ └── sha256/
|
|
356
|
+
│ └── ab/cd/ # Sharded by hash prefix
|
|
357
|
+
│ └── abcdef...1234.txt # Large content (>10KB)
|
|
358
|
+
└── 0001_my_flow/ # Root span (numbered for execution order)
|
|
359
|
+
├── _span.yaml # Span metadata (timing, status, I/O refs)
|
|
360
|
+
├── input.yaml # Structured inputs (inline or refs)
|
|
361
|
+
├── output.yaml # Structured outputs (inline or refs)
|
|
362
|
+
├── 0002_task_1/ # Child spans nested inside parent
|
|
363
|
+
│ ├── _span.yaml
|
|
364
|
+
│ ├── input.yaml
|
|
365
|
+
│ ├── output.yaml
|
|
366
|
+
│ └── 0003_llm_call/
|
|
367
|
+
│ ├── _span.yaml
|
|
368
|
+
│ ├── input.yaml # LLM messages with inline/external content
|
|
369
|
+
│ └── output.yaml
|
|
370
|
+
└── 0004_task_2/
|
|
371
|
+
└── ...
|
|
372
|
+
```
|
|
373
|
+
|
|
374
|
+
**Key Features:**
|
|
375
|
+
- **Automatic Deduplication**: Identical content (e.g., system prompts) stored once in `artifacts/`
|
|
376
|
+
- **Smart Externalization**: Large content (>10KB) externalized with 2KB inline previews
|
|
377
|
+
- **AI-Friendly**: Files capped at 50KB for easy LLM processing
|
|
378
|
+
- **Lossless**: Full content reconstruction via `content_ref` pointers
|
|
379
|
+
|
|
380
|
+
Example `input.yaml` with externalization:
|
|
381
|
+
```yaml
|
|
382
|
+
format_version: 3
|
|
383
|
+
type: llm_messages
|
|
384
|
+
messages:
|
|
385
|
+
- role: system
|
|
386
|
+
parts:
|
|
387
|
+
- type: text
|
|
388
|
+
size_bytes: 28500
|
|
389
|
+
content_ref: # Large content → artifact
|
|
390
|
+
hash: sha256:a1b2c3d4...
|
|
391
|
+
path: artifacts/sha256/a1/b2/a1b2c3d4...txt
|
|
392
|
+
excerpt: "You are a helpful assistant...\n[TRUNCATED]"
|
|
393
|
+
- role: user
|
|
394
|
+
parts:
|
|
395
|
+
- type: text
|
|
396
|
+
content: "Hello!" # Small content stays inline
|
|
397
|
+
```
|
|
398
|
+
|
|
399
|
+
Run `tree` on the output directory to visualize the entire execution hierarchy. Feed `_summary.md` to an LLM for debugging assistance - it combines high-level overview with detailed navigation for comprehensive trace analysis.
|
|
400
|
+
|
|
338
401
|
## Configuration
|
|
339
402
|
|
|
340
403
|
### Environment Variables
|
|
@@ -348,6 +411,9 @@ OPENAI_API_KEY=your-api-key
|
|
|
348
411
|
LMNR_PROJECT_API_KEY=your-lmnr-key
|
|
349
412
|
LMNR_DEBUG=true # Enable debug traces
|
|
350
413
|
|
|
414
|
+
# Optional: Local Trace Debugging
|
|
415
|
+
TRACE_DEBUG_PATH=/path/to/trace/output # Save traces locally for LLM-assisted debugging
|
|
416
|
+
|
|
351
417
|
# Optional: Orchestration
|
|
352
418
|
PREFECT_API_URL=http://localhost:4200/api
|
|
353
419
|
PREFECT_API_KEY=your-prefect-key
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
ai_pipeline_core/__init__.py,sha256=
|
|
1
|
+
ai_pipeline_core/__init__.py,sha256=ZqmgQAauyPZDtlMaWjD7G8DKUs0Ks43w1zT1RdASY5w,7225
|
|
2
2
|
ai_pipeline_core/exceptions.py,sha256=vx-XLTw2fJSPs-vwtXVYtqoQUcOc0JeI7UmHqRqQYWU,1569
|
|
3
3
|
ai_pipeline_core/pipeline.py,sha256=t9qH-V6umpKY5MhGuXFgUGfdzGyxzVlS0n9RoKLfnug,28704
|
|
4
4
|
ai_pipeline_core/prefect.py,sha256=91ZgLJHsDsRUW77CpNmkKxYs3RCJuucPM3pjKmNBeDg,2199
|
|
@@ -6,7 +6,13 @@ ai_pipeline_core/progress.py,sha256=Ppxk4OOm84Y0x3t-Y3CmHsL4PovQLNUxXMu24zRCD-Q,
|
|
|
6
6
|
ai_pipeline_core/prompt_manager.py,sha256=FAtb1yK7bGuAeuIJ523LOX9bd7TrcHG-TqZ7Lz4RJC0,12087
|
|
7
7
|
ai_pipeline_core/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
8
8
|
ai_pipeline_core/settings.py,sha256=IMrFaX0i-WIlaOA5O53ipNSta6KQVSFHc1aJXmS3nSo,5078
|
|
9
|
-
ai_pipeline_core/tracing.py,sha256=
|
|
9
|
+
ai_pipeline_core/tracing.py,sha256=HJ_DJhCEk6W_u3skecjETMQVLyOmbuPcqcOuoMIJlPs,33194
|
|
10
|
+
ai_pipeline_core/debug/__init__.py,sha256=wOc9KotFqGYzBEtZUZ7ATfJf3dXWarYm6PXs6yW9uwE,756
|
|
11
|
+
ai_pipeline_core/debug/config.py,sha256=l5WC2xbd6PgC-CcuioZg696iva_MkqyZj4C9TFdwfMs,3205
|
|
12
|
+
ai_pipeline_core/debug/content.py,sha256=tO0Kvik0NpsJnBU-zVzOur7wLSnHE89mUAnaC9mo6pA,26731
|
|
13
|
+
ai_pipeline_core/debug/processor.py,sha256=Cvm1HKc6lKRm80Xx7WXi_Z8pWoKH6actVZvntP9Mons,3935
|
|
14
|
+
ai_pipeline_core/debug/summary.py,sha256=pzXC7QoFOBeen_XZ-AMFAVvaOtDuf28YB-WwCbsHYdQ,8017
|
|
15
|
+
ai_pipeline_core/debug/writer.py,sha256=IF5eyML10EBFBqCGqlVwcWKraFvTgfqbU8WJ_XG_RU4,33108
|
|
10
16
|
ai_pipeline_core/deployment/__init__.py,sha256=FN2HVoM80x2GJuNs7o4DnccB8HWWibgM1pJesB942CM,1259
|
|
11
17
|
ai_pipeline_core/deployment/base.py,sha256=JYf8XLFR73c0H24dr6atK7yUcoE0vLxbYZ8EkQpEwN4,24791
|
|
12
18
|
ai_pipeline_core/deployment/contract.py,sha256=0DKt5eqNE-grcITwMNq9CuBdo5WxdopEjDeQFzFZxhU,2225
|
|
@@ -15,35 +21,37 @@ ai_pipeline_core/documents/__init__.py,sha256=WHStvGZiSyybOcMTYxSV24U6MA3Am_0_Az
|
|
|
15
21
|
ai_pipeline_core/documents/document.py,sha256=hdTh36KGEcrDollTnQmTI66DJIqYfe4X42Y0q7Cm4fY,68153
|
|
16
22
|
ai_pipeline_core/documents/document_list.py,sha256=Y_NCjfM_CjkIwHRD2iyGgYBuIykN8lT2IIH_uWOiGis,16254
|
|
17
23
|
ai_pipeline_core/documents/flow_document.py,sha256=QK6RxNQu449IRAosOHSk3G_5yIq5I7yLBOSQPCd3m64,4141
|
|
18
|
-
ai_pipeline_core/documents/mime_type.py,sha256=
|
|
24
|
+
ai_pipeline_core/documents/mime_type.py,sha256=UxWtKAlcph0cFzGG5ZB86MCwjwIO30vnsMT4j_2B4FA,8617
|
|
19
25
|
ai_pipeline_core/documents/task_document.py,sha256=uASmAaxNkYtuqQrBM57vutFT9DXNTbqv0wbwwF55E3I,4300
|
|
20
26
|
ai_pipeline_core/documents/temporary_document.py,sha256=jaz2ZHC5CmSbVbkXdI7pOB5DGEuhH16C0Yutv-lS_UI,2708
|
|
21
27
|
ai_pipeline_core/documents/utils.py,sha256=ZyJNjFN7ihWno0K7dJZed7twYmmPLA0z40UzFw1A3A8,5465
|
|
22
28
|
ai_pipeline_core/flow/__init__.py,sha256=2BfWYMOPYW5teGzwo-qzpn_bom1lxxry0bPsjVgcsCk,188
|
|
23
29
|
ai_pipeline_core/flow/config.py,sha256=a9FALpgrFsdz-D7HU3diVeUzbaBvLwI8hsPviuj001s,19389
|
|
24
|
-
ai_pipeline_core/flow/options.py,sha256=
|
|
30
|
+
ai_pipeline_core/flow/options.py,sha256=s5GBTy5lwFa1irf8BKrWO8NMZ5s_f4tqq7Wg9WQ7TTg,2302
|
|
31
|
+
ai_pipeline_core/images/__init__.py,sha256=6R6Ncif6oRyVOH7LsdwNvEuMGHuljo-_gImY8C3Z_ow,9877
|
|
32
|
+
ai_pipeline_core/images/_processing.py,sha256=wKSBAFe5TO-mo64ll20nmN9huazHwvVWFfNJB6g7u2Q,4421
|
|
25
33
|
ai_pipeline_core/llm/__init__.py,sha256=3B_vtEzxrzidP1qOUNQ4RxlUmxZ2MBKQcUhQiTybM9g,661
|
|
26
|
-
ai_pipeline_core/llm/ai_messages.py,sha256=
|
|
27
|
-
ai_pipeline_core/llm/client.py,sha256=
|
|
34
|
+
ai_pipeline_core/llm/ai_messages.py,sha256=YteChzp_aEcUASGKKyz2zSR0rTM0rO5ju9ZrvP4nWoM,16210
|
|
35
|
+
ai_pipeline_core/llm/client.py,sha256=NhSHn4fskirDWOT-4qg3E0-GdGncQPnFW8Ayge4jvJ8,27157
|
|
28
36
|
ai_pipeline_core/llm/model_options.py,sha256=uRNIHfVeh2sgt1mZBiOUx6hPQ6GKjB8b7TytZJ6afKg,11768
|
|
29
|
-
ai_pipeline_core/llm/model_response.py,sha256
|
|
30
|
-
ai_pipeline_core/llm/model_types.py,sha256=
|
|
37
|
+
ai_pipeline_core/llm/model_response.py,sha256=zEANsfuSAYVRKPwKx9gFIqHbdVG_1_JNMRHNoE43_YM,13503
|
|
38
|
+
ai_pipeline_core/llm/model_types.py,sha256=wx-m0up7_NncTmSYmMsL-l-RgydjjJ905u7RMEAg7tI,2710
|
|
31
39
|
ai_pipeline_core/logging/__init__.py,sha256=Nz6-ghAoENsgNmLD2ma9TW9M0U2_QfxuQ5DDW6Vt6M0,651
|
|
32
40
|
ai_pipeline_core/logging/logging.yml,sha256=YTW48keO_K5bkkb-KXGM7ZuaYKiquLsjsURei8Ql0V4,1353
|
|
33
41
|
ai_pipeline_core/logging/logging_config.py,sha256=pV2x6GgMPXrzPH27sicCSXfw56beio4C2JKCJ3NsXrg,6207
|
|
34
|
-
ai_pipeline_core/logging/logging_mixin.py,sha256=
|
|
42
|
+
ai_pipeline_core/logging/logging_mixin.py,sha256=OpdR3ASiM2ZwKZYGjZRJFUloGWUCv2Grnr8RqUWlYn8,8094
|
|
35
43
|
ai_pipeline_core/prompt_builder/__init__.py,sha256=-v0SKZlir07xRzxXwv75VP66aINRUiKH0VUgB-PCDmI,195
|
|
36
44
|
ai_pipeline_core/prompt_builder/documents_prompt.jinja2,sha256=LPql5AaFhFWtDfhnBWvi-bWbz5vdgsWqKGzcqxWfLIM,1075
|
|
37
45
|
ai_pipeline_core/prompt_builder/global_cache.py,sha256=9_9zoF6-sr3KBMxF5QLD3vxqXg9B2tT8o9ViplzUCNg,2811
|
|
38
46
|
ai_pipeline_core/prompt_builder/new_core_documents_prompt.jinja2,sha256=M8uPpwf-uLpsWWJT9DY_DnjrLToGPVnrD-gVhQrQdaQ,229
|
|
39
|
-
ai_pipeline_core/prompt_builder/prompt_builder.py,sha256=
|
|
47
|
+
ai_pipeline_core/prompt_builder/prompt_builder.py,sha256=4TrDRPiOMFwEfi6QGfriTHfjzj_CtbEjAcgQrVfRqhw,9378
|
|
40
48
|
ai_pipeline_core/prompt_builder/system_prompt.jinja2,sha256=-1jLcfvAG07Zfl-dnYrjfVcAG4PWeeoeWpaKJGY3rKQ,3945
|
|
41
49
|
ai_pipeline_core/storage/__init__.py,sha256=tcIkjJ3zPBLCyetwiJDewBvS2sbRJrDlBh3gEsQm08E,184
|
|
42
50
|
ai_pipeline_core/storage/storage.py,sha256=ClMr419Y-eU2RuOjZYd51dC0stWQk28Vb56PvQaoUwc,20007
|
|
43
51
|
ai_pipeline_core/utils/__init__.py,sha256=TJSmEm1Quf-gKwXrxM96u2IGzVolUyeNNfLMPoLstXI,254
|
|
44
|
-
ai_pipeline_core/utils/deploy.py,sha256=
|
|
52
|
+
ai_pipeline_core/utils/deploy.py,sha256=N3i7B97DQJs1lwgYN3sa1UgwCNjseKXfjs50ZJUMCEI,22106
|
|
45
53
|
ai_pipeline_core/utils/remote_deployment.py,sha256=U7MNJ1SU1mg3RrJyLqpuN_4pwqm8LSsFZbypJvjGPoo,4630
|
|
46
|
-
ai_pipeline_core-0.3.
|
|
47
|
-
ai_pipeline_core-0.3.
|
|
48
|
-
ai_pipeline_core-0.3.
|
|
49
|
-
ai_pipeline_core-0.3.
|
|
54
|
+
ai_pipeline_core-0.3.4.dist-info/METADATA,sha256=YtzaamjuVgNOP6TlR49OC3nu6UPMaLb47hIWDQVgEGM,17893
|
|
55
|
+
ai_pipeline_core-0.3.4.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
|
|
56
|
+
ai_pipeline_core-0.3.4.dist-info/licenses/LICENSE,sha256=kKj8mfbdWwkyG3U6n7ztB3bAZlEwShTkAsvaY657i3I,1074
|
|
57
|
+
ai_pipeline_core-0.3.4.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|