fluxloop-cli 0.2.36__tar.gz → 0.3.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/PKG-INFO +1 -1
- {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/__init__.py +1 -1
- {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/commands/__init__.py +2 -1
- {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/commands/config.py +37 -0
- {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/commands/status.py +8 -7
- fluxloop_cli-0.3.1/fluxloop_cli/commands/sync.py +488 -0
- {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/evaluation/engine/reporting/html.py +1 -1
- {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/evaluation/report/renderer.py +1 -1
- {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/main.py +13 -1
- {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli.egg-info/PKG-INFO +1 -1
- {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli.egg-info/SOURCES.txt +1 -0
- {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/pyproject.toml +1 -1
- {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/README.md +0 -0
- {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/arg_binder.py +0 -0
- {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/commands/doctor.py +0 -0
- {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/commands/evaluate.py +0 -0
- {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/commands/generate.py +0 -0
- {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/commands/init.py +0 -0
- {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/commands/parse.py +0 -0
- {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/commands/record.py +0 -0
- {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/commands/run.py +0 -0
- {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/config_loader.py +0 -0
- {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/config_schema.py +0 -0
- {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/constants.py +0 -0
- {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/conversation_supervisor.py +0 -0
- {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/environment.py +0 -0
- {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/evaluation/__init__.py +0 -0
- {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/evaluation/artifacts.py +0 -0
- {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/evaluation/config.py +0 -0
- {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/evaluation/engine/__init__.py +0 -0
- {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/evaluation/engine/analysis.py +0 -0
- {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/evaluation/engine/core.py +0 -0
- {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/evaluation/engine/reporting/__init__.py +0 -0
- {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/evaluation/engine/reporting/markdown.py +0 -0
- {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/evaluation/engine/success.py +0 -0
- {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/evaluation/llm.py +0 -0
- {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/evaluation/prompts/__init__.py +0 -0
- {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/evaluation/prompts/base.py +0 -0
- {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/evaluation/prompts/information_completeness.py +0 -0
- {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/evaluation/prompts/intent_recognition.py +0 -0
- {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/evaluation/prompts/response_clarity.py +0 -0
- {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/evaluation/prompts/response_consistency.py +0 -0
- {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/evaluation/report/__init__.py +0 -0
- {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/evaluation/report/aggregator.py +0 -0
- {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/evaluation/report/generator.py +0 -0
- {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/evaluation/report/pipeline.py +0 -0
- {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/evaluation/rules.py +0 -0
- {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/evaluation/templates/report.html.j2 +0 -0
- {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/input_generator.py +0 -0
- {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/llm_generator.py +0 -0
- {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/project_paths.py +0 -0
- {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/runner.py +0 -0
- {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/target_loader.py +0 -0
- {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/templates.py +0 -0
- {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/testing/__init__.py +0 -0
- {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/testing/pytest_plugin.py +0 -0
- {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/testing/types.py +0 -0
- {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/token_usage.py +0 -0
- {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/validators.py +0 -0
- {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli.egg-info/dependency_links.txt +0 -0
- {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli.egg-info/entry_points.txt +0 -0
- {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli.egg-info/requires.txt +0 -0
- {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli.egg-info/top_level.txt +0 -0
- {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/setup.cfg +0 -0
- {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/tests/test_analysis_recommendations.py +0 -0
- {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/tests/test_arg_binder.py +0 -0
- {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/tests/test_config_command.py +0 -0
- {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/tests/test_conversation_supervisor.py +0 -0
- {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/tests/test_evaluate_command.py +0 -0
- {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/tests/test_evaluation_llm.py +0 -0
- {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/tests/test_init_pytest_template.py +0 -0
- {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/tests/test_input_generator.py +0 -0
- {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/tests/test_prompt_library.py +0 -0
- {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/tests/test_pytest_plugin.py +0 -0
- {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/tests/test_run_command.py +0 -0
- {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/tests/test_runner_multi_turn.py +0 -0
- {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/tests/test_success_criteria.py +0 -0
- {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/tests/test_target_loader.py +0 -0
- {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/tests/test_testing_types.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"""CLI commands."""
|
|
2
2
|
|
|
3
|
-
from . import config, doctor, evaluate, generate, init, parse, record, run, status
|
|
3
|
+
from . import config, doctor, evaluate, generate, init, parse, record, run, status, sync
|
|
4
4
|
|
|
5
5
|
__all__ = [
|
|
6
6
|
"config",
|
|
@@ -12,4 +12,5 @@ __all__ = [
|
|
|
12
12
|
"record",
|
|
13
13
|
"run",
|
|
14
14
|
"status",
|
|
15
|
+
"sync",
|
|
15
16
|
]
|
|
@@ -192,7 +192,9 @@ def env(
|
|
|
192
192
|
|
|
193
193
|
env_vars = [
|
|
194
194
|
("FLUXLOOP_COLLECTOR_URL", "Collector service URL", "http://localhost:8000"),
|
|
195
|
+
("FLUXLOOP_SYNC_URL", "Sync API base URL", None),
|
|
195
196
|
("FLUXLOOP_API_KEY", "API key for authentication", None),
|
|
197
|
+
("FLUXLOOP_SYNC_API_KEY", "Sync API key for uploads", None),
|
|
196
198
|
("FLUXLOOP_ENABLED", "Enable/disable tracing", "true"),
|
|
197
199
|
("FLUXLOOP_DEBUG", "Enable debug mode", "false"),
|
|
198
200
|
("FLUXLOOP_SAMPLE_RATE", "Trace sampling rate (0-1)", "1.0"),
|
|
@@ -323,6 +325,41 @@ def set_llm(
|
|
|
323
325
|
)
|
|
324
326
|
|
|
325
327
|
|
|
328
|
+
@app.command()
|
|
329
|
+
def set_sync_key(
|
|
330
|
+
api_key: str = typer.Argument(..., help="Sync API key"),
|
|
331
|
+
api_url: Optional[str] = typer.Option(None, "--url", help="Sync API base URL"),
|
|
332
|
+
overwrite_env: bool = typer.Option(False, "--overwrite-env", help="Overwrite existing key in .env"),
|
|
333
|
+
env_file: Path = typer.Option(Path(".env"), "--env-file", help="Path to environment file"),
|
|
334
|
+
project: Optional[str] = typer.Option(None, "--project", help="Project name under the FluxLoop root"),
|
|
335
|
+
root: Path = typer.Option(Path(DEFAULT_ROOT_DIR_NAME), "--root", help="FluxLoop root directory"),
|
|
336
|
+
):
|
|
337
|
+
"""Save Sync API credentials to .env."""
|
|
338
|
+
|
|
339
|
+
env_path = resolve_env_path(env_file, project, root)
|
|
340
|
+
env_contents: Dict[str, str] = {}
|
|
341
|
+
if env_path.exists():
|
|
342
|
+
for line in env_path.read_text().splitlines():
|
|
343
|
+
if not line or line.startswith("#") or "=" not in line:
|
|
344
|
+
continue
|
|
345
|
+
key, value = line.split("=", 1)
|
|
346
|
+
env_contents[key.strip()] = value.strip()
|
|
347
|
+
|
|
348
|
+
if "FLUXLOOP_SYNC_API_KEY" in env_contents and not overwrite_env:
|
|
349
|
+
console.print(
|
|
350
|
+
"[yellow]Warning:[/yellow] FLUXLOOP_SYNC_API_KEY already set. Use --overwrite-env to replace it."
|
|
351
|
+
)
|
|
352
|
+
else:
|
|
353
|
+
env_contents["FLUXLOOP_SYNC_API_KEY"] = api_key
|
|
354
|
+
|
|
355
|
+
if api_url:
|
|
356
|
+
env_contents["FLUXLOOP_SYNC_URL"] = api_url
|
|
357
|
+
|
|
358
|
+
env_lines = [f"{key}={value}" for key, value in env_contents.items()]
|
|
359
|
+
env_path.write_text("\n".join(env_lines) + "\n")
|
|
360
|
+
console.print(f"[green]✓[/green] Saved sync credentials to {env_path}")
|
|
361
|
+
|
|
362
|
+
|
|
326
363
|
@app.command()
|
|
327
364
|
def validate(
|
|
328
365
|
config_file: Path = typer.Option(
|
|
@@ -133,17 +133,18 @@ def check(
|
|
|
133
133
|
|
|
134
134
|
# Check environment
|
|
135
135
|
import os
|
|
136
|
-
|
|
136
|
+
sync_key = os.getenv("FLUXLOOP_SYNC_API_KEY") or os.getenv("FLUXLOOP_API_KEY")
|
|
137
|
+
if sync_key:
|
|
137
138
|
status_table.add_row(
|
|
138
|
-
"API Key",
|
|
139
|
+
"Sync API Key",
|
|
139
140
|
"[green]✓ Set[/green]",
|
|
140
|
-
"****" +
|
|
141
|
+
"****" + sync_key[-4:] if verbose else "Configured",
|
|
141
142
|
)
|
|
142
143
|
else:
|
|
143
144
|
status_table.add_row(
|
|
144
|
-
"API Key",
|
|
145
|
+
"Sync API Key",
|
|
145
146
|
"[yellow]- Not set[/yellow]",
|
|
146
|
-
"Set
|
|
147
|
+
"Set FLUXLOOP_SYNC_API_KEY in .env",
|
|
147
148
|
)
|
|
148
149
|
|
|
149
150
|
console.print(status_table)
|
|
@@ -152,8 +153,8 @@ def check(
|
|
|
152
153
|
recommendations = []
|
|
153
154
|
if not resolved_config.exists():
|
|
154
155
|
recommendations.append("Initialize a project: [cyan]fluxloop init project[/cyan]")
|
|
155
|
-
if not
|
|
156
|
-
recommendations.append("Set up API key in .env file")
|
|
156
|
+
if not sync_key:
|
|
157
|
+
recommendations.append("Set up sync API key in .env file")
|
|
157
158
|
|
|
158
159
|
if recommendations:
|
|
159
160
|
console.print("\n[bold]Recommendations:[/bold]")
|
|
@@ -0,0 +1,488 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Sync command for pulling bundles and uploading results.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
|
|
7
|
+
import json
|
|
8
|
+
import os
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
from typing import Any, Dict, Iterable, List, Optional, Tuple
|
|
11
|
+
from uuid import UUID, uuid4
|
|
12
|
+
|
|
13
|
+
import httpx
|
|
14
|
+
import typer
|
|
15
|
+
import yaml
|
|
16
|
+
from rich.console import Console
|
|
17
|
+
|
|
18
|
+
from ..config_loader import load_experiment_config
|
|
19
|
+
from ..constants import DEFAULT_CONFIG_PATH, DEFAULT_ROOT_DIR_NAME
|
|
20
|
+
from ..environment import load_env_chain
|
|
21
|
+
from ..project_paths import (
|
|
22
|
+
resolve_config_path,
|
|
23
|
+
resolve_project_dir,
|
|
24
|
+
resolve_project_relative,
|
|
25
|
+
)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
app = typer.Typer(help="Sync bundle inputs and upload run results.")
|
|
29
|
+
console = Console()
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _resolve_source_dir(project: Optional[str], root: Path) -> Path:
|
|
33
|
+
if project:
|
|
34
|
+
return resolve_project_dir(project, root)
|
|
35
|
+
return Path.cwd().resolve()
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _load_env(project: Optional[str], root: Path) -> None:
|
|
39
|
+
source_dir = _resolve_source_dir(project, root)
|
|
40
|
+
load_env_chain(source_dir, refresh_config=False)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _resolve_api_url(override: Optional[str]) -> str:
|
|
44
|
+
url = (
|
|
45
|
+
override
|
|
46
|
+
or os.getenv("FLUXLOOP_SYNC_URL")
|
|
47
|
+
or os.getenv("FLUXLOOP_COLLECTOR_URL")
|
|
48
|
+
or "http://localhost:8000"
|
|
49
|
+
)
|
|
50
|
+
return url.rstrip("/")
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _resolve_api_key(override: Optional[str]) -> str:
|
|
54
|
+
key = (
|
|
55
|
+
override
|
|
56
|
+
or os.getenv("FLUXLOOP_SYNC_API_KEY")
|
|
57
|
+
or os.getenv("FLUXLOOP_API_KEY")
|
|
58
|
+
)
|
|
59
|
+
if not key:
|
|
60
|
+
raise typer.BadParameter("Sync API key is not set. Use fluxloop config set-sync-key.")
|
|
61
|
+
return key
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _ensure_sync_dir(project_root: Path) -> Path:
|
|
65
|
+
sync_dir = project_root / ".fluxloop" / "sync"
|
|
66
|
+
sync_dir.mkdir(parents=True, exist_ok=True)
|
|
67
|
+
return sync_dir
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def _write_json(path: Path, payload: dict) -> None:
|
|
71
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
72
|
+
path.write_text(json.dumps(payload, indent=2, ensure_ascii=False))
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def _extract_input_text(messages: Any) -> str:
|
|
76
|
+
if isinstance(messages, list):
|
|
77
|
+
for entry in messages:
|
|
78
|
+
if isinstance(entry, dict):
|
|
79
|
+
role = entry.get("role") or entry.get("type")
|
|
80
|
+
if role in ("user", "human"):
|
|
81
|
+
content = entry.get("content")
|
|
82
|
+
if isinstance(content, str):
|
|
83
|
+
return content
|
|
84
|
+
for entry in messages:
|
|
85
|
+
if isinstance(entry, dict):
|
|
86
|
+
content = entry.get("content")
|
|
87
|
+
if isinstance(content, str):
|
|
88
|
+
return content
|
|
89
|
+
if isinstance(messages, dict):
|
|
90
|
+
content = messages.get("content")
|
|
91
|
+
if isinstance(content, str):
|
|
92
|
+
return content
|
|
93
|
+
if isinstance(messages, str):
|
|
94
|
+
return messages
|
|
95
|
+
return ""
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def _resolve_inputs_path(
|
|
99
|
+
project: Optional[str], root: Path, config_file: Path
|
|
100
|
+
) -> Tuple[Path, Optional[str]]:
|
|
101
|
+
resolved = resolve_config_path(config_file, project, root)
|
|
102
|
+
if resolved.exists():
|
|
103
|
+
try:
|
|
104
|
+
config = load_experiment_config(
|
|
105
|
+
resolved, project=project, root=root, require_inputs_file=False
|
|
106
|
+
)
|
|
107
|
+
inputs_file = config.inputs_file or "inputs/generated.yaml"
|
|
108
|
+
return (
|
|
109
|
+
resolve_project_relative(Path(inputs_file), project, root),
|
|
110
|
+
inputs_file,
|
|
111
|
+
)
|
|
112
|
+
except Exception:
|
|
113
|
+
pass
|
|
114
|
+
inputs_file = "inputs/generated.yaml"
|
|
115
|
+
return resolve_project_relative(Path(inputs_file), project, root), inputs_file
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def _load_inputs_mapping(inputs_path: Path) -> Dict[Tuple[str, Optional[str]], List[str]]:
|
|
119
|
+
if not inputs_path.exists():
|
|
120
|
+
return {}
|
|
121
|
+
payload = yaml.safe_load(inputs_path.read_text()) or {}
|
|
122
|
+
entries = payload.get("inputs") if isinstance(payload, dict) else payload
|
|
123
|
+
if not isinstance(entries, list):
|
|
124
|
+
return {}
|
|
125
|
+
|
|
126
|
+
mapping: Dict[Tuple[str, Optional[str]], List[str]] = {}
|
|
127
|
+
for entry in entries:
|
|
128
|
+
if not isinstance(entry, dict):
|
|
129
|
+
continue
|
|
130
|
+
input_text = entry.get("input")
|
|
131
|
+
metadata = entry.get("metadata") or {}
|
|
132
|
+
item_id = metadata.get("input_item_id")
|
|
133
|
+
if not input_text or not item_id:
|
|
134
|
+
continue
|
|
135
|
+
persona = entry.get("persona") or metadata.get("persona")
|
|
136
|
+
mapping.setdefault((input_text, persona), []).append(item_id)
|
|
137
|
+
return mapping
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def _resolve_input_item_id(
|
|
141
|
+
trace: dict, mapping: Dict[Tuple[str, Optional[str]], List[str]]
|
|
142
|
+
) -> Optional[str]:
|
|
143
|
+
conversation_state = trace.get("conversation_state") or {}
|
|
144
|
+
metadata = conversation_state.get("metadata") if isinstance(conversation_state, dict) else {}
|
|
145
|
+
variation = metadata.get("variation") if isinstance(metadata, dict) else None
|
|
146
|
+
if isinstance(variation, dict):
|
|
147
|
+
direct = variation.get("input_item_id")
|
|
148
|
+
if direct:
|
|
149
|
+
return direct
|
|
150
|
+
|
|
151
|
+
input_text = trace.get("input")
|
|
152
|
+
persona = trace.get("persona")
|
|
153
|
+
if input_text:
|
|
154
|
+
key = (input_text, persona)
|
|
155
|
+
if key in mapping and mapping[key]:
|
|
156
|
+
return mapping[key].pop(0)
|
|
157
|
+
key = (input_text, None)
|
|
158
|
+
if key in mapping and mapping[key]:
|
|
159
|
+
return mapping[key].pop(0)
|
|
160
|
+
return None
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def _coerce_run_id(trace_id: Optional[str]) -> str:
|
|
164
|
+
if trace_id:
|
|
165
|
+
try:
|
|
166
|
+
return str(UUID(trace_id))
|
|
167
|
+
except Exception:
|
|
168
|
+
pass
|
|
169
|
+
return str(uuid4())
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def _iter_jsonl(path: Path) -> Iterable[dict]:
|
|
173
|
+
with path.open("r", encoding="utf-8") as handle:
|
|
174
|
+
for line in handle:
|
|
175
|
+
line = line.strip()
|
|
176
|
+
if not line:
|
|
177
|
+
continue
|
|
178
|
+
try:
|
|
179
|
+
yield json.loads(line)
|
|
180
|
+
except json.JSONDecodeError:
|
|
181
|
+
continue
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def _guess_content_type(path: Path) -> str:
|
|
185
|
+
if path.suffix in (".json", ".jsonl"):
|
|
186
|
+
return "application/json"
|
|
187
|
+
if path.suffix == ".md":
|
|
188
|
+
return "text/markdown"
|
|
189
|
+
if path.suffix == ".html":
|
|
190
|
+
return "text/html"
|
|
191
|
+
if path.suffix == ".pdf":
|
|
192
|
+
return "application/pdf"
|
|
193
|
+
return "application/octet-stream"
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
@app.command()
|
|
197
|
+
def pull(
|
|
198
|
+
project_id: Optional[str] = typer.Option(None, "--project-id", help="Project id"),
|
|
199
|
+
bundle_version_id: Optional[str] = typer.Option(
|
|
200
|
+
None, "--bundle-version-id", help="Bundle version id"
|
|
201
|
+
),
|
|
202
|
+
config_file: Path = typer.Option(
|
|
203
|
+
DEFAULT_CONFIG_PATH, "--config", "-c", help="Config file to resolve inputs path"
|
|
204
|
+
),
|
|
205
|
+
project: Optional[str] = typer.Option(None, "--project", help="Project name"),
|
|
206
|
+
root: Path = typer.Option(Path(DEFAULT_ROOT_DIR_NAME), "--root", help="Root dir"),
|
|
207
|
+
api_url: Optional[str] = typer.Option(None, "--api-url", help="Sync API base URL"),
|
|
208
|
+
api_key: Optional[str] = typer.Option(None, "--api-key", help="Sync API key"),
|
|
209
|
+
):
|
|
210
|
+
"""
|
|
211
|
+
Pull bundle inputs/personas/criteria for local execution.
|
|
212
|
+
"""
|
|
213
|
+
_load_env(project, root)
|
|
214
|
+
api_url = _resolve_api_url(api_url)
|
|
215
|
+
api_key = _resolve_api_key(api_key)
|
|
216
|
+
|
|
217
|
+
payload: Dict[str, Any] = {
|
|
218
|
+
"bundle_version_id": bundle_version_id,
|
|
219
|
+
"project_id": project_id,
|
|
220
|
+
"include_inputs": True,
|
|
221
|
+
"include_personas": True,
|
|
222
|
+
"include_criteria": True,
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
with httpx.Client(base_url=api_url, timeout=30.0) as client:
|
|
226
|
+
resp = client.post(
|
|
227
|
+
"/api/sync/pull",
|
|
228
|
+
json=payload,
|
|
229
|
+
headers={"Authorization": f"Bearer {api_key}"},
|
|
230
|
+
)
|
|
231
|
+
resp.raise_for_status()
|
|
232
|
+
data = resp.json()
|
|
233
|
+
|
|
234
|
+
project_root = _resolve_source_dir(project, root)
|
|
235
|
+
sync_dir = _ensure_sync_dir(project_root)
|
|
236
|
+
|
|
237
|
+
personas = data.get("personas") or []
|
|
238
|
+
persona_map = {item.get("id"): item.get("name") for item in personas if item.get("id")}
|
|
239
|
+
|
|
240
|
+
inputs_path, inputs_file = _resolve_inputs_path(project, root, config_file)
|
|
241
|
+
inputs_payload = {"inputs": []}
|
|
242
|
+
for item in data.get("input_items") or []:
|
|
243
|
+
input_text = _extract_input_text(item.get("messages"))
|
|
244
|
+
if not input_text:
|
|
245
|
+
continue
|
|
246
|
+
metadata = dict(item.get("metadata") or {})
|
|
247
|
+
metadata["input_item_id"] = item.get("id")
|
|
248
|
+
persona_id = item.get("persona_id")
|
|
249
|
+
if persona_id:
|
|
250
|
+
metadata["persona_id"] = persona_id
|
|
251
|
+
persona_name = persona_map.get(persona_id)
|
|
252
|
+
if persona_name:
|
|
253
|
+
metadata.setdefault("persona", persona_name)
|
|
254
|
+
inputs_payload["inputs"].append(
|
|
255
|
+
{
|
|
256
|
+
"input": input_text,
|
|
257
|
+
"metadata": metadata,
|
|
258
|
+
}
|
|
259
|
+
)
|
|
260
|
+
|
|
261
|
+
inputs_path.parent.mkdir(parents=True, exist_ok=True)
|
|
262
|
+
inputs_path.write_text(yaml.safe_dump(inputs_payload, sort_keys=False, allow_unicode=True))
|
|
263
|
+
|
|
264
|
+
_write_json(sync_dir / "personas.json", {"items": personas})
|
|
265
|
+
_write_json(sync_dir / "criteria.json", {"items": data.get("criteria_pack") or []})
|
|
266
|
+
|
|
267
|
+
sync_state = {
|
|
268
|
+
"project_id": data.get("bundle_version", {}).get("project_id") or project_id,
|
|
269
|
+
"bundle_version_id": data.get("bundle_version", {}).get("id"),
|
|
270
|
+
"inputs_file": inputs_file,
|
|
271
|
+
"pulled_at": data.get("sync_meta", {}).get("pulled_at"),
|
|
272
|
+
"api_url": api_url,
|
|
273
|
+
}
|
|
274
|
+
_write_json(sync_dir / "sync.json", sync_state)
|
|
275
|
+
|
|
276
|
+
console.print(f"[green]✓[/green] Pulled {len(inputs_payload['inputs'])} inputs")
|
|
277
|
+
console.print(f"[green]✓[/green] Saved inputs to {inputs_path}")
|
|
278
|
+
console.print(f"[green]✓[/green] Saved sync metadata to {sync_dir / 'sync.json'}")
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
@app.command()
|
|
282
|
+
def upload(
|
|
283
|
+
experiment_dir: Optional[Path] = typer.Option(
|
|
284
|
+
None, "--experiment-dir", help="Experiment output directory"
|
|
285
|
+
),
|
|
286
|
+
config_file: Path = typer.Option(
|
|
287
|
+
DEFAULT_CONFIG_PATH, "--config", "-c", help="Config file to resolve output dir"
|
|
288
|
+
),
|
|
289
|
+
project: Optional[str] = typer.Option(None, "--project", help="Project name"),
|
|
290
|
+
root: Path = typer.Option(Path(DEFAULT_ROOT_DIR_NAME), "--root", help="Root dir"),
|
|
291
|
+
api_url: Optional[str] = typer.Option(None, "--api-url", help="Sync API base URL"),
|
|
292
|
+
api_key: Optional[str] = typer.Option(None, "--api-key", help="Sync API key"),
|
|
293
|
+
bundle_version_id: Optional[str] = typer.Option(
|
|
294
|
+
None, "--bundle-version-id", help="Bundle version id"
|
|
295
|
+
),
|
|
296
|
+
):
|
|
297
|
+
"""
|
|
298
|
+
Upload run results and artifacts to the sync API.
|
|
299
|
+
"""
|
|
300
|
+
_load_env(project, root)
|
|
301
|
+
api_url = _resolve_api_url(api_url)
|
|
302
|
+
api_key = _resolve_api_key(api_key)
|
|
303
|
+
|
|
304
|
+
project_root = _resolve_source_dir(project, root)
|
|
305
|
+
sync_state_path = project_root / ".fluxloop" / "sync" / "sync.json"
|
|
306
|
+
sync_state = json.loads(sync_state_path.read_text()) if sync_state_path.exists() else {}
|
|
307
|
+
|
|
308
|
+
bundle_version_id = bundle_version_id or sync_state.get("bundle_version_id")
|
|
309
|
+
if not bundle_version_id:
|
|
310
|
+
raise typer.BadParameter("bundle_version_id is required. Run sync pull first.")
|
|
311
|
+
|
|
312
|
+
resolved_config = resolve_config_path(config_file, project, root)
|
|
313
|
+
config = load_experiment_config(
|
|
314
|
+
resolved_config, project=project, root=root, require_inputs_file=False
|
|
315
|
+
)
|
|
316
|
+
output_base = resolve_project_relative(Path(config.output_directory), project, root)
|
|
317
|
+
|
|
318
|
+
if experiment_dir is None:
|
|
319
|
+
if not output_base.exists():
|
|
320
|
+
raise typer.BadParameter(f"Experiment output directory not found: {output_base}")
|
|
321
|
+
candidates = [
|
|
322
|
+
path
|
|
323
|
+
for path in output_base.iterdir()
|
|
324
|
+
if path.is_dir() and path.name != "artifacts"
|
|
325
|
+
]
|
|
326
|
+
if not candidates:
|
|
327
|
+
raise typer.BadParameter(f"No experiment directories found in {output_base}")
|
|
328
|
+
experiment_dir = sorted(candidates, key=lambda p: p.stat().st_mtime, reverse=True)[0]
|
|
329
|
+
else:
|
|
330
|
+
experiment_dir = experiment_dir.expanduser().resolve()
|
|
331
|
+
|
|
332
|
+
trace_summary_path = experiment_dir / "trace_summary.jsonl"
|
|
333
|
+
if not trace_summary_path.exists():
|
|
334
|
+
raise typer.BadParameter(f"trace_summary.jsonl not found: {trace_summary_path}")
|
|
335
|
+
|
|
336
|
+
inputs_path, _ = _resolve_inputs_path(project, root, config_file)
|
|
337
|
+
input_map = _load_inputs_mapping(inputs_path)
|
|
338
|
+
|
|
339
|
+
traces = list(_iter_jsonl(trace_summary_path))
|
|
340
|
+
run_ids: Dict[str, str] = {}
|
|
341
|
+
runs_payload: List[dict] = []
|
|
342
|
+
results_payload: List[dict] = []
|
|
343
|
+
all_completed = True
|
|
344
|
+
|
|
345
|
+
for trace in traces:
|
|
346
|
+
trace_id = trace.get("trace_id")
|
|
347
|
+
run_id = _coerce_run_id(trace_id)
|
|
348
|
+
run_ids[trace_id or run_id] = run_id
|
|
349
|
+
|
|
350
|
+
input_item_id = _resolve_input_item_id(trace, input_map)
|
|
351
|
+
if not input_item_id:
|
|
352
|
+
raise typer.BadParameter(
|
|
353
|
+
f"input_item_id not found for trace {trace_id}. Run sync pull first."
|
|
354
|
+
)
|
|
355
|
+
|
|
356
|
+
status = "completed" if trace.get("success") else "failed"
|
|
357
|
+
if status != "completed":
|
|
358
|
+
all_completed = False
|
|
359
|
+
|
|
360
|
+
runs_payload.append(
|
|
361
|
+
{
|
|
362
|
+
"id": run_id,
|
|
363
|
+
"input_item_id": input_item_id,
|
|
364
|
+
"persona_id": None,
|
|
365
|
+
"status": status,
|
|
366
|
+
}
|
|
367
|
+
)
|
|
368
|
+
|
|
369
|
+
metrics = {}
|
|
370
|
+
if trace.get("duration_ms") is not None:
|
|
371
|
+
metrics["duration_ms"] = trace.get("duration_ms")
|
|
372
|
+
if trace.get("token_usage"):
|
|
373
|
+
metrics.update(trace.get("token_usage"))
|
|
374
|
+
|
|
375
|
+
transcript = []
|
|
376
|
+
for entry in trace.get("conversation") or []:
|
|
377
|
+
if not isinstance(entry, dict):
|
|
378
|
+
continue
|
|
379
|
+
transcript.append(
|
|
380
|
+
{
|
|
381
|
+
"role": entry.get("role"),
|
|
382
|
+
"content": entry.get("content"),
|
|
383
|
+
"metadata": entry.get("metadata"),
|
|
384
|
+
}
|
|
385
|
+
)
|
|
386
|
+
|
|
387
|
+
local_eval_summary = {
|
|
388
|
+
"verdict": "pass" if trace.get("success") else "fail",
|
|
389
|
+
"scores": {},
|
|
390
|
+
"failures": [],
|
|
391
|
+
}
|
|
392
|
+
|
|
393
|
+
results_payload.append(
|
|
394
|
+
{
|
|
395
|
+
"run_id": run_id,
|
|
396
|
+
"status": status,
|
|
397
|
+
"metrics": metrics,
|
|
398
|
+
"transcript": transcript,
|
|
399
|
+
"local_eval_summary": local_eval_summary,
|
|
400
|
+
}
|
|
401
|
+
)
|
|
402
|
+
|
|
403
|
+
run_batch_status = "completed" if all_completed else "failed"
|
|
404
|
+
|
|
405
|
+
artifacts_payload: List[dict] = []
|
|
406
|
+
project_id = sync_state.get("project_id")
|
|
407
|
+
|
|
408
|
+
def _upload_file(run_id: str, artifact_type: str, file_path: Path) -> None:
|
|
409
|
+
if not file_path.exists():
|
|
410
|
+
return
|
|
411
|
+
content_type = _guess_content_type(file_path)
|
|
412
|
+
with httpx.Client(base_url=api_url, timeout=60.0) as client:
|
|
413
|
+
presign_resp = client.post(
|
|
414
|
+
"/api/storage/presign",
|
|
415
|
+
json={
|
|
416
|
+
"project_id": project_id,
|
|
417
|
+
"run_id": run_id,
|
|
418
|
+
"artifact_type": artifact_type,
|
|
419
|
+
"filename": file_path.name,
|
|
420
|
+
"content_type": content_type,
|
|
421
|
+
},
|
|
422
|
+
headers={"Authorization": f"Bearer {api_key}"},
|
|
423
|
+
)
|
|
424
|
+
presign_resp.raise_for_status()
|
|
425
|
+
presign = presign_resp.json()
|
|
426
|
+
|
|
427
|
+
upload_headers = presign.get("headers") or {}
|
|
428
|
+
upload_url = presign["upload_url"]
|
|
429
|
+
upload_resp = httpx.put(
|
|
430
|
+
upload_url,
|
|
431
|
+
content=file_path.read_bytes(),
|
|
432
|
+
headers=upload_headers,
|
|
433
|
+
timeout=120.0,
|
|
434
|
+
)
|
|
435
|
+
upload_resp.raise_for_status()
|
|
436
|
+
artifacts_payload.append(
|
|
437
|
+
{
|
|
438
|
+
"run_id": run_id,
|
|
439
|
+
"type": artifact_type,
|
|
440
|
+
"storage_url": presign["storage_url"],
|
|
441
|
+
}
|
|
442
|
+
)
|
|
443
|
+
|
|
444
|
+
per_trace_dir = experiment_dir / "per_trace_analysis"
|
|
445
|
+
if per_trace_dir.exists():
|
|
446
|
+
for trace in traces:
|
|
447
|
+
trace_id = trace.get("trace_id")
|
|
448
|
+
if not trace_id:
|
|
449
|
+
continue
|
|
450
|
+
run_id = run_ids.get(trace_id)
|
|
451
|
+
if not run_id:
|
|
452
|
+
continue
|
|
453
|
+
matches = list(per_trace_dir.glob(f"*{trace_id}*.md"))
|
|
454
|
+
if matches:
|
|
455
|
+
_upload_file(run_id, "per_trace_markdown", matches[0])
|
|
456
|
+
|
|
457
|
+
per_trace_jsonl = per_trace_dir / "per_trace.jsonl"
|
|
458
|
+
if traces:
|
|
459
|
+
first_run_id = run_ids.get(traces[0].get("trace_id"), list(run_ids.values())[0])
|
|
460
|
+
_upload_file(first_run_id, "per_trace_index", per_trace_jsonl)
|
|
461
|
+
|
|
462
|
+
if traces:
|
|
463
|
+
first_run_id = run_ids.get(traces[0].get("trace_id"), list(run_ids.values())[0])
|
|
464
|
+
_upload_file(first_run_id, "summary", experiment_dir / "summary.json")
|
|
465
|
+
_upload_file(first_run_id, "trace_summary", experiment_dir / "trace_summary.jsonl")
|
|
466
|
+
_upload_file(first_run_id, "observations", experiment_dir / "observations.jsonl")
|
|
467
|
+
|
|
468
|
+
payload = {
|
|
469
|
+
"bundle_version_id": bundle_version_id,
|
|
470
|
+
"run_batch": {
|
|
471
|
+
"execution_target": "local",
|
|
472
|
+
"status": run_batch_status,
|
|
473
|
+
},
|
|
474
|
+
"runs": runs_payload,
|
|
475
|
+
"run_results": results_payload,
|
|
476
|
+
"artifacts": artifacts_payload,
|
|
477
|
+
"upload_meta": {},
|
|
478
|
+
}
|
|
479
|
+
|
|
480
|
+
with httpx.Client(base_url=api_url, timeout=60.0) as client:
|
|
481
|
+
resp = client.post(
|
|
482
|
+
"/api/sync/upload",
|
|
483
|
+
json=payload,
|
|
484
|
+
headers={"Authorization": f"Bearer {api_key}"},
|
|
485
|
+
)
|
|
486
|
+
resp.raise_for_status()
|
|
487
|
+
|
|
488
|
+
console.print(f"[green]✓[/green] Uploaded {len(runs_payload)} runs")
|
|
@@ -93,7 +93,7 @@ DEFAULT_TEMPLATE = r"""<!DOCTYPE html>
|
|
|
93
93
|
</div>
|
|
94
94
|
</div>
|
|
95
95
|
<a
|
|
96
|
-
href="https://fluxloop.
|
|
96
|
+
href="https://fluxloop.ai/"
|
|
97
97
|
class="inline-flex items-center gap-2 self-start rounded-full border border-sky-400 px-4 py-2 text-sm font-semibold text-sky-200 transition hover:bg-sky-500/10 focus:outline-none focus:ring-2 focus:ring-sky-400 focus:ring-offset-2 focus:ring-offset-slate-950"
|
|
98
98
|
target="_blank"
|
|
99
99
|
rel="noopener noreferrer"
|
|
@@ -223,7 +223,7 @@ class ReportRenderer:
|
|
|
223
223
|
float(defaults["good"]),
|
|
224
224
|
float(defaults["fair"]),
|
|
225
225
|
)
|
|
226
|
-
|
|
226
|
+
|
|
227
227
|
eff_flags = {}
|
|
228
228
|
if "output_tokens" in efficiency:
|
|
229
229
|
eff_flags["verbose"] = efficiency["output_tokens"].get("std_multiplier", 2) * 1000 # Approximation
|
|
@@ -9,7 +9,18 @@ from rich.console import Console
|
|
|
9
9
|
from rich.panel import Panel
|
|
10
10
|
|
|
11
11
|
from . import __version__
|
|
12
|
-
from .commands import
|
|
12
|
+
from .commands import (
|
|
13
|
+
config,
|
|
14
|
+
doctor,
|
|
15
|
+
evaluate,
|
|
16
|
+
generate,
|
|
17
|
+
init,
|
|
18
|
+
parse,
|
|
19
|
+
record,
|
|
20
|
+
run,
|
|
21
|
+
status,
|
|
22
|
+
sync,
|
|
23
|
+
)
|
|
13
24
|
|
|
14
25
|
# Create the main Typer app
|
|
15
26
|
app = typer.Typer(
|
|
@@ -32,6 +43,7 @@ app.add_typer(parse.app, name="parse", help="Parse experiments into readable fil
|
|
|
32
43
|
app.add_typer(record.app, name="record", help="Manage recording mode and settings")
|
|
33
44
|
app.add_typer(doctor.app, name="doctor", help="Diagnose CLI and MCP environment issues")
|
|
34
45
|
app.add_typer(evaluate.app, name="evaluate", help="Evaluate experiment results")
|
|
46
|
+
app.add_typer(sync.app, name="sync", help="Sync bundles and upload results")
|
|
35
47
|
|
|
36
48
|
|
|
37
49
|
def version_callback(value: bool):
|
|
@@ -32,6 +32,7 @@ fluxloop_cli/commands/parse.py
|
|
|
32
32
|
fluxloop_cli/commands/record.py
|
|
33
33
|
fluxloop_cli/commands/run.py
|
|
34
34
|
fluxloop_cli/commands/status.py
|
|
35
|
+
fluxloop_cli/commands/sync.py
|
|
35
36
|
fluxloop_cli/evaluation/__init__.py
|
|
36
37
|
fluxloop_cli/evaluation/artifacts.py
|
|
37
38
|
fluxloop_cli/evaluation/config.py
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/evaluation/engine/reporting/__init__.py
RENAMED
|
File without changes
|
{fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/evaluation/engine/reporting/markdown.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/evaluation/prompts/intent_recognition.py
RENAMED
|
File without changes
|
{fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/evaluation/prompts/response_clarity.py
RENAMED
|
File without changes
|
{fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/evaluation/prompts/response_consistency.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|