fluxloop-cli 0.2.36__tar.gz → 0.3.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/PKG-INFO +1 -1
  2. {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/__init__.py +1 -1
  3. {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/commands/__init__.py +2 -1
  4. {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/commands/config.py +37 -0
  5. {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/commands/status.py +8 -7
  6. fluxloop_cli-0.3.1/fluxloop_cli/commands/sync.py +488 -0
  7. {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/evaluation/engine/reporting/html.py +1 -1
  8. {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/evaluation/report/renderer.py +1 -1
  9. {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/main.py +13 -1
  10. {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli.egg-info/PKG-INFO +1 -1
  11. {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli.egg-info/SOURCES.txt +1 -0
  12. {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/pyproject.toml +1 -1
  13. {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/README.md +0 -0
  14. {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/arg_binder.py +0 -0
  15. {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/commands/doctor.py +0 -0
  16. {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/commands/evaluate.py +0 -0
  17. {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/commands/generate.py +0 -0
  18. {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/commands/init.py +0 -0
  19. {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/commands/parse.py +0 -0
  20. {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/commands/record.py +0 -0
  21. {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/commands/run.py +0 -0
  22. {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/config_loader.py +0 -0
  23. {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/config_schema.py +0 -0
  24. {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/constants.py +0 -0
  25. {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/conversation_supervisor.py +0 -0
  26. {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/environment.py +0 -0
  27. {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/evaluation/__init__.py +0 -0
  28. {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/evaluation/artifacts.py +0 -0
  29. {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/evaluation/config.py +0 -0
  30. {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/evaluation/engine/__init__.py +0 -0
  31. {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/evaluation/engine/analysis.py +0 -0
  32. {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/evaluation/engine/core.py +0 -0
  33. {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/evaluation/engine/reporting/__init__.py +0 -0
  34. {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/evaluation/engine/reporting/markdown.py +0 -0
  35. {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/evaluation/engine/success.py +0 -0
  36. {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/evaluation/llm.py +0 -0
  37. {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/evaluation/prompts/__init__.py +0 -0
  38. {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/evaluation/prompts/base.py +0 -0
  39. {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/evaluation/prompts/information_completeness.py +0 -0
  40. {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/evaluation/prompts/intent_recognition.py +0 -0
  41. {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/evaluation/prompts/response_clarity.py +0 -0
  42. {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/evaluation/prompts/response_consistency.py +0 -0
  43. {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/evaluation/report/__init__.py +0 -0
  44. {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/evaluation/report/aggregator.py +0 -0
  45. {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/evaluation/report/generator.py +0 -0
  46. {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/evaluation/report/pipeline.py +0 -0
  47. {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/evaluation/rules.py +0 -0
  48. {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/evaluation/templates/report.html.j2 +0 -0
  49. {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/input_generator.py +0 -0
  50. {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/llm_generator.py +0 -0
  51. {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/project_paths.py +0 -0
  52. {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/runner.py +0 -0
  53. {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/target_loader.py +0 -0
  54. {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/templates.py +0 -0
  55. {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/testing/__init__.py +0 -0
  56. {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/testing/pytest_plugin.py +0 -0
  57. {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/testing/types.py +0 -0
  58. {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/token_usage.py +0 -0
  59. {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli/validators.py +0 -0
  60. {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli.egg-info/dependency_links.txt +0 -0
  61. {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli.egg-info/entry_points.txt +0 -0
  62. {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli.egg-info/requires.txt +0 -0
  63. {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/fluxloop_cli.egg-info/top_level.txt +0 -0
  64. {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/setup.cfg +0 -0
  65. {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/tests/test_analysis_recommendations.py +0 -0
  66. {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/tests/test_arg_binder.py +0 -0
  67. {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/tests/test_config_command.py +0 -0
  68. {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/tests/test_conversation_supervisor.py +0 -0
  69. {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/tests/test_evaluate_command.py +0 -0
  70. {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/tests/test_evaluation_llm.py +0 -0
  71. {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/tests/test_init_pytest_template.py +0 -0
  72. {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/tests/test_input_generator.py +0 -0
  73. {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/tests/test_prompt_library.py +0 -0
  74. {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/tests/test_pytest_plugin.py +0 -0
  75. {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/tests/test_run_command.py +0 -0
  76. {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/tests/test_runner_multi_turn.py +0 -0
  77. {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/tests/test_success_criteria.py +0 -0
  78. {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/tests/test_target_loader.py +0 -0
  79. {fluxloop_cli-0.2.36 → fluxloop_cli-0.3.1}/tests/test_testing_types.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: fluxloop-cli
3
- Version: 0.2.36
3
+ Version: 0.3.1
4
4
  Summary: FluxLoop CLI for running agent simulations
5
5
  Author-email: FluxLoop Team <team@fluxloop.dev>
6
6
  License: Apache-2.0
@@ -2,7 +2,7 @@
2
2
  FluxLoop CLI - Command-line interface for running agent simulations.
3
3
  """
4
4
 
5
- __version__ = "0.2.36"
5
+ __version__ = "0.3.1"
6
6
 
7
7
  from .main import app
8
8
 
@@ -1,6 +1,6 @@
1
1
  """CLI commands."""
2
2
 
3
- from . import config, doctor, evaluate, generate, init, parse, record, run, status
3
+ from . import config, doctor, evaluate, generate, init, parse, record, run, status, sync
4
4
 
5
5
  __all__ = [
6
6
  "config",
@@ -12,4 +12,5 @@ __all__ = [
12
12
  "record",
13
13
  "run",
14
14
  "status",
15
+ "sync",
15
16
  ]
@@ -192,7 +192,9 @@ def env(
192
192
 
193
193
  env_vars = [
194
194
  ("FLUXLOOP_COLLECTOR_URL", "Collector service URL", "http://localhost:8000"),
195
+ ("FLUXLOOP_SYNC_URL", "Sync API base URL", None),
195
196
  ("FLUXLOOP_API_KEY", "API key for authentication", None),
197
+ ("FLUXLOOP_SYNC_API_KEY", "Sync API key for uploads", None),
196
198
  ("FLUXLOOP_ENABLED", "Enable/disable tracing", "true"),
197
199
  ("FLUXLOOP_DEBUG", "Enable debug mode", "false"),
198
200
  ("FLUXLOOP_SAMPLE_RATE", "Trace sampling rate (0-1)", "1.0"),
@@ -323,6 +325,41 @@ def set_llm(
323
325
  )
324
326
 
325
327
 
328
+ @app.command()
329
+ def set_sync_key(
330
+ api_key: str = typer.Argument(..., help="Sync API key"),
331
+ api_url: Optional[str] = typer.Option(None, "--url", help="Sync API base URL"),
332
+ overwrite_env: bool = typer.Option(False, "--overwrite-env", help="Overwrite existing key in .env"),
333
+ env_file: Path = typer.Option(Path(".env"), "--env-file", help="Path to environment file"),
334
+ project: Optional[str] = typer.Option(None, "--project", help="Project name under the FluxLoop root"),
335
+ root: Path = typer.Option(Path(DEFAULT_ROOT_DIR_NAME), "--root", help="FluxLoop root directory"),
336
+ ):
337
+ """Save Sync API credentials to .env."""
338
+
339
+ env_path = resolve_env_path(env_file, project, root)
340
+ env_contents: Dict[str, str] = {}
341
+ if env_path.exists():
342
+ for line in env_path.read_text().splitlines():
343
+ if not line or line.startswith("#") or "=" not in line:
344
+ continue
345
+ key, value = line.split("=", 1)
346
+ env_contents[key.strip()] = value.strip()
347
+
348
+ if "FLUXLOOP_SYNC_API_KEY" in env_contents and not overwrite_env:
349
+ console.print(
350
+ "[yellow]Warning:[/yellow] FLUXLOOP_SYNC_API_KEY already set. Use --overwrite-env to replace it."
351
+ )
352
+ else:
353
+ env_contents["FLUXLOOP_SYNC_API_KEY"] = api_key
354
+
355
+ if api_url:
356
+ env_contents["FLUXLOOP_SYNC_URL"] = api_url
357
+
358
+ env_lines = [f"{key}={value}" for key, value in env_contents.items()]
359
+ env_path.write_text("\n".join(env_lines) + "\n")
360
+ console.print(f"[green]✓[/green] Saved sync credentials to {env_path}")
361
+
362
+
326
363
  @app.command()
327
364
  def validate(
328
365
  config_file: Path = typer.Option(
@@ -133,17 +133,18 @@ def check(
133
133
 
134
134
  # Check environment
135
135
  import os
136
- if os.getenv("FLUXLOOP_API_KEY"):
136
+ sync_key = os.getenv("FLUXLOOP_SYNC_API_KEY") or os.getenv("FLUXLOOP_API_KEY")
137
+ if sync_key:
137
138
  status_table.add_row(
138
- "API Key",
139
+ "Sync API Key",
139
140
  "[green]✓ Set[/green]",
140
- "****" + os.getenv("FLUXLOOP_API_KEY")[-4:] if verbose else "Configured"
141
+ "****" + sync_key[-4:] if verbose else "Configured",
141
142
  )
142
143
  else:
143
144
  status_table.add_row(
144
- "API Key",
145
+ "Sync API Key",
145
146
  "[yellow]- Not set[/yellow]",
146
- "Set FLUXLOOP_API_KEY in .env"
147
+ "Set FLUXLOOP_SYNC_API_KEY in .env",
147
148
  )
148
149
 
149
150
  console.print(status_table)
@@ -152,8 +153,8 @@ def check(
152
153
  recommendations = []
153
154
  if not resolved_config.exists():
154
155
  recommendations.append("Initialize a project: [cyan]fluxloop init project[/cyan]")
155
- if not os.getenv("FLUXLOOP_API_KEY"):
156
- recommendations.append("Set up API key in .env file")
156
+ if not sync_key:
157
+ recommendations.append("Set up sync API key in .env file")
157
158
 
158
159
  if recommendations:
159
160
  console.print("\n[bold]Recommendations:[/bold]")
@@ -0,0 +1,488 @@
1
+ """
2
+ Sync command for pulling bundles and uploading results.
3
+ """
4
+
5
+ from __future__ import annotations
6
+
7
+ import json
8
+ import os
9
+ from pathlib import Path
10
+ from typing import Any, Dict, Iterable, List, Optional, Tuple
11
+ from uuid import UUID, uuid4
12
+
13
+ import httpx
14
+ import typer
15
+ import yaml
16
+ from rich.console import Console
17
+
18
+ from ..config_loader import load_experiment_config
19
+ from ..constants import DEFAULT_CONFIG_PATH, DEFAULT_ROOT_DIR_NAME
20
+ from ..environment import load_env_chain
21
+ from ..project_paths import (
22
+ resolve_config_path,
23
+ resolve_project_dir,
24
+ resolve_project_relative,
25
+ )
26
+
27
+
28
+ app = typer.Typer(help="Sync bundle inputs and upload run results.")
29
+ console = Console()
30
+
31
+
32
+ def _resolve_source_dir(project: Optional[str], root: Path) -> Path:
33
+ if project:
34
+ return resolve_project_dir(project, root)
35
+ return Path.cwd().resolve()
36
+
37
+
38
+ def _load_env(project: Optional[str], root: Path) -> None:
39
+ source_dir = _resolve_source_dir(project, root)
40
+ load_env_chain(source_dir, refresh_config=False)
41
+
42
+
43
+ def _resolve_api_url(override: Optional[str]) -> str:
44
+ url = (
45
+ override
46
+ or os.getenv("FLUXLOOP_SYNC_URL")
47
+ or os.getenv("FLUXLOOP_COLLECTOR_URL")
48
+ or "http://localhost:8000"
49
+ )
50
+ return url.rstrip("/")
51
+
52
+
53
+ def _resolve_api_key(override: Optional[str]) -> str:
54
+ key = (
55
+ override
56
+ or os.getenv("FLUXLOOP_SYNC_API_KEY")
57
+ or os.getenv("FLUXLOOP_API_KEY")
58
+ )
59
+ if not key:
60
+ raise typer.BadParameter("Sync API key is not set. Use fluxloop config set-sync-key.")
61
+ return key
62
+
63
+
64
+ def _ensure_sync_dir(project_root: Path) -> Path:
65
+ sync_dir = project_root / ".fluxloop" / "sync"
66
+ sync_dir.mkdir(parents=True, exist_ok=True)
67
+ return sync_dir
68
+
69
+
70
+ def _write_json(path: Path, payload: dict) -> None:
71
+ path.parent.mkdir(parents=True, exist_ok=True)
72
+ path.write_text(json.dumps(payload, indent=2, ensure_ascii=False))
73
+
74
+
75
+ def _extract_input_text(messages: Any) -> str:
76
+ if isinstance(messages, list):
77
+ for entry in messages:
78
+ if isinstance(entry, dict):
79
+ role = entry.get("role") or entry.get("type")
80
+ if role in ("user", "human"):
81
+ content = entry.get("content")
82
+ if isinstance(content, str):
83
+ return content
84
+ for entry in messages:
85
+ if isinstance(entry, dict):
86
+ content = entry.get("content")
87
+ if isinstance(content, str):
88
+ return content
89
+ if isinstance(messages, dict):
90
+ content = messages.get("content")
91
+ if isinstance(content, str):
92
+ return content
93
+ if isinstance(messages, str):
94
+ return messages
95
+ return ""
96
+
97
+
98
+ def _resolve_inputs_path(
99
+ project: Optional[str], root: Path, config_file: Path
100
+ ) -> Tuple[Path, Optional[str]]:
101
+ resolved = resolve_config_path(config_file, project, root)
102
+ if resolved.exists():
103
+ try:
104
+ config = load_experiment_config(
105
+ resolved, project=project, root=root, require_inputs_file=False
106
+ )
107
+ inputs_file = config.inputs_file or "inputs/generated.yaml"
108
+ return (
109
+ resolve_project_relative(Path(inputs_file), project, root),
110
+ inputs_file,
111
+ )
112
+ except Exception:
113
+ pass
114
+ inputs_file = "inputs/generated.yaml"
115
+ return resolve_project_relative(Path(inputs_file), project, root), inputs_file
116
+
117
+
118
+ def _load_inputs_mapping(inputs_path: Path) -> Dict[Tuple[str, Optional[str]], List[str]]:
119
+ if not inputs_path.exists():
120
+ return {}
121
+ payload = yaml.safe_load(inputs_path.read_text()) or {}
122
+ entries = payload.get("inputs") if isinstance(payload, dict) else payload
123
+ if not isinstance(entries, list):
124
+ return {}
125
+
126
+ mapping: Dict[Tuple[str, Optional[str]], List[str]] = {}
127
+ for entry in entries:
128
+ if not isinstance(entry, dict):
129
+ continue
130
+ input_text = entry.get("input")
131
+ metadata = entry.get("metadata") or {}
132
+ item_id = metadata.get("input_item_id")
133
+ if not input_text or not item_id:
134
+ continue
135
+ persona = entry.get("persona") or metadata.get("persona")
136
+ mapping.setdefault((input_text, persona), []).append(item_id)
137
+ return mapping
138
+
139
+
140
+ def _resolve_input_item_id(
141
+ trace: dict, mapping: Dict[Tuple[str, Optional[str]], List[str]]
142
+ ) -> Optional[str]:
143
+ conversation_state = trace.get("conversation_state") or {}
144
+ metadata = conversation_state.get("metadata") if isinstance(conversation_state, dict) else {}
145
+ variation = metadata.get("variation") if isinstance(metadata, dict) else None
146
+ if isinstance(variation, dict):
147
+ direct = variation.get("input_item_id")
148
+ if direct:
149
+ return direct
150
+
151
+ input_text = trace.get("input")
152
+ persona = trace.get("persona")
153
+ if input_text:
154
+ key = (input_text, persona)
155
+ if key in mapping and mapping[key]:
156
+ return mapping[key].pop(0)
157
+ key = (input_text, None)
158
+ if key in mapping and mapping[key]:
159
+ return mapping[key].pop(0)
160
+ return None
161
+
162
+
163
+ def _coerce_run_id(trace_id: Optional[str]) -> str:
164
+ if trace_id:
165
+ try:
166
+ return str(UUID(trace_id))
167
+ except Exception:
168
+ pass
169
+ return str(uuid4())
170
+
171
+
172
+ def _iter_jsonl(path: Path) -> Iterable[dict]:
173
+ with path.open("r", encoding="utf-8") as handle:
174
+ for line in handle:
175
+ line = line.strip()
176
+ if not line:
177
+ continue
178
+ try:
179
+ yield json.loads(line)
180
+ except json.JSONDecodeError:
181
+ continue
182
+
183
+
184
+ def _guess_content_type(path: Path) -> str:
185
+ if path.suffix in (".json", ".jsonl"):
186
+ return "application/json"
187
+ if path.suffix == ".md":
188
+ return "text/markdown"
189
+ if path.suffix == ".html":
190
+ return "text/html"
191
+ if path.suffix == ".pdf":
192
+ return "application/pdf"
193
+ return "application/octet-stream"
194
+
195
+
196
+ @app.command()
197
+ def pull(
198
+ project_id: Optional[str] = typer.Option(None, "--project-id", help="Project id"),
199
+ bundle_version_id: Optional[str] = typer.Option(
200
+ None, "--bundle-version-id", help="Bundle version id"
201
+ ),
202
+ config_file: Path = typer.Option(
203
+ DEFAULT_CONFIG_PATH, "--config", "-c", help="Config file to resolve inputs path"
204
+ ),
205
+ project: Optional[str] = typer.Option(None, "--project", help="Project name"),
206
+ root: Path = typer.Option(Path(DEFAULT_ROOT_DIR_NAME), "--root", help="Root dir"),
207
+ api_url: Optional[str] = typer.Option(None, "--api-url", help="Sync API base URL"),
208
+ api_key: Optional[str] = typer.Option(None, "--api-key", help="Sync API key"),
209
+ ):
210
+ """
211
+ Pull bundle inputs/personas/criteria for local execution.
212
+ """
213
+ _load_env(project, root)
214
+ api_url = _resolve_api_url(api_url)
215
+ api_key = _resolve_api_key(api_key)
216
+
217
+ payload: Dict[str, Any] = {
218
+ "bundle_version_id": bundle_version_id,
219
+ "project_id": project_id,
220
+ "include_inputs": True,
221
+ "include_personas": True,
222
+ "include_criteria": True,
223
+ }
224
+
225
+ with httpx.Client(base_url=api_url, timeout=30.0) as client:
226
+ resp = client.post(
227
+ "/api/sync/pull",
228
+ json=payload,
229
+ headers={"Authorization": f"Bearer {api_key}"},
230
+ )
231
+ resp.raise_for_status()
232
+ data = resp.json()
233
+
234
+ project_root = _resolve_source_dir(project, root)
235
+ sync_dir = _ensure_sync_dir(project_root)
236
+
237
+ personas = data.get("personas") or []
238
+ persona_map = {item.get("id"): item.get("name") for item in personas if item.get("id")}
239
+
240
+ inputs_path, inputs_file = _resolve_inputs_path(project, root, config_file)
241
+ inputs_payload = {"inputs": []}
242
+ for item in data.get("input_items") or []:
243
+ input_text = _extract_input_text(item.get("messages"))
244
+ if not input_text:
245
+ continue
246
+ metadata = dict(item.get("metadata") or {})
247
+ metadata["input_item_id"] = item.get("id")
248
+ persona_id = item.get("persona_id")
249
+ if persona_id:
250
+ metadata["persona_id"] = persona_id
251
+ persona_name = persona_map.get(persona_id)
252
+ if persona_name:
253
+ metadata.setdefault("persona", persona_name)
254
+ inputs_payload["inputs"].append(
255
+ {
256
+ "input": input_text,
257
+ "metadata": metadata,
258
+ }
259
+ )
260
+
261
+ inputs_path.parent.mkdir(parents=True, exist_ok=True)
262
+ inputs_path.write_text(yaml.safe_dump(inputs_payload, sort_keys=False, allow_unicode=True))
263
+
264
+ _write_json(sync_dir / "personas.json", {"items": personas})
265
+ _write_json(sync_dir / "criteria.json", {"items": data.get("criteria_pack") or []})
266
+
267
+ sync_state = {
268
+ "project_id": data.get("bundle_version", {}).get("project_id") or project_id,
269
+ "bundle_version_id": data.get("bundle_version", {}).get("id"),
270
+ "inputs_file": inputs_file,
271
+ "pulled_at": data.get("sync_meta", {}).get("pulled_at"),
272
+ "api_url": api_url,
273
+ }
274
+ _write_json(sync_dir / "sync.json", sync_state)
275
+
276
+ console.print(f"[green]✓[/green] Pulled {len(inputs_payload['inputs'])} inputs")
277
+ console.print(f"[green]✓[/green] Saved inputs to {inputs_path}")
278
+ console.print(f"[green]✓[/green] Saved sync metadata to {sync_dir / 'sync.json'}")
279
+
280
+
281
+ @app.command()
282
+ def upload(
283
+ experiment_dir: Optional[Path] = typer.Option(
284
+ None, "--experiment-dir", help="Experiment output directory"
285
+ ),
286
+ config_file: Path = typer.Option(
287
+ DEFAULT_CONFIG_PATH, "--config", "-c", help="Config file to resolve output dir"
288
+ ),
289
+ project: Optional[str] = typer.Option(None, "--project", help="Project name"),
290
+ root: Path = typer.Option(Path(DEFAULT_ROOT_DIR_NAME), "--root", help="Root dir"),
291
+ api_url: Optional[str] = typer.Option(None, "--api-url", help="Sync API base URL"),
292
+ api_key: Optional[str] = typer.Option(None, "--api-key", help="Sync API key"),
293
+ bundle_version_id: Optional[str] = typer.Option(
294
+ None, "--bundle-version-id", help="Bundle version id"
295
+ ),
296
+ ):
297
+ """
298
+ Upload run results and artifacts to the sync API.
299
+ """
300
+ _load_env(project, root)
301
+ api_url = _resolve_api_url(api_url)
302
+ api_key = _resolve_api_key(api_key)
303
+
304
+ project_root = _resolve_source_dir(project, root)
305
+ sync_state_path = project_root / ".fluxloop" / "sync" / "sync.json"
306
+ sync_state = json.loads(sync_state_path.read_text()) if sync_state_path.exists() else {}
307
+
308
+ bundle_version_id = bundle_version_id or sync_state.get("bundle_version_id")
309
+ if not bundle_version_id:
310
+ raise typer.BadParameter("bundle_version_id is required. Run sync pull first.")
311
+
312
+ resolved_config = resolve_config_path(config_file, project, root)
313
+ config = load_experiment_config(
314
+ resolved_config, project=project, root=root, require_inputs_file=False
315
+ )
316
+ output_base = resolve_project_relative(Path(config.output_directory), project, root)
317
+
318
+ if experiment_dir is None:
319
+ if not output_base.exists():
320
+ raise typer.BadParameter(f"Experiment output directory not found: {output_base}")
321
+ candidates = [
322
+ path
323
+ for path in output_base.iterdir()
324
+ if path.is_dir() and path.name != "artifacts"
325
+ ]
326
+ if not candidates:
327
+ raise typer.BadParameter(f"No experiment directories found in {output_base}")
328
+ experiment_dir = sorted(candidates, key=lambda p: p.stat().st_mtime, reverse=True)[0]
329
+ else:
330
+ experiment_dir = experiment_dir.expanduser().resolve()
331
+
332
+ trace_summary_path = experiment_dir / "trace_summary.jsonl"
333
+ if not trace_summary_path.exists():
334
+ raise typer.BadParameter(f"trace_summary.jsonl not found: {trace_summary_path}")
335
+
336
+ inputs_path, _ = _resolve_inputs_path(project, root, config_file)
337
+ input_map = _load_inputs_mapping(inputs_path)
338
+
339
+ traces = list(_iter_jsonl(trace_summary_path))
340
+ run_ids: Dict[str, str] = {}
341
+ runs_payload: List[dict] = []
342
+ results_payload: List[dict] = []
343
+ all_completed = True
344
+
345
+ for trace in traces:
346
+ trace_id = trace.get("trace_id")
347
+ run_id = _coerce_run_id(trace_id)
348
+ run_ids[trace_id or run_id] = run_id
349
+
350
+ input_item_id = _resolve_input_item_id(trace, input_map)
351
+ if not input_item_id:
352
+ raise typer.BadParameter(
353
+ f"input_item_id not found for trace {trace_id}. Run sync pull first."
354
+ )
355
+
356
+ status = "completed" if trace.get("success") else "failed"
357
+ if status != "completed":
358
+ all_completed = False
359
+
360
+ runs_payload.append(
361
+ {
362
+ "id": run_id,
363
+ "input_item_id": input_item_id,
364
+ "persona_id": None,
365
+ "status": status,
366
+ }
367
+ )
368
+
369
+ metrics = {}
370
+ if trace.get("duration_ms") is not None:
371
+ metrics["duration_ms"] = trace.get("duration_ms")
372
+ if trace.get("token_usage"):
373
+ metrics.update(trace.get("token_usage"))
374
+
375
+ transcript = []
376
+ for entry in trace.get("conversation") or []:
377
+ if not isinstance(entry, dict):
378
+ continue
379
+ transcript.append(
380
+ {
381
+ "role": entry.get("role"),
382
+ "content": entry.get("content"),
383
+ "metadata": entry.get("metadata"),
384
+ }
385
+ )
386
+
387
+ local_eval_summary = {
388
+ "verdict": "pass" if trace.get("success") else "fail",
389
+ "scores": {},
390
+ "failures": [],
391
+ }
392
+
393
+ results_payload.append(
394
+ {
395
+ "run_id": run_id,
396
+ "status": status,
397
+ "metrics": metrics,
398
+ "transcript": transcript,
399
+ "local_eval_summary": local_eval_summary,
400
+ }
401
+ )
402
+
403
+ run_batch_status = "completed" if all_completed else "failed"
404
+
405
+ artifacts_payload: List[dict] = []
406
+ project_id = sync_state.get("project_id")
407
+
408
+ def _upload_file(run_id: str, artifact_type: str, file_path: Path) -> None:
409
+ if not file_path.exists():
410
+ return
411
+ content_type = _guess_content_type(file_path)
412
+ with httpx.Client(base_url=api_url, timeout=60.0) as client:
413
+ presign_resp = client.post(
414
+ "/api/storage/presign",
415
+ json={
416
+ "project_id": project_id,
417
+ "run_id": run_id,
418
+ "artifact_type": artifact_type,
419
+ "filename": file_path.name,
420
+ "content_type": content_type,
421
+ },
422
+ headers={"Authorization": f"Bearer {api_key}"},
423
+ )
424
+ presign_resp.raise_for_status()
425
+ presign = presign_resp.json()
426
+
427
+ upload_headers = presign.get("headers") or {}
428
+ upload_url = presign["upload_url"]
429
+ upload_resp = httpx.put(
430
+ upload_url,
431
+ content=file_path.read_bytes(),
432
+ headers=upload_headers,
433
+ timeout=120.0,
434
+ )
435
+ upload_resp.raise_for_status()
436
+ artifacts_payload.append(
437
+ {
438
+ "run_id": run_id,
439
+ "type": artifact_type,
440
+ "storage_url": presign["storage_url"],
441
+ }
442
+ )
443
+
444
+ per_trace_dir = experiment_dir / "per_trace_analysis"
445
+ if per_trace_dir.exists():
446
+ for trace in traces:
447
+ trace_id = trace.get("trace_id")
448
+ if not trace_id:
449
+ continue
450
+ run_id = run_ids.get(trace_id)
451
+ if not run_id:
452
+ continue
453
+ matches = list(per_trace_dir.glob(f"*{trace_id}*.md"))
454
+ if matches:
455
+ _upload_file(run_id, "per_trace_markdown", matches[0])
456
+
457
+ per_trace_jsonl = per_trace_dir / "per_trace.jsonl"
458
+ if traces:
459
+ first_run_id = run_ids.get(traces[0].get("trace_id"), list(run_ids.values())[0])
460
+ _upload_file(first_run_id, "per_trace_index", per_trace_jsonl)
461
+
462
+ if traces:
463
+ first_run_id = run_ids.get(traces[0].get("trace_id"), list(run_ids.values())[0])
464
+ _upload_file(first_run_id, "summary", experiment_dir / "summary.json")
465
+ _upload_file(first_run_id, "trace_summary", experiment_dir / "trace_summary.jsonl")
466
+ _upload_file(first_run_id, "observations", experiment_dir / "observations.jsonl")
467
+
468
+ payload = {
469
+ "bundle_version_id": bundle_version_id,
470
+ "run_batch": {
471
+ "execution_target": "local",
472
+ "status": run_batch_status,
473
+ },
474
+ "runs": runs_payload,
475
+ "run_results": results_payload,
476
+ "artifacts": artifacts_payload,
477
+ "upload_meta": {},
478
+ }
479
+
480
+ with httpx.Client(base_url=api_url, timeout=60.0) as client:
481
+ resp = client.post(
482
+ "/api/sync/upload",
483
+ json=payload,
484
+ headers={"Authorization": f"Bearer {api_key}"},
485
+ )
486
+ resp.raise_for_status()
487
+
488
+ console.print(f"[green]✓[/green] Uploaded {len(runs_payload)} runs")
@@ -93,7 +93,7 @@ DEFAULT_TEMPLATE = r"""<!DOCTYPE html>
93
93
  </div>
94
94
  </div>
95
95
  <a
96
- href="https://fluxloop.io/"
96
+ href="https://fluxloop.ai/"
97
97
  class="inline-flex items-center gap-2 self-start rounded-full border border-sky-400 px-4 py-2 text-sm font-semibold text-sky-200 transition hover:bg-sky-500/10 focus:outline-none focus:ring-2 focus:ring-sky-400 focus:ring-offset-2 focus:ring-offset-slate-950"
98
98
  target="_blank"
99
99
  rel="noopener noreferrer"
@@ -223,7 +223,7 @@ class ReportRenderer:
223
223
  float(defaults["good"]),
224
224
  float(defaults["fair"]),
225
225
  )
226
-
226
+
227
227
  eff_flags = {}
228
228
  if "output_tokens" in efficiency:
229
229
  eff_flags["verbose"] = efficiency["output_tokens"].get("std_multiplier", 2) * 1000 # Approximation
@@ -9,7 +9,18 @@ from rich.console import Console
9
9
  from rich.panel import Panel
10
10
 
11
11
  from . import __version__
12
- from .commands import config, doctor, evaluate, generate, init, parse, record, run, status
12
+ from .commands import (
13
+ config,
14
+ doctor,
15
+ evaluate,
16
+ generate,
17
+ init,
18
+ parse,
19
+ record,
20
+ run,
21
+ status,
22
+ sync,
23
+ )
13
24
 
14
25
  # Create the main Typer app
15
26
  app = typer.Typer(
@@ -32,6 +43,7 @@ app.add_typer(parse.app, name="parse", help="Parse experiments into readable fil
32
43
  app.add_typer(record.app, name="record", help="Manage recording mode and settings")
33
44
  app.add_typer(doctor.app, name="doctor", help="Diagnose CLI and MCP environment issues")
34
45
  app.add_typer(evaluate.app, name="evaluate", help="Evaluate experiment results")
46
+ app.add_typer(sync.app, name="sync", help="Sync bundles and upload results")
35
47
 
36
48
 
37
49
  def version_callback(value: bool):
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: fluxloop-cli
3
- Version: 0.2.36
3
+ Version: 0.3.1
4
4
  Summary: FluxLoop CLI for running agent simulations
5
5
  Author-email: FluxLoop Team <team@fluxloop.dev>
6
6
  License: Apache-2.0
@@ -32,6 +32,7 @@ fluxloop_cli/commands/parse.py
32
32
  fluxloop_cli/commands/record.py
33
33
  fluxloop_cli/commands/run.py
34
34
  fluxloop_cli/commands/status.py
35
+ fluxloop_cli/commands/sync.py
35
36
  fluxloop_cli/evaluation/__init__.py
36
37
  fluxloop_cli/evaluation/artifacts.py
37
38
  fluxloop_cli/evaluation/config.py
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "fluxloop-cli"
7
- version = "0.2.36"
7
+ version = "0.3.1"
8
8
  description = "FluxLoop CLI for running agent simulations"
9
9
  authors = [
10
10
  {name = "FluxLoop Team", email = "team@fluxloop.dev"}
File without changes
File without changes