forgexa-cli 1.18.12__tar.gz → 1.20.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {forgexa_cli-1.18.12 → forgexa_cli-1.20.0}/PKG-INFO +1 -1
- {forgexa_cli-1.18.12 → forgexa_cli-1.20.0}/forgexa_cli/__init__.py +1 -1
- {forgexa_cli-1.18.12 → forgexa_cli-1.20.0}/forgexa_cli/daemon.py +318 -73
- {forgexa_cli-1.18.12 → forgexa_cli-1.20.0}/forgexa_cli.egg-info/PKG-INFO +1 -1
- {forgexa_cli-1.18.12 → forgexa_cli-1.20.0}/pyproject.toml +1 -1
- {forgexa_cli-1.18.12 → forgexa_cli-1.20.0}/README.md +0 -0
- {forgexa_cli-1.18.12 → forgexa_cli-1.20.0}/forgexa_cli/_build_config.py +0 -0
- {forgexa_cli-1.18.12 → forgexa_cli-1.20.0}/forgexa_cli/main.py +0 -0
- {forgexa_cli-1.18.12 → forgexa_cli-1.20.0}/forgexa_cli/py.typed +0 -0
- {forgexa_cli-1.18.12 → forgexa_cli-1.20.0}/forgexa_cli.egg-info/SOURCES.txt +0 -0
- {forgexa_cli-1.18.12 → forgexa_cli-1.20.0}/forgexa_cli.egg-info/dependency_links.txt +0 -0
- {forgexa_cli-1.18.12 → forgexa_cli-1.20.0}/forgexa_cli.egg-info/entry_points.txt +0 -0
- {forgexa_cli-1.18.12 → forgexa_cli-1.20.0}/forgexa_cli.egg-info/requires.txt +0 -0
- {forgexa_cli-1.18.12 → forgexa_cli-1.20.0}/forgexa_cli.egg-info/top_level.txt +0 -0
- {forgexa_cli-1.18.12 → forgexa_cli-1.20.0}/setup.cfg +0 -0
- {forgexa_cli-1.18.12 → forgexa_cli-1.20.0}/tests/test_auth_and_runtime_commands.py +0 -0
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
"""forgexa-cli — Forgexa command-line client."""
|
|
2
|
-
__version__ = "1.
|
|
2
|
+
__version__ = "1.20.0"
|
|
@@ -555,7 +555,7 @@ except (ImportError, ModuleNotFoundError):
|
|
|
555
555
|
# DAEMON_VERSION is the protocol/logic version of the daemon code.
|
|
556
556
|
# Kept in sync with pyproject.toml version via bump-version.sh.
|
|
557
557
|
# CLIENT_TYPE identifies which packaging/distribution this daemon runs in.
|
|
558
|
-
DAEMON_VERSION = "1.
|
|
558
|
+
DAEMON_VERSION = "1.20.0"
|
|
559
559
|
|
|
560
560
|
|
|
561
561
|
def _detect_client_type() -> str:
|
|
@@ -3520,16 +3520,20 @@ def _kill_proc(proc: asyncio.subprocess.Process) -> None:
|
|
|
3520
3520
|
|
|
3521
3521
|
|
|
3522
3522
|
class _IdleTimeoutError(asyncio.TimeoutError):
|
|
3523
|
-
"""Raised when an agent
|
|
3523
|
+
"""Raised when an agent produces no observable activity for AGENT_IDLE_TIMEOUT.
|
|
3524
3524
|
|
|
3525
3525
|
Subclasses asyncio.TimeoutError so existing ``except asyncio.TimeoutError``
|
|
3526
3526
|
handlers catch it, but callers can distinguish it from an absolute wall-clock
|
|
3527
3527
|
timeout via ``isinstance(exc, _IdleTimeoutError)`` or ``exc.idle_seconds``.
|
|
3528
|
+
Partial output is attached before propagation so the failure report retains
|
|
3529
|
+
the evidence needed to diagnose the blocked operation.
|
|
3528
3530
|
"""
|
|
3529
3531
|
|
|
3530
3532
|
def __init__(self, idle_seconds: float) -> None:
|
|
3531
3533
|
super().__init__(f"idle:{idle_seconds:.0f}s")
|
|
3532
3534
|
self.idle_seconds = idle_seconds
|
|
3535
|
+
self.stdout = ""
|
|
3536
|
+
self.stderr = ""
|
|
3533
3537
|
|
|
3534
3538
|
|
|
3535
3539
|
def _workspace_has_recent_activity(
|
|
@@ -4202,13 +4206,15 @@ class ProcessManager:
|
|
|
4202
4206
|
not killed while actively producing output.
|
|
4203
4207
|
- on_chunk(lines) is called with each decoded line so the caller can
|
|
4204
4208
|
forward to the progress reporter without waiting for completion.
|
|
4205
|
-
- Idle timeout: if the agent produces no stdout for
|
|
4206
|
-
seconds the code checks for filesystem activity in
|
|
4209
|
+
- Idle timeout: if the agent produces no stdout or stderr bytes for
|
|
4210
|
+
AGENT_IDLE_TIMEOUT seconds the code checks for filesystem activity in
|
|
4211
|
+
workspace_path
|
|
4207
4212
|
before deciding to kill. If files were recently modified the agent
|
|
4208
4213
|
is doing silent work (npm install, compilation, test runs, etc.) and
|
|
4209
|
-
the idle timer is reset. Only when
|
|
4210
|
-
|
|
4211
|
-
|
|
4214
|
+
the idle timer is reset. Only when output and the filesystem are
|
|
4215
|
+
idle does the process get killed. Reading byte chunks rather than
|
|
4216
|
+
lines is important: JSON and progress streams are not guaranteed to
|
|
4217
|
+
flush a newline while the agent is working.
|
|
4212
4218
|
- Absolute timeout (``timeout`` param): hard ceiling for zombie-process
|
|
4213
4219
|
prevention. Always kills at this boundary (no extension), but logs
|
|
4214
4220
|
filesystem activity status for post-mortem observability.
|
|
@@ -4227,38 +4233,42 @@ class ProcessManager:
|
|
|
4227
4233
|
pass
|
|
4228
4234
|
proc.stdin.close()
|
|
4229
4235
|
|
|
4230
|
-
|
|
4236
|
+
stdout_chunks: list[bytes] = []
|
|
4231
4237
|
stderr_chunks: list[bytes] = []
|
|
4232
4238
|
|
|
4233
4239
|
async def _read_stderr():
|
|
4234
4240
|
if not proc.stderr:
|
|
4235
4241
|
return
|
|
4236
|
-
|
|
4237
|
-
|
|
4238
|
-
|
|
4239
|
-
|
|
4240
|
-
|
|
4241
|
-
|
|
4242
|
-
|
|
4243
|
-
|
|
4244
|
-
|
|
4245
|
-
|
|
4246
|
-
|
|
4247
|
-
|
|
4248
|
-
|
|
4249
|
-
|
|
4250
|
-
if
|
|
4242
|
+
pending_line = ""
|
|
4243
|
+
while True:
|
|
4244
|
+
data = await proc.stderr.read(4096)
|
|
4245
|
+
if not data:
|
|
4246
|
+
break
|
|
4247
|
+
# Stderr often contains startup, authentication, and transport
|
|
4248
|
+
# progress. It is an activity signal even when it is not shown
|
|
4249
|
+
# in the live execution panel.
|
|
4250
|
+
_last_activity_at[0] = time.monotonic()
|
|
4251
|
+
stderr_chunks.append(data)
|
|
4252
|
+
if stream_stderr:
|
|
4253
|
+
pending_line += data.decode(errors="replace")
|
|
4254
|
+
complete, separator, pending_line = pending_line.rpartition("\n")
|
|
4255
|
+
if separator and complete:
|
|
4256
|
+
lines = [line for line in complete.split("\n") if line]
|
|
4257
|
+
if lines and on_chunk:
|
|
4251
4258
|
try:
|
|
4252
|
-
await on_chunk(
|
|
4259
|
+
await on_chunk(lines)
|
|
4253
4260
|
except Exception:
|
|
4254
4261
|
pass
|
|
4255
|
-
|
|
4256
|
-
|
|
4257
|
-
|
|
4262
|
+
if stream_stderr and pending_line and on_chunk:
|
|
4263
|
+
try:
|
|
4264
|
+
await on_chunk([pending_line])
|
|
4265
|
+
except Exception:
|
|
4266
|
+
pass
|
|
4258
4267
|
|
|
4259
4268
|
async def _read_stdout():
|
|
4260
4269
|
if not proc.stdout:
|
|
4261
4270
|
return
|
|
4271
|
+
pending_line = ""
|
|
4262
4272
|
while True:
|
|
4263
4273
|
# ── Timeout checks ────────────────────────────────────────────
|
|
4264
4274
|
now = time.monotonic()
|
|
@@ -4304,7 +4314,7 @@ class ProcessManager:
|
|
|
4304
4314
|
else:
|
|
4305
4315
|
# No stdout AND no filesystem activity → truly hung.
|
|
4306
4316
|
logger.warning(
|
|
4307
|
-
"Task %s agent idle %.0fs — no
|
|
4317
|
+
"Task %s agent idle %.0fs — no output, no "
|
|
4308
4318
|
"filesystem activity; killing hung process",
|
|
4309
4319
|
task_id, idle_elapsed,
|
|
4310
4320
|
)
|
|
@@ -4319,17 +4329,17 @@ class ProcessManager:
|
|
|
4319
4329
|
30.0,
|
|
4320
4330
|
)
|
|
4321
4331
|
|
|
4322
|
-
# ── Read
|
|
4332
|
+
# ── Read available bytes with a bounded wait ───────────────────
|
|
4323
4333
|
try:
|
|
4324
|
-
|
|
4325
|
-
proc.stdout.
|
|
4334
|
+
data = await asyncio.wait_for(
|
|
4335
|
+
proc.stdout.read(4096), timeout=check_interval
|
|
4326
4336
|
)
|
|
4327
4337
|
except asyncio.TimeoutError:
|
|
4328
|
-
#
|
|
4329
|
-
#
|
|
4338
|
+
# No new output yet. Loop back to re-evaluate idle/absolute
|
|
4339
|
+
# conditions.
|
|
4330
4340
|
continue
|
|
4331
4341
|
except (ValueError, asyncio.LimitOverrunError, Exception) as exc:
|
|
4332
|
-
#
|
|
4342
|
+
# Stream read error — drain remaining bulk.
|
|
4333
4343
|
logger.warning(
|
|
4334
4344
|
"Stream read error for task %s (%s: %s), draining remaining output",
|
|
4335
4345
|
task_id, type(exc).__name__, exc,
|
|
@@ -4339,28 +4349,27 @@ class ProcessManager:
|
|
|
4339
4349
|
except Exception:
|
|
4340
4350
|
remaining = b""
|
|
4341
4351
|
if remaining:
|
|
4342
|
-
|
|
4343
|
-
if chunk_line:
|
|
4344
|
-
stdout_lines.append(chunk_line)
|
|
4345
|
-
if on_chunk:
|
|
4346
|
-
try:
|
|
4347
|
-
await on_chunk([chunk_line])
|
|
4348
|
-
except Exception:
|
|
4349
|
-
pass
|
|
4352
|
+
stdout_chunks.append(remaining)
|
|
4350
4353
|
break
|
|
4351
4354
|
|
|
4352
|
-
if not
|
|
4355
|
+
if not data:
|
|
4353
4356
|
break # EOF — process exited normally
|
|
4354
4357
|
|
|
4355
4358
|
# ── New output received — reset idle timer ────────────────────
|
|
4356
4359
|
_last_activity_at[0] = time.monotonic()
|
|
4357
|
-
|
|
4358
|
-
|
|
4359
|
-
|
|
4360
|
+
stdout_chunks.append(data)
|
|
4361
|
+
pending_line += data.decode(errors="replace")
|
|
4362
|
+
complete, separator, pending_line = pending_line.rpartition("\n")
|
|
4363
|
+
if separator and complete and on_chunk:
|
|
4360
4364
|
try:
|
|
4361
|
-
await on_chunk([line])
|
|
4365
|
+
await on_chunk([line for line in complete.split("\n") if line])
|
|
4362
4366
|
except Exception:
|
|
4363
4367
|
pass # never let on_chunk crash the agent run
|
|
4368
|
+
if pending_line and on_chunk:
|
|
4369
|
+
try:
|
|
4370
|
+
await on_chunk([pending_line])
|
|
4371
|
+
except Exception:
|
|
4372
|
+
pass
|
|
4364
4373
|
|
|
4365
4374
|
try:
|
|
4366
4375
|
# Outer wait_for uses timeout+idle_timeout as generous safety net.
|
|
@@ -4376,18 +4385,23 @@ class ProcessManager:
|
|
|
4376
4385
|
_kill_proc(proc)
|
|
4377
4386
|
# Drain any remaining output after kill
|
|
4378
4387
|
try:
|
|
4379
|
-
remaining,
|
|
4388
|
+
remaining, remaining_stderr = await asyncio.wait_for(proc.communicate(), timeout=5)
|
|
4380
4389
|
if remaining:
|
|
4381
|
-
|
|
4382
|
-
|
|
4383
|
-
|
|
4390
|
+
stdout_chunks.append(remaining)
|
|
4391
|
+
if remaining_stderr:
|
|
4392
|
+
stderr_chunks.append(remaining_stderr)
|
|
4384
4393
|
except Exception:
|
|
4385
4394
|
pass
|
|
4395
|
+
if isinstance(_exc, _IdleTimeoutError):
|
|
4396
|
+
_exc.stdout = b"".join(stdout_chunks).decode(errors="replace")
|
|
4397
|
+
_exc.stderr = b"".join(stderr_chunks).decode(errors="replace")
|
|
4386
4398
|
raise # re-raise (_IdleTimeoutError preserves subclass type)
|
|
4387
4399
|
|
|
4388
4400
|
await proc.wait()
|
|
4389
|
-
stdout = "
|
|
4390
|
-
stderr =
|
|
4401
|
+
stdout = b"".join(stdout_chunks).decode(errors="replace")
|
|
4402
|
+
stderr = b"".join(stderr_chunks).decode(errors="replace")
|
|
4403
|
+
if stream_stderr and stderr:
|
|
4404
|
+
stdout = "\n".join(part for part in (stdout, stderr) if part)
|
|
4391
4405
|
return stdout, stderr, proc.returncode or 0
|
|
4392
4406
|
|
|
4393
4407
|
async def _run_claude(
|
|
@@ -4515,10 +4529,17 @@ class ProcessManager:
|
|
|
4515
4529
|
except asyncio.TimeoutError as exc:
|
|
4516
4530
|
_kill_proc(self.active_processes.pop(task_id, None) or proc)
|
|
4517
4531
|
_err = (
|
|
4518
|
-
f"Agent
|
|
4519
|
-
"
|
|
4532
|
+
f"Agent produced no stdout, stderr, or workspace changes for "
|
|
4533
|
+
f"{exc.idle_seconds:.0f}s — process terminated. Inspect retained "
|
|
4534
|
+
"output and the daemon log for the blocked operation."
|
|
4520
4535
|
) if isinstance(exc, _IdleTimeoutError) else f"Timed out after {timeout}s (absolute limit)"
|
|
4521
|
-
return TaskResult(
|
|
4536
|
+
return TaskResult(
|
|
4537
|
+
status="failed",
|
|
4538
|
+
exit_code=-1,
|
|
4539
|
+
stdout=getattr(exc, "stdout", "")[-settings.AGENT_MAX_OUTPUT_SIZE:],
|
|
4540
|
+
stderr=getattr(exc, "stderr", "")[-10000:],
|
|
4541
|
+
error=_err,
|
|
4542
|
+
)
|
|
4522
4543
|
except Exception as exc:
|
|
4523
4544
|
logger.exception("Claude stream error for task %s", task_id)
|
|
4524
4545
|
if task_id in self.active_processes:
|
|
@@ -5314,10 +5335,17 @@ class ProcessManager:
|
|
|
5314
5335
|
except asyncio.TimeoutError as exc:
|
|
5315
5336
|
_kill_proc(self.active_processes.pop(task_id, None) or proc)
|
|
5316
5337
|
_err = (
|
|
5317
|
-
f"Agent
|
|
5318
|
-
"
|
|
5338
|
+
f"Agent produced no stdout, stderr, or workspace changes for "
|
|
5339
|
+
f"{exc.idle_seconds:.0f}s — process terminated. Inspect retained "
|
|
5340
|
+
"output and the daemon log for the blocked operation."
|
|
5319
5341
|
) if isinstance(exc, _IdleTimeoutError) else f"Timed out after {timeout}s (absolute limit)"
|
|
5320
|
-
return TaskResult(
|
|
5342
|
+
return TaskResult(
|
|
5343
|
+
status="failed",
|
|
5344
|
+
exit_code=-1,
|
|
5345
|
+
stdout=getattr(exc, "stdout", "")[-settings.AGENT_MAX_OUTPUT_SIZE:],
|
|
5346
|
+
stderr=getattr(exc, "stderr", "")[-10000:],
|
|
5347
|
+
error=_err,
|
|
5348
|
+
)
|
|
5321
5349
|
except Exception as exc:
|
|
5322
5350
|
logger.exception("Copilot stream error for task %s", task_id)
|
|
5323
5351
|
if task_id in self.active_processes:
|
|
@@ -5383,10 +5411,17 @@ class ProcessManager:
|
|
|
5383
5411
|
except asyncio.TimeoutError as exc:
|
|
5384
5412
|
_kill_proc(self.active_processes.pop(task_id, None) or proc)
|
|
5385
5413
|
_err = (
|
|
5386
|
-
f"Agent
|
|
5387
|
-
"
|
|
5414
|
+
f"Agent produced no stdout, stderr, or workspace changes for "
|
|
5415
|
+
f"{exc.idle_seconds:.0f}s — process terminated. Inspect retained "
|
|
5416
|
+
"output and the daemon log for the blocked operation."
|
|
5388
5417
|
) if isinstance(exc, _IdleTimeoutError) else f"Timed out after {timeout}s (absolute limit)"
|
|
5389
|
-
return TaskResult(
|
|
5418
|
+
return TaskResult(
|
|
5419
|
+
status="failed",
|
|
5420
|
+
exit_code=-1,
|
|
5421
|
+
stdout=getattr(exc, "stdout", "")[-settings.AGENT_MAX_OUTPUT_SIZE:],
|
|
5422
|
+
stderr=getattr(exc, "stderr", "")[-10000:],
|
|
5423
|
+
error=_err,
|
|
5424
|
+
)
|
|
5390
5425
|
except Exception as exc:
|
|
5391
5426
|
logger.exception("CLI stream error for task %s", task_id)
|
|
5392
5427
|
if task_id in self.active_processes:
|
|
@@ -5460,6 +5495,13 @@ class ProcessManager:
|
|
|
5460
5495
|
pass
|
|
5461
5496
|
if data.get("duration_ms") and "duration_seconds" not in metrics:
|
|
5462
5497
|
metrics["duration_seconds"] = data["duration_ms"] / 1000.0
|
|
5498
|
+
# Turn usage (Claude CLI result event) — baseline telemetry for
|
|
5499
|
+
# node-type turn budgeting (see test-script quality audit M0).
|
|
5500
|
+
if data.get("num_turns") and "num_turns" not in metrics:
|
|
5501
|
+
try:
|
|
5502
|
+
metrics["num_turns"] = int(data["num_turns"])
|
|
5503
|
+
except (TypeError, ValueError):
|
|
5504
|
+
pass
|
|
5463
5505
|
|
|
5464
5506
|
# ── 3. Prefer "result" type line — most authoritative ──
|
|
5465
5507
|
for data in reversed(parsed):
|
|
@@ -6662,6 +6704,146 @@ class ServerConnection:
|
|
|
6662
6704
|
# ── Main Daemon ──
|
|
6663
6705
|
|
|
6664
6706
|
|
|
6707
|
+
def _categorize_validation_issue(issue: str) -> str:
|
|
6708
|
+
"""Map a free-text validation issue to a stable category for telemetry.
|
|
6709
|
+
|
|
6710
|
+
Baseline data for the test-script quality initiative (M0): lets us rank
|
|
6711
|
+
validation failure modes without parsing free text. Order matters —
|
|
6712
|
+
specific patterns are checked before generic ones.
|
|
6713
|
+
"""
|
|
6714
|
+
text = issue.lower()
|
|
6715
|
+
if "script_content" in text:
|
|
6716
|
+
return "empty_script_content"
|
|
6717
|
+
if "script_path" in text and "does not exist" in text:
|
|
6718
|
+
return "missing_script_file"
|
|
6719
|
+
if "not valid json" in text:
|
|
6720
|
+
return "invalid_json"
|
|
6721
|
+
if "e2e" in text and "test case" in text:
|
|
6722
|
+
return "missing_e2e_case"
|
|
6723
|
+
if "coverage" in text:
|
|
6724
|
+
return "coverage_gap"
|
|
6725
|
+
if "scope" in text and "violation" in text:
|
|
6726
|
+
return "scope_violation"
|
|
6727
|
+
if "missing" in text or "not found" in text:
|
|
6728
|
+
return "missing_file"
|
|
6729
|
+
if "is empty" in text:
|
|
6730
|
+
return "empty_file"
|
|
6731
|
+
return "other"
|
|
6732
|
+
|
|
6733
|
+
|
|
6734
|
+
def _extract_testing_artifacts(workspace_path: Path, output_dir: str) -> dict | None:
|
|
6735
|
+
"""Summarize test-cases.json artifact counts for telemetry (best effort).
|
|
6736
|
+
|
|
6737
|
+
Returns None when the file is missing/unparseable; never raises.
|
|
6738
|
+
"""
|
|
6739
|
+
try:
|
|
6740
|
+
tc_path = workspace_path / output_dir / "test-cases.json"
|
|
6741
|
+
if not output_dir or not tc_path.exists():
|
|
6742
|
+
return None
|
|
6743
|
+
data = json.loads(tc_path.read_text(encoding="utf-8"))
|
|
6744
|
+
cases = data.get("test_cases") or []
|
|
6745
|
+
by_type: dict[str, int] = {}
|
|
6746
|
+
scripted = 0
|
|
6747
|
+
e2e_scripted = 0
|
|
6748
|
+
for case in cases:
|
|
6749
|
+
if not isinstance(case, dict):
|
|
6750
|
+
continue
|
|
6751
|
+
case_type = (case.get("test_type") or "unknown").lower()
|
|
6752
|
+
by_type[case_type] = by_type.get(case_type, 0) + 1
|
|
6753
|
+
if (case.get("script_content") or "").strip():
|
|
6754
|
+
scripted += 1
|
|
6755
|
+
if case_type == "e2e":
|
|
6756
|
+
e2e_scripted += 1
|
|
6757
|
+
return {
|
|
6758
|
+
"total_cases": len(cases),
|
|
6759
|
+
"by_type": by_type,
|
|
6760
|
+
"scripted_cases": scripted,
|
|
6761
|
+
"e2e_scripted": e2e_scripted,
|
|
6762
|
+
}
|
|
6763
|
+
except Exception:
|
|
6764
|
+
logger.debug("testing artifact telemetry extraction failed", exc_info=True)
|
|
6765
|
+
return None
|
|
6766
|
+
|
|
6767
|
+
|
|
6768
|
+
def _validate_test_evidence(
|
|
6769
|
+
base: Path, workspace_path: Path, e2e_case_ids: list[str], max_anchor_checks: int = 50
|
|
6770
|
+
) -> list[str]:
|
|
6771
|
+
"""Validate the M1 evidence bundle (test-evidence.json) for e2e test cases.
|
|
6772
|
+
|
|
6773
|
+
Deterministic, zero-environment checks: file exists and parses, locators/
|
|
6774
|
+
routes are present, every e2e case has a case_evidence entry, and each
|
|
6775
|
+
evidence anchor is a literal string that actually appears in its declared
|
|
6776
|
+
source_file (rejects invented selectors/routes). Items explicitly marked
|
|
6777
|
+
runtime_evidence_required are exempt from anchor checks by design.
|
|
6778
|
+
"""
|
|
6779
|
+
issues: list[str] = []
|
|
6780
|
+
ev_path = base / "test-evidence.json"
|
|
6781
|
+
if not ev_path.exists():
|
|
6782
|
+
return [
|
|
6783
|
+
"test-evidence.json not found. It is MANDATORY when test-intent.json "
|
|
6784
|
+
"has e2e intents: record every route/selector used by E2E scripts with "
|
|
6785
|
+
"source_file + anchor (a literal string copied from that file), plus "
|
|
6786
|
+
"case_evidence mapping each e2e test case to its route/locator refs."
|
|
6787
|
+
]
|
|
6788
|
+
try:
|
|
6789
|
+
ev = json.loads(ev_path.read_text(encoding="utf-8"))
|
|
6790
|
+
except (json.JSONDecodeError, UnicodeDecodeError) as e:
|
|
6791
|
+
return [f"test-evidence.json is not valid JSON: {e}"]
|
|
6792
|
+
|
|
6793
|
+
locators = ev.get("locators") or []
|
|
6794
|
+
routes = ev.get("routes") or []
|
|
6795
|
+
if not locators and not routes:
|
|
6796
|
+
issues.append(
|
|
6797
|
+
"test-evidence.json has no 'locators' or 'routes' entries — E2E scripts "
|
|
6798
|
+
"must ground their selectors/routes in source-file evidence."
|
|
6799
|
+
)
|
|
6800
|
+
|
|
6801
|
+
case_evidence = ev.get("case_evidence") or []
|
|
6802
|
+
covered_ids = {
|
|
6803
|
+
str(entry.get("test_case_id"))
|
|
6804
|
+
for entry in case_evidence
|
|
6805
|
+
if isinstance(entry, dict) and entry.get("test_case_id")
|
|
6806
|
+
}
|
|
6807
|
+
missing_cases = [cid for cid in e2e_case_ids if cid not in covered_ids]
|
|
6808
|
+
if missing_cases:
|
|
6809
|
+
issues.append(
|
|
6810
|
+
f"test-evidence.json missing case_evidence for e2e test cases: "
|
|
6811
|
+
f"{missing_cases[:10]}. Each e2e test case needs a case_evidence entry "
|
|
6812
|
+
f"with route_refs/locator_refs."
|
|
6813
|
+
)
|
|
6814
|
+
|
|
6815
|
+
checked = 0
|
|
6816
|
+
for item in list(locators) + list(routes):
|
|
6817
|
+
if checked >= max_anchor_checks:
|
|
6818
|
+
break
|
|
6819
|
+
if not isinstance(item, dict) or item.get("runtime_evidence_required"):
|
|
6820
|
+
continue
|
|
6821
|
+
checked += 1
|
|
6822
|
+
label = item.get("selector") or item.get("path") or item.get("id") or "?"
|
|
6823
|
+
src = (item.get("source_file") or "").strip()
|
|
6824
|
+
anchor = (item.get("anchor") or "").strip()
|
|
6825
|
+
if not src or not anchor:
|
|
6826
|
+
issues.append(
|
|
6827
|
+
f"evidence item '{label}' missing 'source_file' or 'anchor' — "
|
|
6828
|
+
f"anchor must be a literal string copied from source_file."
|
|
6829
|
+
)
|
|
6830
|
+
continue
|
|
6831
|
+
fpath = workspace_path / src
|
|
6832
|
+
if not fpath.exists():
|
|
6833
|
+
issues.append(f"evidence source_file not found: '{src}' (for '{label}')")
|
|
6834
|
+
continue
|
|
6835
|
+
try:
|
|
6836
|
+
content = fpath.read_text(encoding="utf-8", errors="replace")
|
|
6837
|
+
except OSError:
|
|
6838
|
+
continue
|
|
6839
|
+
if anchor not in content:
|
|
6840
|
+
issues.append(
|
|
6841
|
+
f"evidence anchor not found in '{src}': '{anchor[:80]}' (for '{label}'). "
|
|
6842
|
+
f"Copy the exact literal from the source file — do not paraphrase."
|
|
6843
|
+
)
|
|
6844
|
+
return issues
|
|
6845
|
+
|
|
6846
|
+
|
|
6665
6847
|
class RuntimeDaemon:
|
|
6666
6848
|
"""Main Daemon orchestrator — runs on the host machine.
|
|
6667
6849
|
|
|
@@ -8422,6 +8604,7 @@ class RuntimeDaemon:
|
|
|
8422
8604
|
# Not gated by _requires_structured_artifacts so that bugfix with e2e
|
|
8423
8605
|
# intents gets the same E2E chain validation as feature/improvement.
|
|
8424
8606
|
_e2e_intent_ids: set[str] = set()
|
|
8607
|
+
_e2e_case_ids: list[str] = []
|
|
8425
8608
|
_analysis_dir = (
|
|
8426
8609
|
_input.get("analysis_output_dir")
|
|
8427
8610
|
or (_input.get("context") or {}).get("analysis_output_dir")
|
|
@@ -8458,6 +8641,13 @@ class RuntimeDaemon:
|
|
|
8458
8641
|
try:
|
|
8459
8642
|
tc_data = _json.loads(tc_path.read_text(encoding="utf-8"))
|
|
8460
8643
|
cases = tc_data.get("test_cases", [])
|
|
8644
|
+
_e2e_case_ids = [
|
|
8645
|
+
str(_tc.get("id"))
|
|
8646
|
+
for _tc in cases
|
|
8647
|
+
if isinstance(_tc, dict)
|
|
8648
|
+
and (_tc.get("test_type") or "").lower() == "e2e"
|
|
8649
|
+
and _tc.get("id")
|
|
8650
|
+
]
|
|
8461
8651
|
if not cases:
|
|
8462
8652
|
issues.append("test-cases.json exists but contains no test cases")
|
|
8463
8653
|
else:
|
|
@@ -8545,6 +8735,15 @@ class RuntimeDaemon:
|
|
|
8545
8735
|
elif _requires_structured_artifacts or _requires_e2e_artifacts:
|
|
8546
8736
|
issues.append(f"coverage-matrix.json not found in {doc_dir or 'workspace root'}")
|
|
8547
8737
|
|
|
8738
|
+
# --- test-evidence.json validation (M1 evidence bundle) ---
|
|
8739
|
+
# Required whenever e2e intents exist: every route/selector
|
|
8740
|
+
# used by E2E scripts must be grounded in source-file evidence
|
|
8741
|
+
# (anchor literals are verified against the declared files).
|
|
8742
|
+
if _requires_e2e_artifacts:
|
|
8743
|
+
issues.extend(
|
|
8744
|
+
_validate_test_evidence(base, workspace_path, _e2e_case_ids)
|
|
8745
|
+
)
|
|
8746
|
+
|
|
8548
8747
|
# --- test-report.md validation ---
|
|
8549
8748
|
report_path = base / "test-report.md"
|
|
8550
8749
|
if not report_path.exists():
|
|
@@ -8862,6 +9061,9 @@ class RuntimeDaemon:
|
|
|
8862
9061
|
)
|
|
8863
9062
|
issues = self._validate_outputs(workspace_path, task, result)
|
|
8864
9063
|
if not issues:
|
|
9064
|
+
self._record_testing_telemetry(
|
|
9065
|
+
workspace_path, task, result, retries_used=attempt
|
|
9066
|
+
)
|
|
8865
9067
|
return result # All validations passed
|
|
8866
9068
|
|
|
8867
9069
|
issues_text = "\n".join(f"- {iss}" for iss in issues)
|
|
@@ -8996,8 +9198,38 @@ class RuntimeDaemon:
|
|
|
8996
9198
|
len(remaining), max_retries, task.task_id,
|
|
8997
9199
|
)
|
|
8998
9200
|
result.metrics["validation_issues"] = remaining
|
|
9201
|
+
self._record_testing_telemetry(workspace_path, task, result, retries_used=max_retries)
|
|
8999
9202
|
return result
|
|
9000
9203
|
|
|
9204
|
+
def _record_testing_telemetry(
|
|
9205
|
+
self, workspace_path: Path, task: TaskInfo, result: TaskResult, retries_used: int
|
|
9206
|
+
) -> None:
|
|
9207
|
+
"""Attach validation/testing telemetry to result.metrics (best effort).
|
|
9208
|
+
|
|
9209
|
+
M0 baseline for the test-script quality initiative: validation retry
|
|
9210
|
+
usage, stable failure categories, and testing artifact counts. These
|
|
9211
|
+
travel with the completion payload (metrics are kept in full by the
|
|
9212
|
+
backend) so baselines can be built without parsing free text.
|
|
9213
|
+
Never raises.
|
|
9214
|
+
"""
|
|
9215
|
+
try:
|
|
9216
|
+
result.metrics["validation_retries_used"] = retries_used
|
|
9217
|
+
issues = result.metrics.get("validation_issues") or []
|
|
9218
|
+
if issues:
|
|
9219
|
+
result.metrics["validation_issue_categories"] = sorted(
|
|
9220
|
+
{_categorize_validation_issue(str(issue)) for issue in issues}
|
|
9221
|
+
)
|
|
9222
|
+
if task.node_type != "testing":
|
|
9223
|
+
return
|
|
9224
|
+
input_data = task.input_data or {}
|
|
9225
|
+
context = input_data.get("context") or {}
|
|
9226
|
+
output_dir = input_data.get("output_dir") or context.get("output_dir") or ""
|
|
9227
|
+
artifacts = _extract_testing_artifacts(workspace_path, output_dir)
|
|
9228
|
+
if artifacts:
|
|
9229
|
+
result.metrics["testing_artifacts"] = artifacts
|
|
9230
|
+
except Exception:
|
|
9231
|
+
logger.debug("testing telemetry recording failed", exc_info=True)
|
|
9232
|
+
|
|
9001
9233
|
async def _collect_analysis_artifacts(
|
|
9002
9234
|
self, workspace_path: Path, task: TaskInfo, result: TaskResult
|
|
9003
9235
|
) -> None:
|
|
@@ -9213,7 +9445,19 @@ class RuntimeDaemon:
|
|
|
9213
9445
|
workspace_path: Path,
|
|
9214
9446
|
*diff_args: str,
|
|
9215
9447
|
) -> str | None:
|
|
9216
|
-
"""
|
|
9448
|
+
"""Detect Git changes that alter only a file's mode, not its content.
|
|
9449
|
+
|
|
9450
|
+
Every Forgexa-managed workspace is configured with
|
|
9451
|
+
``core.filemode=false`` (see ``_prepare_workspace``), which makes Git
|
|
9452
|
+
ignore on-disk executable-bit drift entirely for ``git status``/
|
|
9453
|
+
``git diff``/``git add`` — incidental permission noise from
|
|
9454
|
+
checkouts, worktree creation, or file-rewrite tools can never reach
|
|
9455
|
+
the diff/index we inspect here. The ONLY way a mode-only change can
|
|
9456
|
+
appear is a deliberate ``git update-index --chmod=...`` (e.g. an
|
|
9457
|
+
agent restoring a shell script's executable bit that a prior commit
|
|
9458
|
+
lost). That is legitimate, intentional work — not noise — so it
|
|
9459
|
+
must be allowed to commit; we only log it for audit visibility.
|
|
9460
|
+
"""
|
|
9217
9461
|
proc = await asyncio.create_subprocess_exec(
|
|
9218
9462
|
"git", "diff", "--raw", "--no-abbrev", "--no-renames", "-z", *diff_args,
|
|
9219
9463
|
cwd=str(workspace_path),
|
|
@@ -9240,17 +9484,18 @@ class RuntimeDaemon:
|
|
|
9240
9484
|
if old_object == new_object and old_mode != new_mode:
|
|
9241
9485
|
mode_only_paths.append(path)
|
|
9242
9486
|
|
|
9243
|
-
if
|
|
9244
|
-
|
|
9487
|
+
if mode_only_paths:
|
|
9488
|
+
sample = ", ".join(mode_only_paths[:5])
|
|
9489
|
+
remainder = "" if len(mode_only_paths) <= 5 else ", ..."
|
|
9490
|
+
logger.info(
|
|
9491
|
+
"Auto-commit includes %d permission-only file change(s) with "
|
|
9492
|
+
"unchanged content (%s%s) — allowing (core.filemode=false means "
|
|
9493
|
+
"this can only come from a deliberate executable-bit fix, not "
|
|
9494
|
+
"incidental noise).",
|
|
9495
|
+
len(mode_only_paths), sample, remainder,
|
|
9496
|
+
)
|
|
9245
9497
|
|
|
9246
|
-
|
|
9247
|
-
remainder = "" if len(mode_only_paths) <= 5 else ", ..."
|
|
9248
|
-
return (
|
|
9249
|
-
"Auto-commit blocked: detected "
|
|
9250
|
-
f"{len(mode_only_paths)} permission-only file change(s) with unchanged content "
|
|
9251
|
-
f"({sample}{remainder}). Restore the original modes before retrying; "
|
|
9252
|
-
"Forgexa does not auto-commit permission-only changes."
|
|
9253
|
-
)
|
|
9498
|
+
return None
|
|
9254
9499
|
|
|
9255
9500
|
async def _auto_commit(
|
|
9256
9501
|
self,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|