quantlab-factor-agent 0.1.0__py3-none-any.whl

factor_agent/web/_round.py

@@ -0,0 +1,219 @@
+"""The round in progress, replayed from the events since the run was last opened, and one LLM call read in full.
+
+A round a resume dropped left its events before the resume, so it is not
+among them; a run without events has no round in progress to show. An LLM
+call is read by the number of its call file, from the file itself: tokens,
+truncation and the model's reasoning are the provider's reply's; what it
+concerned (round, slots) only the events file says.
+"""
+
+import json
+from dataclasses import dataclass
+from datetime import datetime, timedelta
+from pathlib import Path
+
+from factor_agent import events
+from factor_agent.llm._raw import raw_facts
+from factor_agent.llm.base import Reply
+from factor_agent.web._files import Replayed, loads, utc
+from factor_agent.web._health import Tokens
+
+
+@dataclass(frozen=True)
+class Check:
+    """One check of a pending trial's candidate: the parser and the Gate on one version of it, or the reviewer.
+
+    ``by`` is ``"gate"`` or ``"reviewer"``. For the Gate, ``step`` is the
+    version's: ``"proposed"`` (the first), ``"fixed"`` (sent back by the
+    parser or the Gate) or ``"rewritten"`` (sent back by the reviewer), and
+    ``stage`` is ``"parse"`` or ``"gate"`` when it refused the version,
+    ``None`` when it passed it. ``passed`` is true when the Gate passed the
+    version or the reviewer found it consistent. ``reasons`` are the Gate's
+    reasons for a refusal, or the reviewer's reason for its verdict either
+    way; ``hints`` are the Gate's.
+    """
+
+    by: str
+    step: str | None
+    passed: bool
+    stage: str | None
+    reasons: tuple[str, ...]
+    hints: tuple[str, ...]
+    time: datetime
+
+
+@dataclass(frozen=True)
+class PendingTrial:
+    """A pending trial: what is known so far of one slot of the round in progress.
+
+    ``stage`` is the furthest it went: ``"planned"`` (its research direction
+    is chosen), ``"proposed"``, ``"fixed"`` (after the Gate refused it),
+    ``"rewritten"`` (after the reviewer found it inconsistent), and
+    ``"consistent"`` or ``"inconsistent"`` as the reviewer last found it.
+    ``direction`` is its research direction as the plan gives it. ``latest``
+    is its latest version as the proposer wrote it, ``None`` before the
+    first. ``checks`` are the Gate's outcomes and the reviewer's verdicts
+    so far, the earliest first. It becomes a trial when its round is written.
+    """
+
+    slot: int
+    stage: str
+    direction: dict
+    latest: object
+    checks: tuple[Check, ...]
+
+
+@dataclass(frozen=True)
+class RoundCall:
+    """An LLM call of the round in progress, as the call timeline shows it.
+
+    ``number`` is its call file's number, by which ``read_call`` reads it;
+    ``None`` while it has not come back, or when it failed (``error`` says
+    how). ``group`` is its proposal group and ``slots`` the slots it
+    concerns; both are empty for the planner, whose call concerns the round.
+    ``sent`` and ``returned`` are when it was sent and came back (UTC).
+    """
+
+    number: int | None
+    role: str
+    model: str
+    group: int | None
+    slots: tuple[int, ...]
+    sent: datetime
+    returned: datetime | None
+    error: str | None
+
+
+@dataclass(frozen=True)
+class RunningRound:
+    """The round in progress, as its events tell since the run was last opened.
+
+    ``started`` is when the round before it was written, or the run was
+    opened when it is the first since. ``calls`` are its LLM calls in the
+    order they were sent, and ``evaluation`` when the Evaluator started on
+    it (``None`` before). ``pending`` holds a pending trial per slot of its
+    plan, in slot order; it is empty until the plan is made.
+    """
+
+    round: int
+    started: datetime
+    calls: tuple[RoundCall, ...]
+    evaluation: datetime | None
+    pending: tuple[PendingTrial, ...]
+
+
+@dataclass(frozen=True)
+class LLMCall:
+    """One LLM call of a run, in full, as its call file holds it.
+
+    ``number`` is the number of its call file. ``round``, ``group`` and
+    ``slots`` are the round, proposal group and slots it concerned, as the
+    events file says (``None`` and empty without one). ``time`` is when it
+    came back (UTC) and ``seconds`` how long it took (``None`` without an
+    events file). ``tokens``, ``truncated`` (cut off at the output limit,
+    ``None`` when the provider's reply does not say) and ``empty`` are read
+    from the provider's reply. ``reasoning`` is the model's reasoning when
+    the provider returns it, ``None`` otherwise.
+    """
+
+    number: int
+    role: str
+    model: str
+    round: int | None
+    group: int | None
+    slots: tuple[int, ...]
+    time: datetime
+    seconds: float | None
+    tokens: Tokens
+    truncated: bool | None
+    empty: bool
+    prompt: str
+    reply: str
+    reasoning: str | None
+
+
+def running(number: int, replayed: Replayed) -> RunningRound | None:
+    """Round ``number``, in progress, from the events since the run was last opened; ``None`` without events."""
+    if replayed.opened is None:
+        return None
+    started = replayed.opened.time
+    calls: dict[int, RoundCall] = {}
+    evaluation = None
+    directions: list[dict] = []
+    stages: dict[int, str] = {}
+    latest: dict[int, object] = {}
+    checks: dict[int, list[Check]] = {}
+    for event in replayed.current:
+        if isinstance(event, events.RoundEnd):
+            started = event.time
+        if getattr(event, "round", None) != number:
+            continue
+        if isinstance(event, events.CallEnd):
+            sent = calls[event.call].sent if event.call in calls else event.time - timedelta(seconds=event.seconds)
+            calls[event.call] = RoundCall(
+                number=_call_number(event.file), role=event.role, model=event.model, group=event.group,
+                slots=tuple(event.slots), sent=sent, returned=event.time, error=event.error,
+            )
+        elif isinstance(event, events.CallStart):
+            calls[event.call] = RoundCall(number=None, role=event.role, model=event.model, group=event.group,
+                                          slots=tuple(event.slots), sent=event.time, returned=None, error=None)
+        elif isinstance(event, events.EvaluationStart):
+            evaluation = event.time
+        elif isinstance(event, events.Plan):
+            directions = event.directions
+            stages = dict.fromkeys(range(len(directions)), "planned")
+        elif isinstance(event, events.GateOutcome):
+            stages[event.slot], latest[event.slot] = event.step, event.item
+            checks.setdefault(event.slot, []).append(Check(
+                by="gate", step=event.step, passed=event.passed, stage=event.stage, reasons=tuple(event.reasons),
+                hints=tuple(event.hints), time=event.time,
+            ))
+        elif isinstance(event, events.ReviewVerdict):
+            stages[event.slot] = "consistent" if event.consistent else "inconsistent"
+            checks.setdefault(event.slot, []).append(Check(
+                by="reviewer", step=None, passed=event.consistent, stage=None, reasons=(event.reason,), hints=(),
+                time=event.time,
+            ))
+    pending = tuple(
+        PendingTrial(slot=slot, stage=stages[slot], direction=direction, latest=latest.get(slot),
+                     checks=tuple(checks.get(slot, ())))
+        for slot, direction in enumerate(directions)
+    )
+    return RunningRound(round=number, started=started, calls=tuple(calls.values()), evaluation=evaluation,
+                        pending=pending)
+
+
+def _call_number(file: str | None) -> int | None:
+    """The number of the call file named ``file``, as ``<number>-<role>.json``."""
+    return int(file.partition("-")[0]) if file else None
+
+
+def read_call(run_dir: str | Path, number: int) -> LLMCall | None:
+    """The LLM call of the run in ``run_dir`` whose call file has ``number``, ``None`` when it has none.
+
+    A call of a round a resume dropped is not read: its file is no longer
+    in the run's sequence of calls.
+    """
+    run_dir = Path(run_dir)
+    files = list((run_dir / "llm_log").glob(f"{number:06d}-*.json")) if number > 0 else []
+    if not files:
+        return None
+    path = files[0]
+    try:
+        call = loads(path.read_text())
+    except (FileNotFoundError, json.JSONDecodeError):
+        return None
+    request, reply = call.get("request") or {}, call.get("reply") or {}
+    text, raw = reply.get("text") or "", reply.get("raw") or ""
+    facts = events.reply_facts(Reply(text=text, usage=reply.get("usage") or {}, model_id=reply.get("model_id") or "",
+                                     raw=raw))
+    end = next((e for e in reversed(events.read(run_dir / events.FILE))
+                if isinstance(e, events.CallEnd) and e.file == path.name), None)
+    return LLMCall(
+        number=number, role=request.get("role") or path.stem.partition("-")[2], model=request.get("model") or "",
+        round=end.round if end else None, group=end.group if end else None, slots=tuple(end.slots) if end else (),
+        time=end.time if end else utc(path.stat().st_mtime), seconds=end.seconds if end else None,
+        tokens=Tokens(facts["input_tokens"], facts["output_tokens"], facts["cache_hits"], facts["reasoning"]),
+        truncated=facts["truncated"], empty=facts["empty"], prompt=request.get("prompt") or "", reply=text,
+        reasoning=raw_facts(raw).reasoning,
+    )

factor_agent/web/_trials.py

@@ -0,0 +1,112 @@
+"""The trials page: the evaluated trials on the admission map, and the others grouped by what refused them."""
+
+import re
+from dataclasses import dataclass
+
+from factor_agent.web._files import Replayed
+from factor_agent.web._ledger import Trial, in_direction, monotonicity, thresholds
+
+
+@dataclass(frozen=True)
+class Placed:
+    """An evaluated trial as the admission map places it, with its statistics in its direction.
+
+    ``round`` and ``slot`` name the trial, ``status`` is accepted or rejected.
+    ``factor_direction`` is +1 when a higher value went with a higher training
+    return, -1 otherwise. ``ic_mean``, ``ic_t``, ``partial_ic_mean`` and
+    ``partial_ic_t`` are in that direction; the trial's ``metrics`` keep the
+    raw values. ``largest_correlation`` is the larger absolute correlation
+    with the nearest reference or accepted factor and the nearest known one,
+    ``None`` when there was none to compare with. ``threshold`` is the t
+    threshold of its round, as ``Factor.threshold`` (``None`` while nothing gives it).
+    ``monotonicity`` is its decile monotonicity in its direction, as admission's.
+    """
+
+    round: int
+    slot: int
+    status: str
+    factor_direction: int
+    ic_mean: float | None
+    ic_t: float | None
+    partial_ic_mean: float | None
+    partial_ic_t: float | None
+    largest_correlation: float | None
+    threshold: float | None
+    monotonicity: float | None
+
+
+@dataclass(frozen=True)
+class TrialRef:
+    """A trial named by its round and its slot in the round."""
+
+    round: int
+    slot: int
+
+
+@dataclass(frozen=True)
+class Refusal:
+    """The trials one stage refused for the same reason before the Evaluator scored them.
+
+    ``stage`` is ``"parse"``, ``"gate"``, ``"review"`` or ``"evaluate"`` (code
+    the Evaluator could not build). ``reason`` is the first trial's final
+    reason; the others' may differ in the numbers, reply or code they quote.
+    ``rounds`` are the rounds the trials belong to, the earliest first.
+    """
+
+    stage: str
+    reason: str
+    rounds: tuple[int, ...]
+    trials: tuple[TrialRef, ...]
+
+
+@dataclass(frozen=True)
+class TrialsPage:
+    """The admission map's trials, the latest round's threshold and the trials never evaluated, as ``RunView`` holds them."""
+
+    evaluated: tuple[Placed, ...]
+    threshold: float | None
+    never_evaluated: tuple[Refusal, ...]
+
+
+#: The order of the stages that refuse a trial before the Evaluator scores it.
+_STAGES = ("parse", "gate", "review", "evaluate")
+
+#: What a reason quotes after its message: a reply, code or a value.
+_QUOTED = re.compile(r": ['\"`].*$", re.DOTALL)
+
+#: A number in a reason.
+_NUMBER = re.compile(r"[-+]?\d+(?:\.\d+)?(?:e[-+]?\d+)?")
+
+
+def trials_page(trials: tuple[Trial, ...], replayed: Replayed) -> TrialsPage:
+    """What the trials page shows of ``trials``."""
+    by_round = thresholds(trials, replayed)
+    evaluated = []
+    for trial in trials:
+        if not trial.metrics:
+            continue
+        metrics = trial.metrics
+        sign, oriented = in_direction(metrics)
+        correlations = [abs(c) for c in (metrics.get("nearest_correlation"), metrics.get("nearest_known_correlation"))
+                        if c is not None]
+        evaluated.append(Placed(
+            round=trial.round, slot=trial.slot, status=trial.status, factor_direction=sign,
+            ic_mean=oriented["ic_mean"], ic_t=oriented["ic_t"], partial_ic_mean=oriented["partial_ic_mean"],
+            partial_ic_t=oriented["partial_ic_t"], largest_correlation=max(correlations) if correlations else None,
+            threshold=by_round[trial.round][0],
+            monotonicity=monotonicity(tuple(metrics.get("decile_returns") or ()), sign),
+        ))
+    groups: dict[tuple[str, str], list[Trial]] = {}
+    for trial in trials:
+        if trial.metrics or trial.status == "accepted":
+            continue
+        reason = trial.reasons[0] if trial.reasons else ""
+        groups.setdefault((trial.stage, _NUMBER.sub("#", _QUOTED.sub("", reason))), []).append(trial)
+    never = [Refusal(stage=stage, reason=members[0].reasons[0] if members[0].reasons else "",
+                     rounds=tuple(sorted({t.round for t in members})),
+                     trials=tuple(TrialRef(t.round, t.slot) for t in members))
+             for (stage, _), members in groups.items()]
+    never.sort(key=lambda g: (_STAGES.index(g.stage) if g.stage in _STAGES else len(_STAGES), g.trials[0].round,
+                              g.trials[0].slot))
+    return TrialsPage(evaluated=tuple(evaluated), threshold=by_round[max(by_round)][0] if by_round else None,
+                      never_evaluated=tuple(never))

factor_agent/web/app.py

@@ -0,0 +1,118 @@
+"""The frontend's web service: JSON routes over ``run_view`` and the built page, on localhost only.
+
+``factor-agent-web <folder>`` serves the run directories under a folder on
+``http://127.0.0.1:<port>``. It binds the loopback interface and has no
+option to bind another: the pages show prompts, code and statistics and
+there is no login. On a server, forward the port over SSH.
+
+Routes:
+
+- ``GET /api/runs``: every run under the folder (``list_runs``), the latest written first;
+- ``GET /api/run?name=<name>``: one run (``RunView.read``) by its name in the list, 404 for any other name;
+- ``GET /api/run/call?name=<name>&number=<number>``: one LLM call of a run (``read_call``) by the number
+  of its call file, 404 when the run has no call file of that number;
+- ``GET /api/run/files?name=<name>``: the config of each start or resume of a run and its snapshot's
+  files (``read_files``), read only when the page shows them;
+- ``GET /api/run/ic?name=<name>&round=<round>&slot=<slot>``: a trial's daily IC and partial IC (``read_ic``),
+  404 when it was not evaluated or the run kept none;
+- ``GET /api/run/version?name=<name>``: a text that changes whenever the run writes (``run_version``),
+  which the page polls before reading the run again;
+- ``GET /``: the page, built from ``frontend/`` into ``factor_agent/web/static``.
+
+FastAPI and uvicorn come with the ``web`` extra and are imported only when
+the app is made, so the module imports without them.
+"""
+
+import argparse
+from pathlib import Path
+
+from factor_agent.web.run_view import RunView, find_run, list_runs, read_call, read_files, read_ic, run_version
+
+#: The built page: ``npm run build`` in ``frontend/`` writes it, and it is committed.
+STATIC = Path(__file__).parent / "static"
+
+#: The interface the service binds: the loopback interface only.
+HOST = "127.0.0.1"
+
+
+def create_app(root: str | Path):
+    """The web service over the run directories under ``root``.
+
+    Parameters
+    ----------
+    root : str or Path
+        A folder of run directories, at any depth, or a run directory itself.
+
+    Returns
+    -------
+    fastapi.FastAPI
+
+    Raises
+    ------
+    ImportError
+        When the ``web`` extra is not installed.
+    """
+    from fastapi import FastAPI, HTTPException
+    from fastapi.staticfiles import StaticFiles
+
+    root = Path(root).resolve()
+    app = FastAPI(title="factor-agent", docs_url=None, redoc_url=None, openapi_url=None)
+
+    @app.get("/api/runs")
+    def runs():
+        return list_runs(root)
+
+    def run_dir(name: str) -> Path:
+        # Only a listed name is read: a name never reaches the file system unchecked.
+        found = find_run(root, name)
+        if found is None:
+            raise HTTPException(404, f"no run named {name!r} under {root}")
+        return found
+
+    @app.get("/api/run")
+    def run(name: str):
+        return RunView.read(run_dir(name), name=name)
+
+    @app.get("/api/run/call")
+    def call(name: str, number: int):
+        found = read_call(run_dir(name), number)
+        if found is None:
+            raise HTTPException(404, f"run {name!r} has no LLM call number {number}")
+        return found
+
+    @app.get("/api/run/files")
+    def files(name: str):
+        return read_files(run_dir(name))
+
+    @app.get("/api/run/ic")
+    def ic(name: str, round: int, slot: int):
+        found = read_ic(run_dir(name), round, slot)
+        if found is None:
+            raise HTTPException(404, f"run {name!r} kept no daily IC of round {round}, slot {slot}")
+        return found
+
+    @app.get("/api/run/version")
+    def version(name: str):
+        return run_version(run_dir(name))
+
+    app.mount("/", StaticFiles(directory=STATIC, html=True), name="page")
+    return app
+
+
+def main(argv: list[str] | None = None) -> None:
+    """Serve the run directories under a folder on localhost: the ``factor-agent-web`` console script."""
+    parser = argparse.ArgumentParser(
+        prog="factor-agent-web",
+        description="Serve the run directories under a folder on http://127.0.0.1:<port> (localhost only).",
+    )
+    parser.add_argument("root", type=Path, help="a folder of run directories, or a run directory")
+    parser.add_argument("--port", type=int, default=8000, help="the port to listen on (default: 8000)")
+    args = parser.parse_args(argv)
+    try:
+        import uvicorn
+    except ImportError:
+        parser.exit(1, "factor-agent-web: the frontend needs the web extra: uv sync --extra web\n")
+    if not args.root.is_dir():
+        parser.exit(1, f"factor-agent-web: {args.root} is not a folder\n")
+    print(f"factor-agent-web: serving the runs under {args.root.resolve()} on http://{HOST}:{args.port}", flush=True)
+    uvicorn.run(create_app(args.root), host=HOST, port=args.port)