windhover 0.9.0__tar.gz → 0.11.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {windhover-0.9.0 → windhover-0.11.0}/PKG-INFO +12 -5
- {windhover-0.9.0 → windhover-0.11.0}/README.md +11 -4
- {windhover-0.9.0 → windhover-0.11.0}/pyproject.toml +1 -1
- {windhover-0.9.0 → windhover-0.11.0}/tests/test_smoke.py +130 -0
- {windhover-0.9.0 → windhover-0.11.0}/windhover/demo_graph.py +9 -1
- {windhover-0.9.0 → windhover-0.11.0}/windhover/extract.py +18 -3
- {windhover-0.9.0 → windhover-0.11.0}/windhover/server.py +106 -20
- {windhover-0.9.0 → windhover-0.11.0}/windhover/static/index.html +158 -5
- {windhover-0.9.0 → windhover-0.11.0}/windhover/store.py +7 -6
- {windhover-0.9.0 → windhover-0.11.0}/windhover/tracer.py +83 -9
- {windhover-0.9.0 → windhover-0.11.0}/.github/workflows/ci.yml +0 -0
- {windhover-0.9.0 → windhover-0.11.0}/.gitignore +0 -0
- {windhover-0.9.0 → windhover-0.11.0}/LICENSE +0 -0
- {windhover-0.9.0 → windhover-0.11.0}/SPEC.md +0 -0
- {windhover-0.9.0 → windhover-0.11.0}/docs/graph.png +0 -0
- {windhover-0.9.0 → windhover-0.11.0}/docs/logo.svg +0 -0
- {windhover-0.9.0 → windhover-0.11.0}/docs/runs.png +0 -0
- {windhover-0.9.0 → windhover-0.11.0}/docs/social-preview.png +0 -0
- {windhover-0.9.0 → windhover-0.11.0}/docs/stats.png +0 -0
- {windhover-0.9.0 → windhover-0.11.0}/docs/trace.png +0 -0
- {windhover-0.9.0 → windhover-0.11.0}/tests/__init__.py +0 -0
- {windhover-0.9.0 → windhover-0.11.0}/windhover/__init__.py +0 -0
- {windhover-0.9.0 → windhover-0.11.0}/windhover/config.py +0 -0
- {windhover-0.9.0 → windhover-0.11.0}/windhover/pricing.json +0 -0
- {windhover-0.9.0 → windhover-0.11.0}/windhover/static/icon.svg +0 -0
- {windhover-0.9.0 → windhover-0.11.0}/windhover/static/manifest.json +0 -0
- {windhover-0.9.0 → windhover-0.11.0}/windhover/static/vendor/cytoscape-dagre.js +0 -0
- {windhover-0.9.0 → windhover-0.11.0}/windhover/static/vendor/cytoscape.min.js +0 -0
- {windhover-0.9.0 → windhover-0.11.0}/windhover/static/vendor/dagre.min.js +0 -0
- {windhover-0.9.0 → windhover-0.11.0}/windhover/static/vendor/fonts/fraunces-500.woff2 +0 -0
- {windhover-0.9.0 → windhover-0.11.0}/windhover/static/vendor/fonts/fraunces-600-italic.woff2 +0 -0
- {windhover-0.9.0 → windhover-0.11.0}/windhover/static/vendor/fonts/fraunces-600.woff2 +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: windhover
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.11.0
|
|
4
4
|
Summary: Self-hosted, mobile-friendly observability for LangGraph — traces, run history, cost, and a living graph view.
|
|
5
5
|
Project-URL: Repository, https://github.com/justfeltlikerunning/windhover
|
|
6
6
|
Author: justfeltlikerunning
|
|
@@ -81,8 +81,11 @@ graph.invoke(input, config={
|
|
|
81
81
|
- **Clickable graph** — tap a node for health, latency, wiring, its **source code**, and recent executions with payloads.
|
|
82
82
|
- **Error forensics** — failed runs show the full traceback; the failing node turns red on the
|
|
83
83
|
graph, and the node's source renders with the **throwing line highlighted**.
|
|
84
|
-
- **Human-in-the-loop
|
|
85
|
-
|
|
84
|
+
- **Human-in-the-loop console** — a paused graph shows an amber **interrupted** status with the
|
|
85
|
+
question it's asking; answer it (`Command(resume=…)`), redirect it (`Command(goto=…)`), set
|
|
86
|
+
static breakpoints per run (`interrupt_before`), **edit state** at any checkpoint
|
|
87
|
+
(`update_state`), or **fork** a thread from any historical checkpoint — all from the UI,
|
|
88
|
+
all pure LangGraph primitives.
|
|
86
89
|
- **State evolution** — every trace shows which state keys each node wrote, in order.
|
|
87
90
|
- **X-ray** — graphs with subgraphs get a canvas toggle that expands composite nodes
|
|
88
91
|
(`get_graph(xray=True)`).
|
|
@@ -90,8 +93,12 @@ graph.invoke(input, config={
|
|
|
90
93
|
status/tag/session filters, bookmarks, pagination, CSV/JSON export.
|
|
91
94
|
- **Sessions** — group runs into threads/batches; roll-up tokens, cost, errors.
|
|
92
95
|
- **Scores** — attach numeric evals to runs (API or UI): eval harnesses, LLM-as-judge, human review.
|
|
93
|
-
- **Live tail** — open a running run and watch
|
|
94
|
-
|
|
96
|
+
- **Live tail** — open a running run and watch spans arrive — including **the model typing**
|
|
97
|
+
(streamed tokens flush into the span twice a second); nodes push progress via
|
|
98
|
+
`get_stream_writer()`.
|
|
99
|
+
- **Call configs** — every LLM span records temperature/max-tokens/stream **and the tools the
|
|
100
|
+
model was offered**; conditional-edge branch labels and `add_node(metadata=…)` render on the
|
|
101
|
+
graph and node pane; graphs with a context schema get a runtime-context box on New run.
|
|
95
102
|
- **Custom events** — `dispatch_custom_event("name", {...})` anywhere in your app lands as an
|
|
96
103
|
event marker in the trace, parented to the node that fired it.
|
|
97
104
|
- **Retries + TTFT** — tenacity retries badge the span (`↻2`); streaming LLM calls record
|
|
@@ -62,8 +62,11 @@ graph.invoke(input, config={
|
|
|
62
62
|
- **Clickable graph** — tap a node for health, latency, wiring, its **source code**, and recent executions with payloads.
|
|
63
63
|
- **Error forensics** — failed runs show the full traceback; the failing node turns red on the
|
|
64
64
|
graph, and the node's source renders with the **throwing line highlighted**.
|
|
65
|
-
- **Human-in-the-loop
|
|
66
|
-
|
|
65
|
+
- **Human-in-the-loop console** — a paused graph shows an amber **interrupted** status with the
|
|
66
|
+
question it's asking; answer it (`Command(resume=…)`), redirect it (`Command(goto=…)`), set
|
|
67
|
+
static breakpoints per run (`interrupt_before`), **edit state** at any checkpoint
|
|
68
|
+
(`update_state`), or **fork** a thread from any historical checkpoint — all from the UI,
|
|
69
|
+
all pure LangGraph primitives.
|
|
67
70
|
- **State evolution** — every trace shows which state keys each node wrote, in order.
|
|
68
71
|
- **X-ray** — graphs with subgraphs get a canvas toggle that expands composite nodes
|
|
69
72
|
(`get_graph(xray=True)`).
|
|
@@ -71,8 +74,12 @@ graph.invoke(input, config={
|
|
|
71
74
|
status/tag/session filters, bookmarks, pagination, CSV/JSON export.
|
|
72
75
|
- **Sessions** — group runs into threads/batches; roll-up tokens, cost, errors.
|
|
73
76
|
- **Scores** — attach numeric evals to runs (API or UI): eval harnesses, LLM-as-judge, human review.
|
|
74
|
-
- **Live tail** — open a running run and watch
|
|
75
|
-
|
|
77
|
+
- **Live tail** — open a running run and watch spans arrive — including **the model typing**
|
|
78
|
+
(streamed tokens flush into the span twice a second); nodes push progress via
|
|
79
|
+
`get_stream_writer()`.
|
|
80
|
+
- **Call configs** — every LLM span records temperature/max-tokens/stream **and the tools the
|
|
81
|
+
model was offered**; conditional-edge branch labels and `add_node(metadata=…)` render on the
|
|
82
|
+
graph and node pane; graphs with a context schema get a runtime-context box on New run.
|
|
76
83
|
- **Custom events** — `dispatch_custom_event("name", {...})` anywhere in your app lands as an
|
|
77
84
|
event marker in the trace, parented to the node that fired it.
|
|
78
85
|
- **Retries + TTFT** — tenacity retries badge the span (`↻2`); streaming LLM calls record
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "windhover"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.11.0"
|
|
8
8
|
description = "Self-hosted, mobile-friendly observability for LangGraph — traces, run history, cost, and a living graph view."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.10"
|
|
@@ -360,6 +360,132 @@ def test_demo_memory_and_events_end_to_end():
|
|
|
360
360
|
print("demo memory + custom event end-to-end OK")
|
|
361
361
|
|
|
362
362
|
|
|
363
|
+
def test_hitl_interrupt_resume():
|
|
364
|
+
"""Dynamic interrupt pauses (status interrupted); Command(resume) finishes."""
|
|
365
|
+
from typing import TypedDict
|
|
366
|
+
from langgraph.graph import StateGraph, START, END
|
|
367
|
+
from langgraph.checkpoint.memory import MemorySaver
|
|
368
|
+
from langgraph.types import Command, interrupt
|
|
369
|
+
p = tempfile.mktemp(suffix=".db")
|
|
370
|
+
s = Store(p)
|
|
371
|
+
|
|
372
|
+
class St(TypedDict):
|
|
373
|
+
x: int
|
|
374
|
+
def gate(st):
|
|
375
|
+
ok = interrupt({"question": f"allow x={st['x']}?"})
|
|
376
|
+
return {"x": st["x"] if ok else 0}
|
|
377
|
+
g = StateGraph(St)
|
|
378
|
+
g.add_node("gate", gate)
|
|
379
|
+
g.add_edge(START, "gate"); g.add_edge("gate", END)
|
|
380
|
+
app = g.compile(checkpointer=MemorySaver())
|
|
381
|
+
cfgc = {"configurable": {"thread_id": "hitl-1"}}
|
|
382
|
+
|
|
383
|
+
tr1 = SpanBuilder(db_sink(s), run_name="hitl")
|
|
384
|
+
out = app.invoke({"x": 5}, config={"callbacks": [tr1], **cfgc})
|
|
385
|
+
assert "__interrupt__" in out
|
|
386
|
+
time.sleep(.05)
|
|
387
|
+
assert s.run_detail(tr1.run_id)["status"] == "interrupted"
|
|
388
|
+
|
|
389
|
+
tr2 = SpanBuilder(db_sink(s), run_name="hitl")
|
|
390
|
+
out2 = app.invoke(Command(resume=True), config={"callbacks": [tr2], **cfgc})
|
|
391
|
+
assert out2["x"] == 5
|
|
392
|
+
time.sleep(.05)
|
|
393
|
+
assert s.run_detail(tr2.run_id)["status"] == "done"
|
|
394
|
+
print("HITL interrupt/resume OK")
|
|
395
|
+
|
|
396
|
+
|
|
397
|
+
def test_static_breakpoint_and_state_edit():
|
|
398
|
+
"""interrupt_before pauses with pending next; update_state edit sticks."""
|
|
399
|
+
from typing import TypedDict
|
|
400
|
+
from langgraph.graph import StateGraph, START, END
|
|
401
|
+
from langgraph.checkpoint.memory import MemorySaver
|
|
402
|
+
p = tempfile.mktemp(suffix=".db")
|
|
403
|
+
s = Store(p)
|
|
404
|
+
|
|
405
|
+
class St(TypedDict):
|
|
406
|
+
x: int
|
|
407
|
+
g = StateGraph(St)
|
|
408
|
+
g.add_node("inc", lambda st: {"x": st["x"] + 1})
|
|
409
|
+
g.add_node("mul", lambda st: {"x": st["x"] * 10})
|
|
410
|
+
g.add_edge(START, "inc"); g.add_edge("inc", "mul"); g.add_edge("mul", END)
|
|
411
|
+
app = g.compile(checkpointer=MemorySaver())
|
|
412
|
+
cfgc = {"configurable": {"thread_id": "bp-1"}}
|
|
413
|
+
|
|
414
|
+
tr = SpanBuilder(db_sink(s), run_name="bp")
|
|
415
|
+
for _ in app.stream({"x": 1}, config={"callbacks": [tr], **cfgc},
|
|
416
|
+
stream_mode="updates", interrupt_before=["mul"]):
|
|
417
|
+
pass
|
|
418
|
+
st = app.get_state(cfgc)
|
|
419
|
+
assert st.next == ("mul",), st.next # paused before mul
|
|
420
|
+
app.update_state(cfgc, {"x": 100}) # human edits the state
|
|
421
|
+
out = app.invoke(None, config=cfgc) # continue
|
|
422
|
+
assert out["x"] == 1000, out # 100 * 10 — edit took effect
|
|
423
|
+
print("static breakpoint + state edit OK")
|
|
424
|
+
|
|
425
|
+
|
|
426
|
+
def test_streaming_partials_params_tools():
|
|
427
|
+
"""Token stream flushes partial spans (live 'model typing'); invocation
|
|
428
|
+
params + offered tools captured per LLM span."""
|
|
429
|
+
p = tempfile.mktemp(suffix=".db")
|
|
430
|
+
s = Store(p)
|
|
431
|
+
tr = SpanBuilder(db_sink(s), run_name="stream")
|
|
432
|
+
tr.on_chain_start({}, {"q": 1}, run_id="root", parent_run_id=None)
|
|
433
|
+
tr.on_chat_model_start(
|
|
434
|
+
{"kwargs": {"model": "fable-fast"}}, [[]], run_id="L", parent_run_id="root",
|
|
435
|
+
invocation_params={"model": "fable-fast", "temperature": 0.1, "stream": True,
|
|
436
|
+
"tools": [{"function": {"name": "get_weather"}}]})
|
|
437
|
+
tr.on_llm_new_token("Hello ", run_id="L")
|
|
438
|
+
tr._open["L"]["flushed"] = 0 # force the throttle window open
|
|
439
|
+
tr.on_llm_new_token("wind", run_id="L") # -> partial flush
|
|
440
|
+
mid = s.run_detail(tr.run_id)
|
|
441
|
+
llm_mid = [x for x in mid["spans"] if x["type"] == "llm"][0]
|
|
442
|
+
assert llm_mid["status"] == "running" and "Hello wind" in llm_mid["output"]
|
|
443
|
+
|
|
444
|
+
class Msg:
|
|
445
|
+
content = "Hello windhover"
|
|
446
|
+
usage_metadata = {"input_tokens": 10, "output_tokens": 3}
|
|
447
|
+
class Gen:
|
|
448
|
+
text = "Hello windhover"; message = Msg()
|
|
449
|
+
class Resp:
|
|
450
|
+
llm_output = None; generations = [[Gen()]]
|
|
451
|
+
tr.on_llm_end(Resp(), run_id="L")
|
|
452
|
+
tr.on_chain_end({}, run_id="root")
|
|
453
|
+
d = s.run_detail(tr.run_id)
|
|
454
|
+
llm = [x for x in d["spans"] if x["type"] == "llm"][0]
|
|
455
|
+
assert llm["status"] == "ok" and llm["output"] == "Hello windhover"
|
|
456
|
+
assert llm["params"]["temperature"] == 0.1
|
|
457
|
+
assert llm["params"]["tools_offered"] == ["get_weather"]
|
|
458
|
+
assert len([x for x in d["spans"] if x["type"] == "llm"]) == 1 # partial replaced
|
|
459
|
+
print("streaming partials + params/tools OK")
|
|
460
|
+
|
|
461
|
+
|
|
462
|
+
def test_edge_labels_and_node_metadata():
|
|
463
|
+
from typing import TypedDict, Literal
|
|
464
|
+
from langgraph.graph import StateGraph, START, END
|
|
465
|
+
from windhover.extract import topology
|
|
466
|
+
|
|
467
|
+
class St(TypedDict):
|
|
468
|
+
x: int
|
|
469
|
+
def route(st) -> Literal["yes", "no"]:
|
|
470
|
+
return "yes"
|
|
471
|
+
g = StateGraph(St)
|
|
472
|
+
g.add_node("decide", lambda st: {"x": st["x"]},
|
|
473
|
+
metadata={"owner": "team-a", "doc": "routing"})
|
|
474
|
+
g.add_node("approve", lambda st: {"x": 1})
|
|
475
|
+
g.add_node("reject", lambda st: {"x": 0})
|
|
476
|
+
g.add_edge(START, "decide")
|
|
477
|
+
g.add_conditional_edges("decide", route, {"yes": "approve", "no": "reject"})
|
|
478
|
+
g.add_edge("approve", END); g.add_edge("reject", END)
|
|
479
|
+
topo = topology(g.compile())
|
|
480
|
+
decide = next(n for n in topo["nodes"] if n["id"] == "decide")
|
|
481
|
+
assert decide["metadata"] == {"owner": "team-a", "doc": "routing"}
|
|
482
|
+
labels = {(e["source"], e["target"]): e["label"] for e in topo["edges"]}
|
|
483
|
+
assert labels[("decide", "approve")] == "yes"
|
|
484
|
+
assert labels[("decide", "reject")] == "no"
|
|
485
|
+
assert labels[("__start__", "decide")] is None
|
|
486
|
+
print("edge labels + node metadata OK")
|
|
487
|
+
|
|
488
|
+
|
|
363
489
|
if __name__ == "__main__":
|
|
364
490
|
test_cost(); print("cost OK")
|
|
365
491
|
test_store_roundtrip()
|
|
@@ -377,4 +503,8 @@ if __name__ == "__main__":
|
|
|
377
503
|
test_datasets_and_scoring()
|
|
378
504
|
test_ttft_retry_custom_event_usage_detail()
|
|
379
505
|
test_demo_memory_and_events_end_to_end()
|
|
506
|
+
test_hitl_interrupt_resume()
|
|
507
|
+
test_static_breakpoint_and_state_edit()
|
|
508
|
+
test_streaming_partials_params_tools()
|
|
509
|
+
test_edge_labels_and_node_metadata()
|
|
380
510
|
print("ALL SMOKE TESTS PASSED")
|
|
@@ -24,7 +24,15 @@ def seed(s):
|
|
|
24
24
|
if n < 0:
|
|
25
25
|
raise ValueError(f"n must be non-negative, got {n} — the demo guard tripped")
|
|
26
26
|
return {"n": n}
|
|
27
|
-
def grow(s):
|
|
27
|
+
def grow(s):
|
|
28
|
+
time.sleep(.3)
|
|
29
|
+
grown = s["n"] * 3
|
|
30
|
+
if grown > 500: # human-in-the-loop: big growth needs approval
|
|
31
|
+
from langgraph.types import interrupt
|
|
32
|
+
approved = interrupt({"question": f"grow {s['n']} -> {grown}? (true/false)"})
|
|
33
|
+
if not approved:
|
|
34
|
+
return {"n": s["n"]}
|
|
35
|
+
return {"n": grown}
|
|
28
36
|
def parity(s): time.sleep(.4); return {"notes": ["even" if s["n"] % 2 == 0 else "odd"]}
|
|
29
37
|
def sign(s): time.sleep(.25); return {"notes": ["positive" if s["n"] > 0 else "non-positive"]}
|
|
30
38
|
def magnitude(s): time.sleep(.35); return {"notes": ["big" if abs(s["n"]) > 100 else "small"]}
|
|
@@ -10,10 +10,12 @@ import sys, json, importlib, inspect, functools
|
|
|
10
10
|
def topology(graph, xray: bool = False) -> dict:
|
|
11
11
|
g = graph.get_graph(xray=True) if xray else graph.get_graph()
|
|
12
12
|
nodes = [{"id": nid, "label": str(getattr(n, "name", nid)).replace("__", ""),
|
|
13
|
-
"terminal": nid.endswith(("__start__", "__end__"))
|
|
13
|
+
"terminal": nid.endswith(("__start__", "__end__")),
|
|
14
|
+
"metadata": getattr(n, "metadata", None) or None}
|
|
14
15
|
for nid, n in g.nodes.items()]
|
|
15
16
|
edges = [{"id": f"e{i}", "source": e.source, "target": e.target,
|
|
16
|
-
"conditional": bool(getattr(e, "conditional", False))
|
|
17
|
+
"conditional": bool(getattr(e, "conditional", False)),
|
|
18
|
+
"label": str(e.data) if getattr(e, "data", None) not in (None, e.target) else None}
|
|
17
19
|
for i, e in enumerate(g.edges)]
|
|
18
20
|
return {"nodes": nodes, "edges": edges}
|
|
19
21
|
|
|
@@ -28,6 +30,18 @@ def input_schema(graph) -> dict:
|
|
|
28
30
|
return {}
|
|
29
31
|
|
|
30
32
|
|
|
33
|
+
def context_schema(graph) -> dict:
|
|
34
|
+
"""Runtime context/config schema (langgraph >=1.0 name, older fallback)."""
|
|
35
|
+
for meth in ("get_context_jsonschema", "get_config_jsonschema"):
|
|
36
|
+
try:
|
|
37
|
+
s = getattr(graph, meth)()
|
|
38
|
+
if isinstance(s, dict) and s.get("properties"):
|
|
39
|
+
return s
|
|
40
|
+
except Exception:
|
|
41
|
+
continue
|
|
42
|
+
return {}
|
|
43
|
+
|
|
44
|
+
|
|
31
45
|
def _unwrap(fn):
|
|
32
46
|
"""Peel partials, decorators, and bound methods down to the user function."""
|
|
33
47
|
seen = set()
|
|
@@ -93,7 +107,8 @@ def load(ref: str, dir_: str):
|
|
|
93
107
|
if __name__ == "__main__":
|
|
94
108
|
ref, dir_ = sys.argv[1], sys.argv[2]
|
|
95
109
|
g = load(ref, dir_)
|
|
96
|
-
out = {"topology": topology(g), "schema": input_schema(g), "sources": sources(g)
|
|
110
|
+
out = {"topology": topology(g), "schema": input_schema(g), "sources": sources(g),
|
|
111
|
+
"context_schema": context_schema(g)}
|
|
97
112
|
try: # subgraph x-ray view, only when it actually differs
|
|
98
113
|
tx = topology(g, xray=True)
|
|
99
114
|
if tx != out["topology"]:
|
|
@@ -156,38 +156,29 @@ def api_graph():
|
|
|
156
156
|
@app.get("/api/schema")
|
|
157
157
|
def api_schema():
|
|
158
158
|
s = TOPO.schema()
|
|
159
|
-
|
|
159
|
+
with TOPO.lock:
|
|
160
|
+
ctx = TOPO.data.get("context_schema") or {}
|
|
161
|
+
return JSONResponse({"schema": s, "template": _template(s),
|
|
162
|
+
"context_schema": ctx, "context_template": _template(ctx)})
|
|
160
163
|
|
|
161
164
|
|
|
162
165
|
def _sse(ev: str, data: dict) -> str:
|
|
163
166
|
return f"event: {ev}\ndata: {json.dumps(data)}\n\n"
|
|
164
167
|
|
|
165
168
|
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
try:
|
|
171
|
-
payload = await request.json()
|
|
172
|
-
except Exception:
|
|
173
|
-
payload = {}
|
|
174
|
-
session = payload.pop("_session", None)
|
|
175
|
-
tags = payload.pop("_tags", None)
|
|
176
|
-
thread = payload.pop("_thread", None)
|
|
177
|
-
tracer = SpanBuilder(db_sink(store), run_name=cfg.graph_ref, session=session, tags=tags)
|
|
169
|
+
def _stream_execution(graph_input, config, tracer, stream_kwargs=None):
|
|
170
|
+
"""Shared SSE executor for /api/run and /api/threads/…/resume. Detects both
|
|
171
|
+
dynamic interrupts (__interrupt__ updates) and static breakpoints (stream
|
|
172
|
+
ends with pending next-nodes) and corrects the run status accordingly."""
|
|
178
173
|
run_id = tracer.run_id
|
|
179
|
-
config = {"callbacks": [tracer]}
|
|
180
|
-
if thread or getattr(graph, "checkpointer", None) is not None:
|
|
181
|
-
# a checkpointed graph needs a thread; default to the run id so
|
|
182
|
-
# time-travel works out of the box
|
|
183
|
-
config["configurable"] = {"thread_id": thread or run_id}
|
|
184
174
|
q: "queue.Queue" = queue.Queue()
|
|
185
175
|
|
|
186
176
|
def worker():
|
|
187
177
|
try:
|
|
188
178
|
interrupted = False
|
|
189
|
-
for mode, chunk in graph.stream(
|
|
190
|
-
stream_mode=["updates", "custom"]
|
|
179
|
+
for mode, chunk in graph.stream(graph_input, config=config,
|
|
180
|
+
stream_mode=["updates", "custom"],
|
|
181
|
+
**(stream_kwargs or {})):
|
|
191
182
|
if mode == "custom":
|
|
192
183
|
# get_stream_writer() output from inside a node -> live progress
|
|
193
184
|
q.put(("progress", {"data": _trunc(chunk, 600)}))
|
|
@@ -198,6 +189,18 @@ async def api_run(request: Request):
|
|
|
198
189
|
q.put(("interrupt", {"run_id": run_id}))
|
|
199
190
|
else:
|
|
200
191
|
q.put(("node", {"node": node}))
|
|
192
|
+
if not interrupted and getattr(graph, "checkpointer", None) is not None:
|
|
193
|
+
try: # static breakpoint: stream ended but nodes are pending
|
|
194
|
+
st = graph.get_state(config)
|
|
195
|
+
if st.next:
|
|
196
|
+
interrupted = True
|
|
197
|
+
q.put(("interrupt", {"run_id": run_id,
|
|
198
|
+
"next": list(st.next)}))
|
|
199
|
+
except Exception:
|
|
200
|
+
pass
|
|
201
|
+
if interrupted:
|
|
202
|
+
# tracer closed the run as done on root end; correct it
|
|
203
|
+
store.close_run(run_id, "interrupted", int(time.time() * 1000))
|
|
201
204
|
q.put(("interrupted" if interrupted else "done", {"run_id": run_id}))
|
|
202
205
|
except Exception as e:
|
|
203
206
|
# tracer already recorded the error span/close; surface to the client too
|
|
@@ -216,6 +219,89 @@ async def api_run(request: Request):
|
|
|
216
219
|
return StreamingResponse(stream(), media_type="text/event-stream")
|
|
217
220
|
|
|
218
221
|
|
|
222
|
+
@app.post("/api/run")
|
|
223
|
+
async def api_run(request: Request):
|
|
224
|
+
if graph is None:
|
|
225
|
+
return JSONResponse({"error": "no local graph (WINDHOVER_GRAPH unset)"}, 400)
|
|
226
|
+
try:
|
|
227
|
+
payload = await request.json()
|
|
228
|
+
except Exception:
|
|
229
|
+
payload = {}
|
|
230
|
+
session = payload.pop("_session", None)
|
|
231
|
+
tags = payload.pop("_tags", None)
|
|
232
|
+
thread = payload.pop("_thread", None)
|
|
233
|
+
pause_before = payload.pop("_interrupt_before", None)
|
|
234
|
+
pause_after = payload.pop("_interrupt_after", None)
|
|
235
|
+
extra_conf = payload.pop("_configurable", None)
|
|
236
|
+
tracer = SpanBuilder(db_sink(store), run_name=cfg.graph_ref, session=session, tags=tags)
|
|
237
|
+
config = {"callbacks": [tracer]}
|
|
238
|
+
if thread or getattr(graph, "checkpointer", None) is not None:
|
|
239
|
+
# a checkpointed graph needs a thread; default to the run id so
|
|
240
|
+
# time-travel works out of the box
|
|
241
|
+
config["configurable"] = {"thread_id": thread or tracer.run_id}
|
|
242
|
+
if isinstance(extra_conf, dict) and extra_conf:
|
|
243
|
+
config.setdefault("configurable", {}).update(extra_conf)
|
|
244
|
+
sk = {}
|
|
245
|
+
if pause_before:
|
|
246
|
+
sk["interrupt_before"] = [str(n) for n in pause_before]
|
|
247
|
+
if pause_after:
|
|
248
|
+
sk["interrupt_after"] = [str(n) for n in pause_after]
|
|
249
|
+
return _stream_execution(payload, config, tracer, sk)
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
@app.post("/api/threads/{thread_id}/resume")
|
|
253
|
+
async def api_thread_resume(request: Request, thread_id: str):
|
|
254
|
+
"""Human-in-the-loop: answer an interrupt (Command(resume=…)), redirect
|
|
255
|
+
(Command(goto=…)), continue past a static breakpoint (no body), or fork
|
|
256
|
+
from an earlier checkpoint (checkpoint_id)."""
|
|
257
|
+
if graph is None or getattr(graph, "checkpointer", None) is None:
|
|
258
|
+
return JSONResponse({"error": "no local graph with a checkpointer"}, 400)
|
|
259
|
+
try:
|
|
260
|
+
body = await request.json()
|
|
261
|
+
except Exception:
|
|
262
|
+
body = {}
|
|
263
|
+
try:
|
|
264
|
+
from langgraph.types import Command
|
|
265
|
+
except Exception:
|
|
266
|
+
return JSONResponse({"error": "langgraph.types.Command unavailable"}, 500)
|
|
267
|
+
if "value" in body:
|
|
268
|
+
graph_input = Command(resume=body["value"])
|
|
269
|
+
elif body.get("goto"):
|
|
270
|
+
graph_input = Command(goto=str(body["goto"]), update=body.get("update"))
|
|
271
|
+
else:
|
|
272
|
+
graph_input = None # plain continue (static breakpoint / after state edit)
|
|
273
|
+
tracer = SpanBuilder(db_sink(store), run_name=cfg.graph_ref,
|
|
274
|
+
session=body.get("_session"),
|
|
275
|
+
tags=(body.get("_tags") or []) + ["resume"])
|
|
276
|
+
configurable = {"thread_id": thread_id}
|
|
277
|
+
if body.get("checkpoint_id"):
|
|
278
|
+
configurable["checkpoint_id"] = str(body["checkpoint_id"])
|
|
279
|
+
config = {"callbacks": [tracer], "configurable": configurable}
|
|
280
|
+
return _stream_execution(graph_input, config, tracer)
|
|
281
|
+
|
|
282
|
+
|
|
283
|
+
@app.post("/api/threads/{thread_id}/state")
|
|
284
|
+
async def api_thread_update_state(request: Request, thread_id: str):
|
|
285
|
+
"""Human-in-the-loop: edit state at the current (or a given) checkpoint —
|
|
286
|
+
LangGraph's update_state. Follow with …/resume to continue on the edit."""
|
|
287
|
+
if graph is None or getattr(graph, "checkpointer", None) is None:
|
|
288
|
+
return JSONResponse({"error": "no local graph with a checkpointer"}, 400)
|
|
289
|
+
body = await request.json()
|
|
290
|
+
values = body.get("values")
|
|
291
|
+
if not isinstance(values, dict):
|
|
292
|
+
return JSONResponse({"error": "requires values (object of state keys)"}, 400)
|
|
293
|
+
configurable = {"thread_id": thread_id}
|
|
294
|
+
if body.get("checkpoint_id"):
|
|
295
|
+
configurable["checkpoint_id"] = str(body["checkpoint_id"])
|
|
296
|
+
try:
|
|
297
|
+
new_cfg = graph.update_state({"configurable": configurable}, values,
|
|
298
|
+
as_node=body.get("as_node"))
|
|
299
|
+
return JSONResponse({"ok": True, "checkpoint_id":
|
|
300
|
+
(new_cfg.get("configurable") or {}).get("checkpoint_id")})
|
|
301
|
+
except Exception as e:
|
|
302
|
+
return JSONResponse({"error": str(e)}, 400)
|
|
303
|
+
|
|
304
|
+
|
|
219
305
|
@app.post("/api/ingest")
|
|
220
306
|
async def api_ingest(request: Request):
|
|
221
307
|
ev = await request.json()
|
|
@@ -130,6 +130,58 @@ td.r,th.r{text-align:right}
|
|
|
130
130
|
.empty{padding:56px 20px;text-align:center;color:var(--muted)}
|
|
131
131
|
.empty .t{font-weight:600;color:var(--text2);margin-bottom:4px}
|
|
132
132
|
|
|
133
|
+
/* ---------- human-in-the-loop ---------- */
|
|
134
|
+
async function consumeRunStream(res){
|
|
135
|
+
const rd=res.body.getReader(),dec=new TextDecoder(); let buf='';
|
|
136
|
+
while(true){ const {value,done}=await rd.read(); if(done)break; buf+=dec.decode(value,{stream:true});
|
|
137
|
+
let ps=buf.split('\n\n'); buf=ps.pop();
|
|
138
|
+
for(const p of ps){ const ev=(p.match(/event: (.*)/)||[])[1],dm=(p.match(/data: (.*)/s)||[])[1]; if(!ev)continue; const d=dm?JSON.parse(dm):{};
|
|
139
|
+
if(ev==='node'&&cy){ cy.$id(d.node).addClass('running'); }
|
|
140
|
+
else if(ev==='progress'){ toast('progress: '+JSON.stringify(d.data).slice(0,120)); }
|
|
141
|
+
else if(ev==='interrupted'){ toast('Paused again — awaiting human'); }
|
|
142
|
+
else if(ev==='done'){ toast('Resumed run complete'); }
|
|
143
|
+
else if(ev==='error'){ toast('Resume error: '+(d.message||'failed')); } } }
|
|
144
|
+
if(VIEW==='runs') loadRuns();
|
|
145
|
+
}
|
|
146
|
+
function parseMaybeJSON(v){ if(v==='')return undefined;
|
|
147
|
+
try{ return JSON.parse(v); }catch(e){ return v; } }
|
|
148
|
+
window.resumeThread=async(tid)=>{
|
|
149
|
+
const raw=document.getElementById('hitl-val').value.trim();
|
|
150
|
+
const body={}; const v=parseMaybeJSON(raw); if(v!==undefined) body.value=v;
|
|
151
|
+
closeAll(); switchView('graph'); toast('Resuming thread '+tid.slice(0,8)+'…');
|
|
152
|
+
const res=await fetch('/api/threads/'+encodeURIComponent(tid)+'/resume',
|
|
153
|
+
{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify(body)});
|
|
154
|
+
if(!res.ok){ toast((await res.json()).error||'resume failed'); return; }
|
|
155
|
+
await consumeRunStream(res);
|
|
156
|
+
};
|
|
157
|
+
window.resumeGoto=async(tid)=>{
|
|
158
|
+
const g=document.getElementById('hitl-goto').value; if(!g) return toast('Pick a node first');
|
|
159
|
+
closeAll(); switchView('graph'); toast('Redirecting thread to '+g+'…');
|
|
160
|
+
const res=await fetch('/api/threads/'+encodeURIComponent(tid)+'/resume',
|
|
161
|
+
{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({goto:g})});
|
|
162
|
+
if(!res.ok){ toast((await res.json()).error||'resume failed'); return; }
|
|
163
|
+
await consumeRunStream(res);
|
|
164
|
+
};
|
|
165
|
+
window.rerunFrom=async(tid,cid)=>{
|
|
166
|
+
closeAll(); switchView('graph'); toast('Forking from checkpoint '+String(cid).slice(-8)+'…');
|
|
167
|
+
const res=await fetch('/api/threads/'+encodeURIComponent(tid)+'/resume',
|
|
168
|
+
{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({checkpoint_id:cid})});
|
|
169
|
+
if(!res.ok){ toast((await res.json()).error||'fork failed'); return; }
|
|
170
|
+
await consumeRunStream(res);
|
|
171
|
+
};
|
|
172
|
+
window.editState=async(tid,cid,valuesJson)=>{
|
|
173
|
+
const cur=window.prompt('Edit state values (JSON object) — applied via update_state:',valuesJson||'{}');
|
|
174
|
+
if(cur==null) return;
|
|
175
|
+
let values; try{ values=JSON.parse(cur); }catch(e){ return toast('Invalid JSON'); }
|
|
176
|
+
const body={values}; if(cid) body.checkpoint_id=cid;
|
|
177
|
+
const res=await fetch('/api/threads/'+encodeURIComponent(tid)+'/state',
|
|
178
|
+
{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify(body)});
|
|
179
|
+
const d=await res.json();
|
|
180
|
+
if(!res.ok) return toast(d.error||'update_state failed');
|
|
181
|
+
toast('State updated — new checkpoint '+String(d.checkpoint_id||'').slice(-8)+'; Resume to continue');
|
|
182
|
+
showHistory(tid);
|
|
183
|
+
};
|
|
184
|
+
|
|
133
185
|
/* ---------- time-travel (checkpoint history) ---------- */
|
|
134
186
|
window.showHistory=async(tid)=>{
|
|
135
187
|
let d=null; try{ const r=await fetch('/api/threads/'+encodeURIComponent(tid)+'/history');
|
|
@@ -153,6 +205,10 @@ window.showHistory=async(tid)=>{
|
|
|
153
205
|
<div class="tr-detail" id="${did}">
|
|
154
206
|
${s.writes!=null?kv('writes',s.writes):''}
|
|
155
207
|
${kv('state at this checkpoint',s.values)}
|
|
208
|
+
<div style="display:flex;gap:8px;margin:4px 0 10px">
|
|
209
|
+
<button class="btn ghost" onclick='rerunFrom("${esc(tid)}","${esc(s.checkpoint_id||'')}")'>⑂ Re-run from here</button>
|
|
210
|
+
<button class="btn ghost" onclick='editState("${esc(tid)}","${esc(s.checkpoint_id||'')}",${JSON.stringify(JSON.stringify(s.values??{}))})'>✎ Edit state</button>
|
|
211
|
+
</div>
|
|
156
212
|
</div></div>`;
|
|
157
213
|
}).join('')||'<div class="empty">No checkpoints recorded for this thread</div>';
|
|
158
214
|
$('#scrim').classList.add('on'); $('#drawer').classList.add('on');
|
|
@@ -297,6 +353,8 @@ textarea:focus{border-color:var(--accent)}
|
|
|
297
353
|
.star.on{color:var(--warn)}
|
|
298
354
|
.tagchip{display:inline-block;background:var(--accent-weak);color:var(--accent-text);font-size:10.5px;font-weight:600;
|
|
299
355
|
padding:1.5px 7px;border-radius:5px;margin:1px 4px 1px 0;cursor:pointer}
|
|
356
|
+
.pause-chip{background:var(--surface2);color:var(--text2);border:1px dashed var(--border2)}
|
|
357
|
+
.pause-chip.on{background:var(--warn-weak);color:var(--warn);border-style:solid;border-color:var(--warn)}
|
|
300
358
|
.scorechip{display:inline-block;background:var(--ok-weak);color:var(--ok);font-size:10.5px;font-weight:600;
|
|
301
359
|
padding:1.5px 7px;border-radius:5px;margin:1px 4px 1px 0}
|
|
302
360
|
.pager{display:flex;align-items:center;gap:12px;justify-content:flex-end;margin-top:10px;color:var(--muted);font-size:12.5px}
|
|
@@ -396,7 +454,7 @@ th{font-family:var(--mono);letter-spacing:.11em;font-size:10.5px}
|
|
|
396
454
|
<button data-v="memory" id="nav-mem" style="display:none">${ICON.mem} Memory</button>
|
|
397
455
|
<button data-v="stats">${ICON.chart} Stats</button>
|
|
398
456
|
</nav>
|
|
399
|
-
<div class="side-foot"><span class="live-dot" id="live-dot" title="live topology"></span><span id="foot-note">watching</span><span class="spacer"></span><span class="mono" id="ver">v0.
|
|
457
|
+
<div class="side-foot"><span class="live-dot" id="live-dot" title="live topology"></span><span id="foot-note">watching</span><span class="spacer"></span><span class="mono" id="ver">v0.11</span></div>
|
|
400
458
|
</aside>
|
|
401
459
|
|
|
402
460
|
<main>
|
|
@@ -496,6 +554,10 @@ th{font-family:var(--mono);letter-spacing:.11em;font-size:10.5px}
|
|
|
496
554
|
<input id="run-session" placeholder="session (optional)" style="flex:1;min-width:130px">
|
|
497
555
|
<input id="run-tags" placeholder="tags, comma-separated (optional)" style="flex:1;min-width:160px">
|
|
498
556
|
</div>
|
|
557
|
+
<div id="ctx-wrap" style="display:none"><div class="hint" style="margin:8px 0 4px">Runtime context / configurable (from the graph's context schema):</div>
|
|
558
|
+
<textarea id="ctx-json" spellcheck="false" style="min-height:70px"></textarea></div>
|
|
559
|
+
<div class="hint" style="margin:8px 0 4px">Pause before (static breakpoints — resume from the run drawer):</div>
|
|
560
|
+
<div id="pause-chips" style="display:flex;flex-wrap:wrap;gap:6px"></div>
|
|
499
561
|
<div class="actions">
|
|
500
562
|
<button class="btn ghost" onclick="closeAll()">Cancel</button>
|
|
501
563
|
<button class="btn primary" id="do-run">${ICON.play} Run</button>
|
|
@@ -553,6 +615,9 @@ function cyStyle(){ return [
|
|
|
553
615
|
{selector:'node.err',style:{'border-color':cssv('--err'),'background-color':cssv('--err-weak')}},
|
|
554
616
|
{selector:'edge',style:{'width':1.6,'line-color':cssv('--border2'),'target-arrow-color':cssv('--border2'),'target-arrow-shape':'triangle','curve-style':'bezier','arrow-scale':1}},
|
|
555
617
|
{selector:'edge[?conditional]',style:{'line-style':'dashed'}},
|
|
618
|
+
{selector:'edge[label]',style:{'label':'data(label)','font-size':10,
|
|
619
|
+
'font-family':'ui-monospace, Menlo, Consolas, monospace','color':cssv('--text2'),
|
|
620
|
+
'text-background-color':cssv('--bg'),'text-background-opacity':.9,'text-background-padding':2}},
|
|
556
621
|
{selector:'edge.flow',style:{'line-color':cssv('--accent'),'target-arrow-color':cssv('--accent'),'width':2.4}},
|
|
557
622
|
];}
|
|
558
623
|
function applyCyStyle(){ cy.style(cyStyle()); }
|
|
@@ -566,9 +631,10 @@ async function loadGraph(force){
|
|
|
566
631
|
const xb=$('#xray'); if(xb) xb.style.display=t.xray?'':'none';
|
|
567
632
|
if(!t.nodes.length){ $('#view-graph').innerHTML='<div class="empty" style="padding-top:120px"><div class="t">No local graph configured</div>Set <span class="mono">WINDHOVER_GRAPH</span>, or trace runs in from your app and open Runs.</div>'; return; }
|
|
568
633
|
const topo=(XRAY&&t.xray)?t.xray:t;
|
|
634
|
+
window.__nodes=Object.fromEntries(topo.nodes.map(n=>[n.id,n]));
|
|
569
635
|
const els=[];
|
|
570
636
|
topo.nodes.forEach(n=>els.push({data:{id:n.id,label:n.label,terminal:n.terminal?1:undefined}}));
|
|
571
|
-
topo.edges.forEach(e=>els.push({data:{id:e.id,source:e.source,target:e.target,conditional:e.conditional?1:undefined}}));
|
|
637
|
+
topo.edges.forEach(e=>els.push({data:{id:e.id,source:e.source,target:e.target,conditional:e.conditional?1:undefined,label:e.label||undefined}}));
|
|
572
638
|
if(cy&&!force){return;}
|
|
573
639
|
cy=cytoscape({container:$('#cy'),elements:els,style:cyStyle(),
|
|
574
640
|
layout:layoutOpts(),minZoom:.2,maxZoom:3,wheelSensitivity:.12});
|
|
@@ -644,6 +710,8 @@ async function openNode(name){
|
|
|
644
710
|
const hiAll=new Set();
|
|
645
711
|
if(src)(d.recent||[]).forEach(r=>{ if(r.error) tbLines(r.error,src).forEach(x=>hiAll.add(x)); });
|
|
646
712
|
let html=`<div class="kv"><div class="k">wiring</div><pre>${esc((inn.join(', ')||'START')+' → '+name+' → '+(out.join(', ')||'END'))}</pre></div>`;
|
|
713
|
+
const nmeta=(window.__nodes||{})[name]?.metadata;
|
|
714
|
+
if(nmeta) html+=kv('node metadata', nmeta);
|
|
647
715
|
if(src) html+=`<div class="kv"><div class="k">source · ${esc((src.file||'').split('/').pop())}:${src.line_start}${hiAll.size?` · <span style="color:var(--err)">error line${hiAll.size>1?'s':''} ${[...hiAll].join(', ')}</span>`:''}
|
|
648
716
|
<button class="copy" onclick="navigator.clipboard.writeText(__nodeSrc.code);toast('Copied source')">copy</button></div>${codeBlock(src,hiAll)}</div>`;
|
|
649
717
|
html+=`<div class="kv"><div class="k">executions — newest first, expand for payloads</div></div>`;
|
|
@@ -698,12 +766,25 @@ function switchView(v){ VIEW=v;
|
|
|
698
766
|
$$('.nav button,.bottom-nav button').forEach(b=>b.onclick=()=>switchView(b.dataset.v));
|
|
699
767
|
|
|
700
768
|
/* ---------- new run ---------- */
|
|
701
|
-
$('#newrun').onclick=async()=>{ let tpl={}; try{
|
|
702
|
-
$('#input-json').value=JSON.stringify(tpl,null,2);
|
|
769
|
+
$('#newrun').onclick=async()=>{ let tpl={},ctx={},cs={}; try{const sd=await(await fetch('/api/schema')).json(); tpl=sd.template||{}; ctx=sd.context_template||{}; cs=sd.context_schema||{};}catch(e){}
|
|
770
|
+
$('#input-json').value=JSON.stringify(tpl,null,2);
|
|
771
|
+
const cw=document.getElementById('ctx-wrap');
|
|
772
|
+
if(cw){ const has=cs&&cs.properties&&Object.keys(cs.properties).length;
|
|
773
|
+
cw.style.display=has?'':'none';
|
|
774
|
+
if(has) document.getElementById('ctx-json').value=JSON.stringify(ctx,null,2); }
|
|
775
|
+
const pc=document.getElementById('pause-chips');
|
|
776
|
+
if(pc&&cy) pc.innerHTML=cy.nodes().filter(n=>!n.data('terminal')).map(n=>
|
|
777
|
+
`<span class="tagchip pause-chip" data-n="${esc(n.id())}" onclick="this.classList.toggle('on')">${esc(n.id())}</span>`).join('');
|
|
778
|
+
$('#scrim').classList.add('on'); $('#modal').classList.add('on'); };
|
|
703
779
|
$('#do-run').onclick=async()=>{ let input; try{input=JSON.parse($('#input-json').value||'{}');}catch(e){return toast('Invalid JSON input');}
|
|
704
780
|
const sess=$('#run-session').value.trim();
|
|
705
781
|
const tags=$('#run-tags').value.split(',').map(s=>s.trim()).filter(Boolean);
|
|
706
782
|
if(sess)input._session=sess; if(tags.length)input._tags=tags;
|
|
783
|
+
const paused=[...document.querySelectorAll('.pause-chip.on')].map(e=>e.dataset.n);
|
|
784
|
+
if(paused.length) input._interrupt_before=paused;
|
|
785
|
+
const cw=document.getElementById('ctx-wrap');
|
|
786
|
+
if(cw&&cw.style.display!=='none'){ try{ const c=JSON.parse(document.getElementById('ctx-json').value||'{}');
|
|
787
|
+
if(Object.keys(c).length) input._configurable=c; }catch(e){ return toast('Invalid context JSON'); } }
|
|
707
788
|
closeAll(); switchView('graph'); await runNow(input); };
|
|
708
789
|
|
|
709
790
|
async function runNow(input){
|
|
@@ -822,6 +903,19 @@ async function openRun(id){
|
|
|
822
903
|
<span class="chip node">${esc(s.name)}</span>
|
|
823
904
|
${Object.keys(s.output).map(k=>`<span class="tagchip" style="cursor:default">${esc(k)}</span>`).join('')}
|
|
824
905
|
<span style="color:var(--muted);font-size:11px;margin-left:auto" class="mono">${s.dur_ms!=null?fmtms(s.dur_ms):''}</span></div>`).join('');
|
|
906
|
+
if(r.status==='interrupted'&&r.thread_id){
|
|
907
|
+
const ivs=r.spans.filter(s=>s.type==='interrupt'); const iv=ivs.find(s=>s.output!=null)||ivs.pop();
|
|
908
|
+
html += `<div class="kv" style="margin-top:14px"><div class="k" style="color:var(--warn)">interrupt — awaiting human</div>
|
|
909
|
+
${iv&&iv.output!=null?`<pre>${esc(JSON.stringify(iv.output,null,2))}</pre>`:'<pre>paused at a breakpoint</pre>'}
|
|
910
|
+
<div class="scoreadd" style="margin-top:8px">
|
|
911
|
+
<input id="hitl-val" placeholder='resume value — JSON or text (blank = just continue)' style="flex:1;min-width:200px">
|
|
912
|
+
<button class="btn primary" onclick="resumeThread('${esc(r.thread_id)}')">Resume</button>
|
|
913
|
+
</div>
|
|
914
|
+
<div class="scoreadd">
|
|
915
|
+
<select id="hitl-goto"><option value="">goto node (optional)…</option></select>
|
|
916
|
+
<button class="btn ghost" onclick="resumeGoto('${esc(r.thread_id)}')">Send to node</button>
|
|
917
|
+
</div></div>`;
|
|
918
|
+
}
|
|
825
919
|
html += `<div class="kv" style="margin-top:14px"><div class="k">scores</div></div>
|
|
826
920
|
${(r.scores||[]).map(sc=>`<div class="scorerow"><span class="scorechip">${esc(sc.name)} ${sc.value}</span>
|
|
827
921
|
${sc.comment?`<span style="color:var(--muted);font-size:12px">${esc(sc.comment)}</span>`:''}
|
|
@@ -838,6 +932,9 @@ async function openRun(id){
|
|
|
838
932
|
$('#d-body').innerHTML=html;
|
|
839
933
|
openIds.forEach(i=>{const e=document.getElementById(i); if(e){e.style.display='block';
|
|
840
934
|
const row=e.previousElementSibling; row&&row.querySelector('.caret')?.classList.add('open');}});
|
|
935
|
+
const gsel=document.getElementById('hitl-goto');
|
|
936
|
+
if(gsel&&cy) gsel.innerHTML='<option value="">goto node (optional)…</option>'+
|
|
937
|
+
cy.nodes().filter(n=>!n.data('terminal')).map(n=>`<option>${esc(n.id())}</option>`).join('');
|
|
841
938
|
$('#scrim').classList.add('on'); $('#drawer').classList.add('on');
|
|
842
939
|
clearTimeout(window.__liveT);
|
|
843
940
|
if(r.status==='running')
|
|
@@ -874,7 +971,7 @@ function renderSpans(list,byParent,total,depth){
|
|
|
874
971
|
<div class="tr-meta">${meta}</div>
|
|
875
972
|
</div>
|
|
876
973
|
<div class="tr-detail" id="${did}">
|
|
877
|
-
${s.model?`<div class="kv"><div class="k">model</div><pre>${esc(s.model)}${s.prompt_tokens!=null?` · ${s.prompt_tokens} in / ${s.completion_tokens||0} out`:''}${s.usage_detail?' · '+esc(Object.entries(s.usage_detail).map(([k,v])=>`${v} ${k.replace('input_','').replace('output_','')}`).join(', ')):''}</pre></div>`:''}
|
|
974
|
+
${s.model?`<div class="kv"><div class="k">model</div><pre>${esc(s.model)}${s.prompt_tokens!=null?` · ${s.prompt_tokens} in / ${s.completion_tokens||0} out`:''}${s.usage_detail?' · '+esc(Object.entries(s.usage_detail).map(([k,v])=>`${v} ${k.replace('input_','').replace('output_','')}`).join(', ')):''}${s.params?'\n'+esc(Object.entries(s.params).map(([k,v])=>k==='tools_offered'?'tools: '+v.join(', '):k+'='+v).join(' · ')):''}</pre></div>`:''}
|
|
878
975
|
${s.error?`<div class="kv"><div class="k">error${s.type==='node'?`<button class="copy" onclick="showSpanSource('${s.id}')">view source</button>`:''}</div><pre class="err">${esc(s.error)}</pre><div id="src-${s.id}"></div></div>`:''}
|
|
879
976
|
${s.input!=null?kv('input',s.input):''}
|
|
880
977
|
${s.output!=null?kv('output',s.output):''}
|
|
@@ -906,6 +1003,58 @@ window.showSpanSource=async(sid)=>{
|
|
|
906
1003
|
:'<pre>no source available for this node (external run, or inspect could not trace it)</pre>';
|
|
907
1004
|
};
|
|
908
1005
|
|
|
1006
|
+
/* ---------- human-in-the-loop ---------- */
|
|
1007
|
+
async function consumeRunStream(res){
|
|
1008
|
+
const rd=res.body.getReader(),dec=new TextDecoder(); let buf='';
|
|
1009
|
+
while(true){ const {value,done}=await rd.read(); if(done)break; buf+=dec.decode(value,{stream:true});
|
|
1010
|
+
let ps=buf.split('\n\n'); buf=ps.pop();
|
|
1011
|
+
for(const p of ps){ const ev=(p.match(/event: (.*)/)||[])[1],dm=(p.match(/data: (.*)/s)||[])[1]; if(!ev)continue; const d=dm?JSON.parse(dm):{};
|
|
1012
|
+
if(ev==='node'&&cy){ cy.$id(d.node).addClass('running'); }
|
|
1013
|
+
else if(ev==='progress'){ toast('progress: '+JSON.stringify(d.data).slice(0,120)); }
|
|
1014
|
+
else if(ev==='interrupted'){ toast('Paused again — awaiting human'); }
|
|
1015
|
+
else if(ev==='done'){ toast('Resumed run complete'); }
|
|
1016
|
+
else if(ev==='error'){ toast('Resume error: '+(d.message||'failed')); } } }
|
|
1017
|
+
if(VIEW==='runs') loadRuns();
|
|
1018
|
+
}
|
|
1019
|
+
function parseMaybeJSON(v){ if(v==='')return undefined;
|
|
1020
|
+
try{ return JSON.parse(v); }catch(e){ return v; } }
|
|
1021
|
+
window.resumeThread=async(tid)=>{
|
|
1022
|
+
const raw=document.getElementById('hitl-val').value.trim();
|
|
1023
|
+
const body={}; const v=parseMaybeJSON(raw); if(v!==undefined) body.value=v;
|
|
1024
|
+
closeAll(); switchView('graph'); toast('Resuming thread '+tid.slice(0,8)+'…');
|
|
1025
|
+
const res=await fetch('/api/threads/'+encodeURIComponent(tid)+'/resume',
|
|
1026
|
+
{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify(body)});
|
|
1027
|
+
if(!res.ok){ toast((await res.json()).error||'resume failed'); return; }
|
|
1028
|
+
await consumeRunStream(res);
|
|
1029
|
+
};
|
|
1030
|
+
window.resumeGoto=async(tid)=>{
|
|
1031
|
+
const g=document.getElementById('hitl-goto').value; if(!g) return toast('Pick a node first');
|
|
1032
|
+
closeAll(); switchView('graph'); toast('Redirecting thread to '+g+'…');
|
|
1033
|
+
const res=await fetch('/api/threads/'+encodeURIComponent(tid)+'/resume',
|
|
1034
|
+
{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({goto:g})});
|
|
1035
|
+
if(!res.ok){ toast((await res.json()).error||'resume failed'); return; }
|
|
1036
|
+
await consumeRunStream(res);
|
|
1037
|
+
};
|
|
1038
|
+
window.rerunFrom=async(tid,cid)=>{
|
|
1039
|
+
closeAll(); switchView('graph'); toast('Forking from checkpoint '+String(cid).slice(-8)+'…');
|
|
1040
|
+
const res=await fetch('/api/threads/'+encodeURIComponent(tid)+'/resume',
|
|
1041
|
+
{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({checkpoint_id:cid})});
|
|
1042
|
+
if(!res.ok){ toast((await res.json()).error||'fork failed'); return; }
|
|
1043
|
+
await consumeRunStream(res);
|
|
1044
|
+
};
|
|
1045
|
+
window.editState=async(tid,cid,valuesJson)=>{
|
|
1046
|
+
const cur=window.prompt('Edit state values (JSON object) — applied via update_state:',valuesJson||'{}');
|
|
1047
|
+
if(cur==null) return;
|
|
1048
|
+
let values; try{ values=JSON.parse(cur); }catch(e){ return toast('Invalid JSON'); }
|
|
1049
|
+
const body={values}; if(cid) body.checkpoint_id=cid;
|
|
1050
|
+
const res=await fetch('/api/threads/'+encodeURIComponent(tid)+'/state',
|
|
1051
|
+
{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify(body)});
|
|
1052
|
+
const d=await res.json();
|
|
1053
|
+
if(!res.ok) return toast(d.error||'update_state failed');
|
|
1054
|
+
toast('State updated — new checkpoint '+String(d.checkpoint_id||'').slice(-8)+'; Resume to continue');
|
|
1055
|
+
showHistory(tid);
|
|
1056
|
+
};
|
|
1057
|
+
|
|
909
1058
|
/* ---------- time-travel (checkpoint history) ---------- */
|
|
910
1059
|
window.showHistory=async(tid)=>{
|
|
911
1060
|
let d=null; try{ const r=await fetch('/api/threads/'+encodeURIComponent(tid)+'/history');
|
|
@@ -929,6 +1078,10 @@ window.showHistory=async(tid)=>{
|
|
|
929
1078
|
<div class="tr-detail" id="${did}">
|
|
930
1079
|
${s.writes!=null?kv('writes',s.writes):''}
|
|
931
1080
|
${kv('state at this checkpoint',s.values)}
|
|
1081
|
+
<div style="display:flex;gap:8px;margin:4px 0 10px">
|
|
1082
|
+
<button class="btn ghost" onclick='rerunFrom("${esc(tid)}","${esc(s.checkpoint_id||'')}")'>⑂ Re-run from here</button>
|
|
1083
|
+
<button class="btn ghost" onclick='editState("${esc(tid)}","${esc(s.checkpoint_id||'')}",${JSON.stringify(JSON.stringify(s.values??{}))})'>✎ Edit state</button>
|
|
1084
|
+
</div>
|
|
932
1085
|
</div></div>`;
|
|
933
1086
|
}).join('')||'<div class="empty">No checkpoints recorded for this thread</div>';
|
|
934
1087
|
$('#scrim').classList.add('on'); $('#drawer').classList.add('on');
|
|
@@ -13,7 +13,7 @@ from __future__ import annotations
|
|
|
13
13
|
import json, os, sqlite3, threading, time, uuid
|
|
14
14
|
from typing import Any, Optional
|
|
15
15
|
|
|
16
|
-
SCHEMA_VERSION =
|
|
16
|
+
SCHEMA_VERSION = 7
|
|
17
17
|
_lock = threading.Lock()
|
|
18
18
|
|
|
19
19
|
|
|
@@ -70,7 +70,7 @@ class Store:
|
|
|
70
70
|
c.execute("ALTER TABLE runs ADD COLUMN thread_id TEXT")
|
|
71
71
|
scols = [r[1] for r in c.execute("PRAGMA table_info(spans)").fetchall()]
|
|
72
72
|
for col, typ in (("retries", "INTEGER"), ("ttft_ms", "INTEGER"),
|
|
73
|
-
("usage_detail", "TEXT")):
|
|
73
|
+
("usage_detail", "TEXT"), ("params", "TEXT")):
|
|
74
74
|
if col not in scols:
|
|
75
75
|
c.execute(f"ALTER TABLE spans ADD COLUMN {col} {typ}")
|
|
76
76
|
try:
|
|
@@ -139,8 +139,8 @@ class Store:
|
|
|
139
139
|
c.execute("""INSERT OR REPLACE INTO spans
|
|
140
140
|
(id,run_id,parent_id,seq,type,name,status,started_ms,ended_ms,offset_ms,
|
|
141
141
|
dur_ms,input,output,model,prompt_tokens,completion_tokens,cost_usd,error,
|
|
142
|
-
retries,ttft_ms,usage_detail)
|
|
143
|
-
VALUES(
|
|
142
|
+
retries,ttft_ms,usage_detail,params)
|
|
143
|
+
VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)""",
|
|
144
144
|
(s["id"], s["run_id"], s.get("parent_id"), s.get("seq", 0), s["type"],
|
|
145
145
|
s.get("name"), s.get("status", "ok"), s.get("started_ms"), s.get("ended_ms"),
|
|
146
146
|
s.get("offset_ms"), s.get("dur_ms"),
|
|
@@ -149,7 +149,8 @@ class Store:
|
|
|
149
149
|
s.get("model"), s.get("prompt_tokens"), s.get("completion_tokens"),
|
|
150
150
|
s.get("cost_usd"), s.get("error"),
|
|
151
151
|
s.get("retries"), s.get("ttft_ms"),
|
|
152
|
-
json.dumps(s.get("usage_detail")) if s.get("usage_detail") else None
|
|
152
|
+
json.dumps(s.get("usage_detail")) if s.get("usage_detail") else None,
|
|
153
|
+
json.dumps(s.get("params")) if s.get("params") else None))
|
|
153
154
|
if self.has_fts:
|
|
154
155
|
c.execute("DELETE FROM span_fts WHERE span_id=?", (s["id"],))
|
|
155
156
|
c.execute("INSERT INTO span_fts(text,span_id,run_id) VALUES(?,?,?)",
|
|
@@ -324,7 +325,7 @@ class Store:
|
|
|
324
325
|
for k in ("input", "tags"):
|
|
325
326
|
d[k] = json.loads(d[k]) if d.get(k) else None
|
|
326
327
|
for s in spans:
|
|
327
|
-
for k in ("input", "output", "usage_detail"):
|
|
328
|
+
for k in ("input", "output", "usage_detail", "params"):
|
|
328
329
|
s[k] = json.loads(s[k]) if s.get(k) else None
|
|
329
330
|
d["spans"] = spans
|
|
330
331
|
d["scores"] = scores
|
|
@@ -137,6 +137,7 @@ class SpanBuilder(BaseCallbackHandler):
|
|
|
137
137
|
self._open: dict[str, dict] = {} # langchain run_id -> pending span
|
|
138
138
|
self._span_of: dict[str, str] = {} # langchain run_id -> our span id (for parent links)
|
|
139
139
|
self._seq = 0
|
|
140
|
+
self._interrupted = False
|
|
140
141
|
|
|
141
142
|
# -- infra --
|
|
142
143
|
def _emit(self, ev: dict) -> None:
|
|
@@ -152,9 +153,13 @@ class SpanBuilder(BaseCallbackHandler):
|
|
|
152
153
|
model=None, pt=None, ct=None, usage_detail=None):
|
|
153
154
|
now = time.time()
|
|
154
155
|
sid = info["span_id"]
|
|
156
|
+
seq = info.get("seq")
|
|
157
|
+
if seq is None:
|
|
158
|
+
seq = self._seq
|
|
159
|
+
self._seq += 1
|
|
155
160
|
self._emit({"kind": "span", "id": sid, "run_id": self.run_id,
|
|
156
161
|
"parent_id": self._span_of.get(info.get("parent")),
|
|
157
|
-
"seq":
|
|
162
|
+
"seq": seq, "type": info["type"], "name": info["name"],
|
|
158
163
|
"status": status, "started_ms": int(info["t"] * 1000),
|
|
159
164
|
"ended_ms": int(now * 1000), "offset_ms": self._rel(info["t"]),
|
|
160
165
|
"dur_ms": int((now - info["t"]) * 1000),
|
|
@@ -163,8 +168,7 @@ class SpanBuilder(BaseCallbackHandler):
|
|
|
163
168
|
"cost_usd": cost_of(model, pt, ct), "error": error,
|
|
164
169
|
"retries": info.get("retries"),
|
|
165
170
|
"ttft_ms": int(info["ttft"] * 1000) if info.get("ttft") is not None else None,
|
|
166
|
-
"usage_detail": usage_detail})
|
|
167
|
-
self._seq += 1
|
|
171
|
+
"usage_detail": usage_detail, "params": info.get("params")})
|
|
168
172
|
|
|
169
173
|
# -- chains / nodes --
|
|
170
174
|
_INTERNAL_TAG_PREFIXES = ("graph:", "langsmith:", "seq:", "langgraph_")
|
|
@@ -179,6 +183,8 @@ class SpanBuilder(BaseCallbackHandler):
|
|
|
179
183
|
# config={"metadata": {"windhover_session": ..., "windhover_tags": [...]},
|
|
180
184
|
# "tags": [...]}
|
|
181
185
|
md = metadata or {}
|
|
186
|
+
if md.get("windhover_run_name"):
|
|
187
|
+
self.run_name = str(md["windhover_run_name"])
|
|
182
188
|
if md.get("windhover_session"):
|
|
183
189
|
self.session = str(md["windhover_session"])
|
|
184
190
|
user_tags = [t for t in (tags or [])
|
|
@@ -206,8 +212,11 @@ class SpanBuilder(BaseCallbackHandler):
|
|
|
206
212
|
# LangGraph human-in-the-loop: a paused graph surfaces __interrupt__
|
|
207
213
|
# in its final output — that's a pause awaiting Command(resume=...),
|
|
208
214
|
# not a completion.
|
|
209
|
-
interrupted =
|
|
210
|
-
|
|
215
|
+
interrupted = self._interrupted or (
|
|
216
|
+
isinstance(outputs, dict) and "__interrupt__" in outputs)
|
|
217
|
+
if interrupted and not self._interrupted:
|
|
218
|
+
# marker not yet emitted by the GraphInterrupt path
|
|
219
|
+
self._interrupted = True
|
|
211
220
|
self._finish_span(
|
|
212
221
|
{"span_id": uuid.uuid4().hex[:12], "type": "interrupt",
|
|
213
222
|
"name": "interrupt", "t": time.time(), "parent": None,
|
|
@@ -219,6 +228,32 @@ class SpanBuilder(BaseCallbackHandler):
|
|
|
219
228
|
|
|
220
229
|
def on_chain_error(self, error, *, run_id=None, **kw):
|
|
221
230
|
info = self._open.pop(run_id, None)
|
|
231
|
+
# LangGraph signals a human-in-the-loop pause by raising GraphInterrupt
|
|
232
|
+
# through the node — that's a pause, not a failure.
|
|
233
|
+
if type(error).__name__ in ("GraphInterrupt", "NodeInterrupt"):
|
|
234
|
+
payload = []
|
|
235
|
+
try:
|
|
236
|
+
for i in (error.args[0] if error.args else []):
|
|
237
|
+
payload.append(getattr(i, "value", str(i)))
|
|
238
|
+
except Exception:
|
|
239
|
+
pass
|
|
240
|
+
if info:
|
|
241
|
+
self._finish_span(info, status="interrupted",
|
|
242
|
+
output=_trunc(payload) if payload else None)
|
|
243
|
+
if not self._interrupted: # one marker span per pause
|
|
244
|
+
self._interrupted = True
|
|
245
|
+
now = time.time()
|
|
246
|
+
self._emit({"kind": "span", "id": uuid.uuid4().hex[:12],
|
|
247
|
+
"run_id": self.run_id, "parent_id": None,
|
|
248
|
+
"seq": self._seq, "type": "interrupt", "name": "interrupt",
|
|
249
|
+
"status": "ok", "started_ms": int(now * 1000),
|
|
250
|
+
"ended_ms": int(now * 1000), "offset_ms": self._rel(now),
|
|
251
|
+
"dur_ms": 0, "input": None,
|
|
252
|
+
"output": _trunc(payload) if payload else None,
|
|
253
|
+
"model": None, "prompt_tokens": None,
|
|
254
|
+
"completion_tokens": None, "cost_usd": None, "error": None})
|
|
255
|
+
self._seq += 1
|
|
256
|
+
return
|
|
222
257
|
err = _err_text(error)
|
|
223
258
|
if info:
|
|
224
259
|
self._finish_span(info, status="error", error=err)
|
|
@@ -227,12 +262,31 @@ class SpanBuilder(BaseCallbackHandler):
|
|
|
227
262
|
"ended_ms": int(time.time() * 1000), "error": err})
|
|
228
263
|
|
|
229
264
|
# -- LLMs --
|
|
265
|
+
@staticmethod
|
|
266
|
+
def _llm_params(kw) -> Optional[dict]:
|
|
267
|
+
inv = kw.get("invocation_params") or {}
|
|
268
|
+
out = {}
|
|
269
|
+
for k in ("temperature", "max_tokens", "max_completion_tokens", "stream", "top_p"):
|
|
270
|
+
if inv.get(k) is not None:
|
|
271
|
+
out[k] = inv[k]
|
|
272
|
+
tools = []
|
|
273
|
+
for tl in (inv.get("tools") or []):
|
|
274
|
+
fn = tl.get("function", tl) if isinstance(tl, dict) else {}
|
|
275
|
+
if isinstance(fn, dict) and fn.get("name"):
|
|
276
|
+
tools.append(fn["name"])
|
|
277
|
+
if tools:
|
|
278
|
+
out["tools_offered"] = tools
|
|
279
|
+
return out or None
|
|
280
|
+
|
|
230
281
|
def _llm_start(self, serialized, prompt, run_id, parent_run_id, kw):
|
|
231
282
|
sid = uuid.uuid4().hex[:12]
|
|
232
283
|
self._span_of[run_id] = sid
|
|
284
|
+
seq = self._seq
|
|
285
|
+
self._seq += 1 # reserved now so streaming partials keep a stable order
|
|
233
286
|
self._open[run_id] = {"span_id": sid, "type": "llm",
|
|
234
287
|
"name": _model_name(serialized, kw), "t": time.time(),
|
|
235
|
-
"parent": parent_run_id, "input": _trunc(prompt)
|
|
288
|
+
"parent": parent_run_id, "input": _trunc(prompt),
|
|
289
|
+
"seq": seq, "params": self._llm_params(kw)}
|
|
236
290
|
|
|
237
291
|
def on_llm_start(self, serialized, prompts, *, run_id=None, parent_run_id=None, **kw):
|
|
238
292
|
self._llm_start(serialized, prompts, run_id, parent_run_id, kw)
|
|
@@ -255,10 +309,30 @@ class SpanBuilder(BaseCallbackHandler):
|
|
|
255
309
|
self._finish_span(info, status="error", error=_err_text(error), model=info["name"])
|
|
256
310
|
|
|
257
311
|
def on_llm_new_token(self, token, *, run_id=None, **kw):
|
|
258
|
-
# streaming: first token stamps time-to-first-token
|
|
312
|
+
# streaming: first token stamps time-to-first-token; the growing text is
|
|
313
|
+
# flushed as a partial span every ~0.5s so a live-tailed drawer shows
|
|
314
|
+
# the model typing
|
|
259
315
|
info = self._open.get(run_id)
|
|
260
|
-
if info is
|
|
261
|
-
|
|
316
|
+
if info is None:
|
|
317
|
+
return
|
|
318
|
+
now = time.time()
|
|
319
|
+
if "ttft" not in info:
|
|
320
|
+
info["ttft"] = now - info["t"]
|
|
321
|
+
info["buf"] = (info.get("buf") or "") + (token or "")
|
|
322
|
+
if now - info.get("flushed", 0) >= 0.5 and info.get("buf"):
|
|
323
|
+
info["flushed"] = now
|
|
324
|
+
self._emit({"kind": "span", "id": info["span_id"], "run_id": self.run_id,
|
|
325
|
+
"parent_id": self._span_of.get(info.get("parent")),
|
|
326
|
+
"seq": info["seq"], "type": "llm", "name": info["name"],
|
|
327
|
+
"status": "running", "started_ms": int(info["t"] * 1000),
|
|
328
|
+
"ended_ms": None, "offset_ms": self._rel(info["t"]),
|
|
329
|
+
"dur_ms": int((now - info["t"]) * 1000),
|
|
330
|
+
"input": info.get("input"), "output": info["buf"][-4000:],
|
|
331
|
+
"model": info["name"], "prompt_tokens": None,
|
|
332
|
+
"completion_tokens": None, "cost_usd": None, "error": None,
|
|
333
|
+
"retries": info.get("retries"),
|
|
334
|
+
"ttft_ms": int(info["ttft"] * 1000),
|
|
335
|
+
"usage_detail": None, "params": info.get("params")})
|
|
262
336
|
|
|
263
337
|
# -- retries (tenacity, e.g. rate limits) --
|
|
264
338
|
def on_retry(self, retry_state, *, run_id=None, **kw):
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{windhover-0.9.0 → windhover-0.11.0}/windhover/static/vendor/fonts/fraunces-600-italic.woff2
RENAMED
|
File without changes
|
|
File without changes
|