switchroom 0.18.29 → 0.18.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/handoff-briefing.sh +8 -2
- package/dist/agent-scheduler/index.js +111 -7
- package/dist/auth-broker/index.js +154 -16
- package/dist/cli/autoaccept-poll.js +8 -3
- package/dist/cli/drive-write-pretool.mjs +8 -3
- package/dist/cli/ms-365-write-pretool.mjs +158 -11
- package/dist/cli/notion-write-pretool.mjs +103 -4
- package/dist/cli/switchroom.js +2074 -1585
- package/dist/host-control/main.js +110 -13
- package/dist/vault/approvals/kernel-server.js +116 -13
- package/dist/vault/broker/server.js +314 -145
- package/package.json +3 -3
- package/profiles/_base/start.sh.hbs +73 -20
- package/telegram-plugin/dist/bridge/bridge.js +71 -47
- package/telegram-plugin/dist/gateway/gateway.js +560 -96
- package/telegram-plugin/dist/server.js +89 -64
- package/telegram-plugin/gateway/gateway.ts +212 -17
- package/telegram-plugin/gateway/model-command.ts +104 -0
- package/telegram-plugin/gateway/session-model-file.ts +40 -0
- package/telegram-plugin/gateway/unhandled-message.ts +177 -0
- package/telegram-plugin/llm-error-present.ts +24 -0
- package/telegram-plugin/model-unavailable.ts +55 -0
- package/telegram-plugin/operator-events.ts +113 -0
- package/telegram-plugin/pending-user-notice.ts +88 -0
- package/telegram-plugin/shared/local-time.ts +43 -0
- package/telegram-plugin/tests/catch-all-forwarded-history.test.ts +103 -0
- package/telegram-plugin/tests/catch-all-unhandled-message.test.ts +264 -0
- package/telegram-plugin/tests/gateway-session-model-relaunch.test.ts +26 -2
- package/telegram-plugin/tests/litellm-proxy-auth-misconfig.test.ts +278 -0
- package/telegram-plugin/tests/local-time.test.ts +68 -1
- package/telegram-plugin/tests/model-command.test.ts +133 -0
- package/telegram-plugin/tests/session-model-file.test.ts +23 -0
- package/vendor/hindsight-memory/scripts/backfill_transcripts.py +399 -2
- package/vendor/hindsight-memory/scripts/lib/client.py +47 -0
- package/vendor/hindsight-memory/scripts/lib/content.py +53 -1
- package/vendor/hindsight-memory/scripts/lib/turnlog.py +450 -0
- package/vendor/hindsight-memory/scripts/tests/test_backfill_from_logs.py +467 -0
- package/vendor/hindsight-memory/tests/test_content.py +35 -0
|
@@ -98,6 +98,12 @@ from lib.config import debug_log, load_config
|
|
|
98
98
|
from lib.content import _is_tool_result_only_user_message
|
|
99
99
|
from lib.daemon import get_api_url
|
|
100
100
|
from lib.pacing import inflight_lock
|
|
101
|
+
from lib.turnlog import (
|
|
102
|
+
read_candidate_turns,
|
|
103
|
+
resolve_transcript_for_turn,
|
|
104
|
+
resolve_true_bank,
|
|
105
|
+
transcript_span,
|
|
106
|
+
)
|
|
101
107
|
from retain import build_retain_payload, read_transcript
|
|
102
108
|
|
|
103
109
|
# Injectable sleep so tests assert pacing via a fake clock, not wall time.
|
|
@@ -146,6 +152,36 @@ def _min_idle_s(override: Optional[int] = None) -> float:
|
|
|
146
152
|
return 3600.0
|
|
147
153
|
|
|
148
154
|
|
|
155
|
+
def _window_slack_s(override: Optional[int] = None) -> float:
|
|
156
|
+
"""Slack (seconds) added to each side of a transcript's event span when
|
|
157
|
+
resolving the historical span-containment join (turnlog #3). Default 120s;
|
|
158
|
+
override with ``--window-slack-s`` / ``HINDSIGHT_BACKFILL_WINDOW_SLACK_S``."""
|
|
159
|
+
if override is not None:
|
|
160
|
+
return max(0, override)
|
|
161
|
+
try:
|
|
162
|
+
return max(0.0, float(os.environ.get("HINDSIGHT_BACKFILL_WINDOW_SLACK_S", "120")))
|
|
163
|
+
except ValueError:
|
|
164
|
+
return 120.0
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def _switchroom_yaml_path(agents_root: str) -> str:
|
|
168
|
+
"""Path to switchroom.yaml. Override with ``HINDSIGHT_SWITCHROOM_YAML``;
|
|
169
|
+
otherwise the sibling of the agents root (``~/.switchroom/switchroom.yaml``)."""
|
|
170
|
+
override = os.environ.get("HINDSIGHT_SWITCHROOM_YAML")
|
|
171
|
+
if override:
|
|
172
|
+
return override
|
|
173
|
+
return os.path.join(os.path.dirname(os.path.normpath(agents_root)), "switchroom.yaml")
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def _registry_tmpl(agents_root: str) -> str:
|
|
177
|
+
"""Template (with ``{agent}``) for a per-agent turns registry. Override with
|
|
178
|
+
``HINDSIGHT_REGISTRY_TMPL``; default ``<agents>/{agent}/telegram/registry.db``."""
|
|
179
|
+
override = os.environ.get("HINDSIGHT_REGISTRY_TMPL")
|
|
180
|
+
if override:
|
|
181
|
+
return override
|
|
182
|
+
return os.path.join(agents_root, "{agent}", "telegram", "registry.db")
|
|
183
|
+
|
|
184
|
+
|
|
149
185
|
def _is_dynamic_bank(config: dict) -> bool:
|
|
150
186
|
"""True when the loaded config resolves banks dynamically (per project /
|
|
151
187
|
channel / user). A transcript scan cannot reliably reconstruct such a bank
|
|
@@ -338,6 +374,11 @@ class Backfill:
|
|
|
338
374
|
min_idle_s: Optional[int] = None,
|
|
339
375
|
client: Optional[HindsightClient] = None,
|
|
340
376
|
progress: Optional[Progress] = None,
|
|
377
|
+
from_logs: bool = False,
|
|
378
|
+
switchroom_yaml: Optional[str] = None,
|
|
379
|
+
registry_tmpl: Optional[str] = None,
|
|
380
|
+
window_slack_s: Optional[int] = None,
|
|
381
|
+
require_direct_sessionid: bool = False,
|
|
341
382
|
):
|
|
342
383
|
self.config = config
|
|
343
384
|
self.commit = commit
|
|
@@ -347,6 +388,12 @@ class Backfill:
|
|
|
347
388
|
self.agents_root = agents_root or agents_dir()
|
|
348
389
|
self.min_idle_s = _min_idle_s(min_idle_s)
|
|
349
390
|
self.dynamic_bank = _is_dynamic_bank(config)
|
|
391
|
+
# --from-logs (log-driven recovery) wiring.
|
|
392
|
+
self.from_logs = from_logs
|
|
393
|
+
self.switchroom_yaml = switchroom_yaml or _switchroom_yaml_path(self.agents_root)
|
|
394
|
+
self.registry_tmpl = registry_tmpl or _registry_tmpl(self.agents_root)
|
|
395
|
+
self.window_slack_ms = int(_window_slack_s(window_slack_s) * 1000)
|
|
396
|
+
self.require_direct_sessionid = require_direct_sessionid
|
|
350
397
|
self.client = client
|
|
351
398
|
self.progress = progress or Progress()
|
|
352
399
|
self._api_url = None
|
|
@@ -473,14 +520,27 @@ class Backfill:
|
|
|
473
520
|
"sessions": [],
|
|
474
521
|
})
|
|
475
522
|
|
|
523
|
+
def _registry_path(self, agent: str) -> str:
|
|
524
|
+
return self.registry_tmpl.replace("{agent}", agent)
|
|
525
|
+
|
|
526
|
+
def _settings_path(self, agent: str) -> str:
|
|
527
|
+
return os.path.join(self.agents_root, agent, ".claude", "settings.json")
|
|
528
|
+
|
|
476
529
|
def run(self, agent_filter: Optional[set] = None) -> dict:
|
|
477
530
|
# F2: on a commit run, pin the id-affecting slice_turns; refuse to resume
|
|
478
531
|
# a progress file written with a different value (raises ValueError).
|
|
479
532
|
if self.commit:
|
|
480
533
|
self.progress.ensure_slice_turns(self.slice_turns)
|
|
534
|
+
self.report["mode"] = "from_logs" if self.from_logs else "watermark"
|
|
481
535
|
for agent in self._list_agents(agent_filter):
|
|
482
|
-
self.
|
|
483
|
-
|
|
536
|
+
if self.from_logs:
|
|
537
|
+
self._backfill_agent_from_logs(agent)
|
|
538
|
+
else:
|
|
539
|
+
self._backfill_agent(agent)
|
|
540
|
+
if self.from_logs:
|
|
541
|
+
self._finalize_report_from_logs()
|
|
542
|
+
else:
|
|
543
|
+
self._finalize_report()
|
|
484
544
|
return self.report
|
|
485
545
|
|
|
486
546
|
def _backfill_agent(self, agent: str) -> None:
|
|
@@ -600,6 +660,259 @@ class Backfill:
|
|
|
600
660
|
else:
|
|
601
661
|
self.progress.record(agent, session, "failed")
|
|
602
662
|
|
|
663
|
+
# -- from-logs (log-driven recovery) ------------------------------------ #
|
|
664
|
+
def _logs_agent_report(self, agent: str) -> dict:
|
|
665
|
+
return self.report["agents"].setdefault(agent, {
|
|
666
|
+
"bank_id": None,
|
|
667
|
+
"bank_refused": False,
|
|
668
|
+
"refuse_reason": None,
|
|
669
|
+
"candidate_turns": 0,
|
|
670
|
+
"candidate_sessions": 0,
|
|
671
|
+
"ambiguous_turns": 0,
|
|
672
|
+
"unmatched_turns": 0,
|
|
673
|
+
"sessions_has_documents": 0,
|
|
674
|
+
"sessions_total_loss": 0,
|
|
675
|
+
"sessions_membership_error": 0,
|
|
676
|
+
"sessions_active_skipped": 0,
|
|
677
|
+
"sessions_already_done": 0,
|
|
678
|
+
"sessions_restored": 0,
|
|
679
|
+
"slices_restored": 0,
|
|
680
|
+
"turns_recovered": 0,
|
|
681
|
+
"posts_ok": 0,
|
|
682
|
+
"posts_failed": 0,
|
|
683
|
+
"sessions": [],
|
|
684
|
+
})
|
|
685
|
+
|
|
686
|
+
def _session_membership(self, bank_id: str, session_id: str) -> str:
|
|
687
|
+
"""SESSION-LEVEL, id-scheme-AGNOSTIC membership (BLOCKER-1 fix).
|
|
688
|
+
|
|
689
|
+
Returns ``"present"`` when the bank holds ANY document belonging to the
|
|
690
|
+
session, ``"absent"`` when it holds NONE (true total loss), or
|
|
691
|
+
``"error"`` on any query failure (caller fails CLOSED — treats it as
|
|
692
|
+
present ⇒ skip, never restores on uncertainty).
|
|
693
|
+
|
|
694
|
+
We deliberately do NOT compare against the tool's ``-r{uuid}`` slice ids:
|
|
695
|
+
the deployed production banks are keyed with the LEGACY
|
|
696
|
+
``{session_id}-{epoch_ms}`` scheme (``retain.py``'s not-yet-live
|
|
697
|
+
``slice_document_id`` at line ~430 is the replacement), so an exact slice
|
|
698
|
+
id set would never match a legacy doc → every slice looks "missing" →
|
|
699
|
+
every intact session gets duplicated. Presence is decided by the id
|
|
700
|
+
PREFIX ``{session_id}`` which BOTH schemes share (``{session_id}-...``
|
|
701
|
+
legacy/new, or the bare ``{session_id}`` chunk-0 id). Restore fires ONLY
|
|
702
|
+
for a session with ZERO documents — so partial-within-a-populated-session
|
|
703
|
+
recovery is OUT OF SCOPE on legacy banks (we cannot tell WHICH turns a
|
|
704
|
+
legacy epoch-ms id covers) and can only be added once the ``-r{uuid}``
|
|
705
|
+
scheme is deployed and banks are re-keyed.
|
|
706
|
+
|
|
707
|
+
The GET is paced through the shared inflight lock so must-fix #1's broad
|
|
708
|
+
candidate set can never create a read storm (should-fix 4b)."""
|
|
709
|
+
client = self._ensure_client()
|
|
710
|
+
if client is None:
|
|
711
|
+
return "error"
|
|
712
|
+
self._pace_before_post()
|
|
713
|
+
try:
|
|
714
|
+
with inflight_lock(blocking=True) as acquired:
|
|
715
|
+
if not acquired: # pragma: no cover - blocking acquire fails open
|
|
716
|
+
return "error"
|
|
717
|
+
ids = client.list_session_document_ids(bank_id, session_id)
|
|
718
|
+
except Exception as e:
|
|
719
|
+
debug_log(self.config, f"backfill(from-logs): membership query failed "
|
|
720
|
+
f"for {session_id}: {e} — failing closed (skip)")
|
|
721
|
+
return "error"
|
|
722
|
+
finally:
|
|
723
|
+
# A membership GET counts toward pacing state just like a POST.
|
|
724
|
+
self._posted_count += 1
|
|
725
|
+
self._post_times.append(time.monotonic())
|
|
726
|
+
prefix = f"{session_id}-"
|
|
727
|
+
for did in ids:
|
|
728
|
+
if isinstance(did, str) and (did == session_id or did.startswith(prefix)):
|
|
729
|
+
return "present"
|
|
730
|
+
return "absent"
|
|
731
|
+
|
|
732
|
+
def _backfill_agent_from_logs(self, agent: str) -> None:
|
|
733
|
+
ar = self._logs_agent_report(agent)
|
|
734
|
+
|
|
735
|
+
# Correct-bank targeting: resolve TRUE bank or REFUSE (zero writes).
|
|
736
|
+
res = resolve_true_bank(
|
|
737
|
+
agent,
|
|
738
|
+
switchroom_yaml_path=self.switchroom_yaml,
|
|
739
|
+
settings_path=self._settings_path(agent),
|
|
740
|
+
)
|
|
741
|
+
if not res.ok:
|
|
742
|
+
ar["bank_refused"] = True
|
|
743
|
+
ar["refuse_reason"] = res.reason
|
|
744
|
+
print(f"[Hindsight] backfill(from-logs): REFUSING {agent} — {res.reason} "
|
|
745
|
+
f"(no writes).", file=sys.stderr)
|
|
746
|
+
return
|
|
747
|
+
bank_id = res.bank_id
|
|
748
|
+
ar["bank_id"] = bank_id
|
|
749
|
+
|
|
750
|
+
# BROAD candidate classifier over the durable turn-lifecycle log.
|
|
751
|
+
turns = read_candidate_turns(self._registry_path(agent))
|
|
752
|
+
ar["candidate_turns"] = len(turns)
|
|
753
|
+
if not turns:
|
|
754
|
+
return
|
|
755
|
+
|
|
756
|
+
# Event-span containment join (must-fix #3).
|
|
757
|
+
spans = [transcript_span(p) for p in self._agent_transcripts(agent)]
|
|
758
|
+
sessions: dict = {}
|
|
759
|
+
for t in turns:
|
|
760
|
+
jr = resolve_transcript_for_turn(
|
|
761
|
+
t, spans,
|
|
762
|
+
slack_ms=self.window_slack_ms,
|
|
763
|
+
require_direct_sessionid=self.require_direct_sessionid,
|
|
764
|
+
)
|
|
765
|
+
if jr.kind in ("span", "direct") and jr.session_id:
|
|
766
|
+
sessions.setdefault(jr.session_id, {"path": jr.transcript_path, "join": jr.kind})
|
|
767
|
+
elif jr.kind == "ambiguous":
|
|
768
|
+
ar["ambiguous_turns"] += 1
|
|
769
|
+
ar["sessions"].append({"turn_key": t.turn_key, "class": "ambiguous",
|
|
770
|
+
"action": "refused", "detail": jr.detail})
|
|
771
|
+
else:
|
|
772
|
+
ar["unmatched_turns"] += 1
|
|
773
|
+
ar["candidate_sessions"] = len(sessions)
|
|
774
|
+
|
|
775
|
+
now = time.time()
|
|
776
|
+
for session, info in sessions.items():
|
|
777
|
+
path = info["path"]
|
|
778
|
+
|
|
779
|
+
if self.progress.is_done(agent, session):
|
|
780
|
+
ar["sessions_already_done"] += 1
|
|
781
|
+
continue
|
|
782
|
+
|
|
783
|
+
# Never touch an ACTIVE / recently-written session.
|
|
784
|
+
try:
|
|
785
|
+
idle = now - os.path.getmtime(path)
|
|
786
|
+
except OSError:
|
|
787
|
+
continue
|
|
788
|
+
if idle < self.min_idle_s:
|
|
789
|
+
ar["sessions_active_skipped"] += 1
|
|
790
|
+
ar["sessions"].append({"session": session, "class": "active",
|
|
791
|
+
"action": "skipped_recent", "idle_s": round(idle, 1)})
|
|
792
|
+
continue
|
|
793
|
+
|
|
794
|
+
# SESSION-LEVEL, scheme-agnostic membership (BLOCKER-1 fix): restore
|
|
795
|
+
# ONLY a session the bank has ZERO documents for (true total loss).
|
|
796
|
+
# A session with ANY doc (legacy epoch-ms OR new -r{uuid} scheme) is
|
|
797
|
+
# PRESENT ⇒ skipped — we cannot safely fill a partial tail on a
|
|
798
|
+
# legacy-keyed bank without duplicating the intact head. Fail CLOSED
|
|
799
|
+
# (error ⇒ treated present ⇒ skip).
|
|
800
|
+
membership = self._session_membership(bank_id, session)
|
|
801
|
+
if membership == "error":
|
|
802
|
+
ar["sessions_membership_error"] += 1
|
|
803
|
+
ar["sessions"].append({"session": session, "class": "membership_error",
|
|
804
|
+
"join": info["join"], "action": "skipped_fail_closed"})
|
|
805
|
+
continue
|
|
806
|
+
if membership == "present":
|
|
807
|
+
ar["sessions_has_documents"] += 1
|
|
808
|
+
ar["sessions"].append({"session": session, "class": "has_documents",
|
|
809
|
+
"join": info["join"], "action": "skip_has_documents"})
|
|
810
|
+
continue
|
|
811
|
+
|
|
812
|
+
# membership == "absent" → true total loss → restore the whole session.
|
|
813
|
+
messages = read_transcript(path)
|
|
814
|
+
if not messages:
|
|
815
|
+
continue
|
|
816
|
+
slices = chunk_by_human_turns(messages, self.slice_turns)
|
|
817
|
+
if not slices:
|
|
818
|
+
continue
|
|
819
|
+
built_slices = []
|
|
820
|
+
for sl in slices:
|
|
821
|
+
built = build_retain_payload(
|
|
822
|
+
self.config, session, sl, messages,
|
|
823
|
+
bank_id=bank_id, api_url=self._api_url or "http://backfill-client",
|
|
824
|
+
api_token=self._api_token, retain_full_window=True, document_id=None,
|
|
825
|
+
)
|
|
826
|
+
if built is not None:
|
|
827
|
+
built_slices.append((built, sl))
|
|
828
|
+
if not built_slices:
|
|
829
|
+
continue
|
|
830
|
+
|
|
831
|
+
total = len(built_slices)
|
|
832
|
+
n_turns = sum(len(_human_turn_indices(sl)) for _, sl in built_slices)
|
|
833
|
+
ar["sessions_total_loss"] += 1
|
|
834
|
+
|
|
835
|
+
session_entry = {
|
|
836
|
+
"session": session, "class": "total_loss",
|
|
837
|
+
"join": info["join"], "bank_id": bank_id,
|
|
838
|
+
"slices": total, "turns": n_turns,
|
|
839
|
+
"document_ids": [b["document_id"] for b, _ in built_slices],
|
|
840
|
+
"action": None,
|
|
841
|
+
}
|
|
842
|
+
|
|
843
|
+
if not self.commit:
|
|
844
|
+
# Dry-run: build-only, ZERO writes. Report the gap it WOULD fill.
|
|
845
|
+
session_entry["action"] = "would_restore"
|
|
846
|
+
ar["turns_recovered"] += n_turns
|
|
847
|
+
ar["slices_restored"] += total
|
|
848
|
+
ar["sessions"].append(session_entry)
|
|
849
|
+
continue
|
|
850
|
+
|
|
851
|
+
all_ok = True
|
|
852
|
+
last_built = None
|
|
853
|
+
for built, _ in built_slices:
|
|
854
|
+
if self._post_slice(bank_id, built):
|
|
855
|
+
ar["posts_ok"] += 1
|
|
856
|
+
ar["slices_restored"] += 1
|
|
857
|
+
last_built = built
|
|
858
|
+
else:
|
|
859
|
+
ar["posts_failed"] += 1
|
|
860
|
+
all_ok = False
|
|
861
|
+
|
|
862
|
+
if all_ok and last_built is not None:
|
|
863
|
+
ar["sessions_restored"] += 1
|
|
864
|
+
ar["turns_recovered"] += n_turns
|
|
865
|
+
session_entry["action"] = "restored"
|
|
866
|
+
# Advance the live watermark so the agent's own boot reconcile
|
|
867
|
+
# sees the tail as committed.
|
|
868
|
+
try:
|
|
869
|
+
watermark.commit(
|
|
870
|
+
session, last_built["last_uuid"], last_built["document_id"],
|
|
871
|
+
transcript_path=path, ordered_uuids=last_built["ordered_uuids"],
|
|
872
|
+
)
|
|
873
|
+
except Exception: # pragma: no cover - defensive
|
|
874
|
+
pass
|
|
875
|
+
self.progress.record(agent, session, "done", slices=total, turns=n_turns)
|
|
876
|
+
else:
|
|
877
|
+
session_entry["action"] = "restore_failed"
|
|
878
|
+
self.progress.record(agent, session, "failed")
|
|
879
|
+
ar["sessions"].append(session_entry)
|
|
880
|
+
|
|
881
|
+
def _finalize_report_from_logs(self) -> None:
|
|
882
|
+
roll = {
|
|
883
|
+
"mode": "from_logs",
|
|
884
|
+
"agents": len(self.report["agents"]),
|
|
885
|
+
"banks_refused": 0,
|
|
886
|
+
"candidate_turns": 0,
|
|
887
|
+
"candidate_sessions": 0,
|
|
888
|
+
"ambiguous_turns": 0,
|
|
889
|
+
"unmatched_turns": 0,
|
|
890
|
+
"sessions_has_documents": 0,
|
|
891
|
+
"sessions_total_loss": 0,
|
|
892
|
+
"sessions_membership_error": 0,
|
|
893
|
+
"sessions_active_skipped": 0,
|
|
894
|
+
"sessions_already_done": 0,
|
|
895
|
+
"sessions_restored": 0,
|
|
896
|
+
"slices_restored": 0,
|
|
897
|
+
"turns_recovered": 0,
|
|
898
|
+
"posts_ok": 0,
|
|
899
|
+
"posts_failed": 0,
|
|
900
|
+
"slice_turns": self.slice_turns,
|
|
901
|
+
"committed": self.commit,
|
|
902
|
+
}
|
|
903
|
+
for ar in self.report["agents"].values():
|
|
904
|
+
if ar.get("bank_refused"):
|
|
905
|
+
roll["banks_refused"] += 1
|
|
906
|
+
for k in (
|
|
907
|
+
"candidate_turns", "candidate_sessions", "ambiguous_turns",
|
|
908
|
+
"unmatched_turns", "sessions_has_documents", "sessions_total_loss",
|
|
909
|
+
"sessions_membership_error", "sessions_active_skipped",
|
|
910
|
+
"sessions_already_done", "sessions_restored", "slices_restored",
|
|
911
|
+
"turns_recovered", "posts_ok", "posts_failed",
|
|
912
|
+
):
|
|
913
|
+
roll[k] += ar.get(k, 0)
|
|
914
|
+
self.report["rollup"] = roll
|
|
915
|
+
|
|
603
916
|
def _finalize_report(self) -> None:
|
|
604
917
|
roll = {
|
|
605
918
|
"agents": len(self.report["agents"]),
|
|
@@ -631,7 +944,68 @@ class Backfill:
|
|
|
631
944
|
# --------------------------------------------------------------------------- #
|
|
632
945
|
# Reporting / CLI.
|
|
633
946
|
# --------------------------------------------------------------------------- #
|
|
947
|
+
def format_report_from_logs(report: dict) -> str:
|
|
948
|
+
roll = report.get("rollup", {})
|
|
949
|
+
mode = "COMMIT" if roll.get("committed") else "DRY-RUN (no writes)"
|
|
950
|
+
lines = [f"[Hindsight] log-driven recovery report — {mode}", ""]
|
|
951
|
+
for agent in sorted(report.get("agents", {})):
|
|
952
|
+
ar = report["agents"][agent]
|
|
953
|
+
if ar.get("bank_refused"):
|
|
954
|
+
lines.append(f" {agent}: BANK REFUSED — {ar.get('refuse_reason')} (no writes)")
|
|
955
|
+
continue
|
|
956
|
+
lines.append(
|
|
957
|
+
f" {agent} [bank={ar.get('bank_id')}]: "
|
|
958
|
+
f"cand_turns={ar['candidate_turns']} cand_sessions={ar['candidate_sessions']} "
|
|
959
|
+
f"ambiguous={ar['ambiguous_turns']} unmatched={ar['unmatched_turns']} "
|
|
960
|
+
f"has_documents={ar['sessions_has_documents']} "
|
|
961
|
+
f"total_loss={ar['sessions_total_loss']} "
|
|
962
|
+
f"membership_err={ar['sessions_membership_error']} "
|
|
963
|
+
f"active_skip={ar['sessions_active_skipped']} "
|
|
964
|
+
f"already_done={ar['sessions_already_done']} "
|
|
965
|
+
f"restored={ar['sessions_restored']} "
|
|
966
|
+
f"slices_restored={ar['slices_restored']} "
|
|
967
|
+
f"turns={ar['turns_recovered']} "
|
|
968
|
+
f"posts_ok={ar['posts_ok']} posts_failed={ar['posts_failed']}"
|
|
969
|
+
)
|
|
970
|
+
for s in ar.get("sessions", []):
|
|
971
|
+
if s.get("class") in ("total_loss", "membership_error", "ambiguous"):
|
|
972
|
+
lines.append(
|
|
973
|
+
f" - {s.get('session', s.get('turn_key'))} [{s.get('class')}] "
|
|
974
|
+
f"join={s.get('join', '-')} "
|
|
975
|
+
f"slices={s.get('slices', '-')} "
|
|
976
|
+
f"action={s.get('action')} {s.get('detail', '')}".rstrip()
|
|
977
|
+
)
|
|
978
|
+
lines.append("")
|
|
979
|
+
lines.append(
|
|
980
|
+
f" ROLLUP: agents={roll.get('agents', 0)} "
|
|
981
|
+
f"banks_refused={roll.get('banks_refused', 0)} "
|
|
982
|
+
f"candidate_sessions={roll.get('candidate_sessions', 0)} "
|
|
983
|
+
f"ambiguous={roll.get('ambiguous_turns', 0)} "
|
|
984
|
+
f"has_documents(skipped)={roll.get('sessions_has_documents', 0)} "
|
|
985
|
+
f"total_loss={roll.get('sessions_total_loss', 0)} "
|
|
986
|
+
f"membership_err={roll.get('sessions_membership_error', 0)} "
|
|
987
|
+
f"restored={roll.get('sessions_restored', 0)} "
|
|
988
|
+
f"slices_restored={roll.get('slices_restored', 0)} "
|
|
989
|
+
f"turns_would_recover={roll.get('turns_recovered', 0)} "
|
|
990
|
+
f"posts_ok={roll.get('posts_ok', 0)} posts_failed={roll.get('posts_failed', 0)}"
|
|
991
|
+
)
|
|
992
|
+
lines.append(" NOTE: recovers only ZERO-document (total-loss) sessions. "
|
|
993
|
+
"Partial-tail recovery within a populated session is OUT OF SCOPE "
|
|
994
|
+
"on legacy epoch-ms banks (needs the -r{uuid} id scheme deployed).")
|
|
995
|
+
if not roll.get("committed"):
|
|
996
|
+
est_posts = roll.get("slices_restored", 0)
|
|
997
|
+
delay_ms = _delay_ms()
|
|
998
|
+
est_secs = est_posts * (delay_ms / 1000.0)
|
|
999
|
+
lines.append(
|
|
1000
|
+
f" ESTIMATE (at {delay_ms}ms/post): ~{est_posts} restore POSTs, "
|
|
1001
|
+
f"~{est_secs:.0f}s wall-clock. Re-run with --commit --agent <name> to write."
|
|
1002
|
+
)
|
|
1003
|
+
return "\n".join(lines)
|
|
1004
|
+
|
|
1005
|
+
|
|
634
1006
|
def format_report(report: dict) -> str:
|
|
1007
|
+
if report.get("mode") == "from_logs" or report.get("rollup", {}).get("mode") == "from_logs":
|
|
1008
|
+
return format_report_from_logs(report)
|
|
635
1009
|
roll = report.get("rollup", {})
|
|
636
1010
|
mode = "COMMIT" if roll.get("committed") else "DRY-RUN (no writes)"
|
|
637
1011
|
lines = [f"[Hindsight] backfill report — {mode}", ""]
|
|
@@ -707,6 +1081,24 @@ def main(argv=None) -> int:
|
|
|
707
1081
|
parser.add_argument("--min-idle-s", type=int, default=None,
|
|
708
1082
|
help="Skip sessions whose transcript was modified within this many seconds "
|
|
709
1083
|
"(treated as active/in-flight). Default 3600.")
|
|
1084
|
+
parser.add_argument("--from-logs", action="store_true", dest="from_logs",
|
|
1085
|
+
help="Log-driven recovery mode: narrow candidate sessions via the per-agent "
|
|
1086
|
+
"turns registry, then restore ONLY sessions the agent's true bank has ZERO "
|
|
1087
|
+
"documents for (session-level, scheme-agnostic total-loss gate). "
|
|
1088
|
+
"Partial-tail recovery within a populated session is OUT OF SCOPE on legacy "
|
|
1089
|
+
"epoch-ms banks (needs the -r{uuid} id scheme deployed).")
|
|
1090
|
+
parser.add_argument("--switchroom-yaml", default=None,
|
|
1091
|
+
help="Path to switchroom.yaml for true-bank resolution (--from-logs). "
|
|
1092
|
+
"Default: sibling of the agents dir.")
|
|
1093
|
+
parser.add_argument("--registry-tmpl", default=None,
|
|
1094
|
+
help="Template (with {agent}) for a per-agent turns registry.db (--from-logs). "
|
|
1095
|
+
"Default: <agents>/{agent}/telegram/registry.db")
|
|
1096
|
+
parser.add_argument("--window-slack-s", type=int, default=None,
|
|
1097
|
+
help="Slack (seconds) on each side of a transcript event-span for the "
|
|
1098
|
+
"historical containment join (--from-logs). Default 120.")
|
|
1099
|
+
parser.add_argument("--require-direct-sessionid", action="store_true",
|
|
1100
|
+
help="Refuse any turn without a stamped session_id rather than use the "
|
|
1101
|
+
"span-containment join (strict, going-forward-only; --from-logs).")
|
|
710
1102
|
parser.add_argument("--json", action="store_true", help="Emit the machine-readable report as JSON.")
|
|
711
1103
|
args = parser.parse_args(argv)
|
|
712
1104
|
|
|
@@ -736,6 +1128,11 @@ def main(argv=None) -> int:
|
|
|
736
1128
|
max_per_min=args.max_per_min,
|
|
737
1129
|
agents_root=args.agents_dir,
|
|
738
1130
|
min_idle_s=args.min_idle_s,
|
|
1131
|
+
from_logs=args.from_logs,
|
|
1132
|
+
switchroom_yaml=args.switchroom_yaml,
|
|
1133
|
+
registry_tmpl=args.registry_tmpl,
|
|
1134
|
+
window_slack_s=args.window_slack_s,
|
|
1135
|
+
require_direct_sessionid=args.require_direct_sessionid,
|
|
739
1136
|
)
|
|
740
1137
|
agent_filter = set(args.agents) if args.agents else None
|
|
741
1138
|
try:
|
|
@@ -206,6 +206,53 @@ class HindsightClient:
|
|
|
206
206
|
}
|
|
207
207
|
return self._request("POST", path, body, timeout=timeout)
|
|
208
208
|
|
|
209
|
+
def list_session_document_ids(
|
|
210
|
+
self,
|
|
211
|
+
bank_id: str,
|
|
212
|
+
session_id: str,
|
|
213
|
+
page: int = 200,
|
|
214
|
+
max_pages: int = 50,
|
|
215
|
+
timeout: int = 10,
|
|
216
|
+
) -> set:
|
|
217
|
+
"""Return the set of document ids in ``bank_id`` whose id contains
|
|
218
|
+
``session_id`` (switchroom #3244 log-driven recovery, session-level
|
|
219
|
+
membership).
|
|
220
|
+
|
|
221
|
+
Uses the daemon's server-side ``q=`` filter — a case-insensitive
|
|
222
|
+
substring match on the document id (``GET .../documents?q=...``,
|
|
223
|
+
``http.py:api_list_documents``). Every per-turn transcript retain the
|
|
224
|
+
live path OR this backfill ever wrote for a session carries the session
|
|
225
|
+
id as an id prefix — BOTH the legacy ``{session_id}-{epoch_ms}`` scheme
|
|
226
|
+
(what production banks hold today) and the new ``{session_id}-r{uuid}``
|
|
227
|
+
scheme — so this ONE server-side query (paged, bounded by ``max_pages``)
|
|
228
|
+
returns exactly this session's docs without a per-slice GET storm. The
|
|
229
|
+
caller tests the returned ids for the ``{session_id}`` PREFIX (scheme-
|
|
230
|
+
agnostic) to decide presence: ANY doc ⇒ present ⇒ skip; ZERO ⇒
|
|
231
|
+
total-loss ⇒ restore.
|
|
232
|
+
|
|
233
|
+
Raises on any HTTP/transport error so the caller can fail CLOSED (treat
|
|
234
|
+
the session as PRESENT ⇒ skip) and never risk a duplicate restore.
|
|
235
|
+
"""
|
|
236
|
+
found: set = set()
|
|
237
|
+
offset = 0
|
|
238
|
+
q = urllib.parse.quote(session_id, safe="")
|
|
239
|
+
bank = urllib.parse.quote(bank_id, safe="")
|
|
240
|
+
for _ in range(max(1, max_pages)):
|
|
241
|
+
path = (
|
|
242
|
+
f"/v1/default/banks/{bank}/documents"
|
|
243
|
+
f"?q={q}&limit={int(page)}&offset={int(offset)}"
|
|
244
|
+
)
|
|
245
|
+
resp = self._request("GET", path, timeout=timeout)
|
|
246
|
+
items = resp.get("items") or []
|
|
247
|
+
for it in items:
|
|
248
|
+
did = it.get("id") if isinstance(it, dict) else None
|
|
249
|
+
if did:
|
|
250
|
+
found.add(did)
|
|
251
|
+
if len(items) < page:
|
|
252
|
+
break
|
|
253
|
+
offset += page
|
|
254
|
+
return found
|
|
255
|
+
|
|
209
256
|
def list_directives(
|
|
210
257
|
self,
|
|
211
258
|
bank_id: str,
|
|
@@ -255,7 +255,13 @@ def format_memories(results: list) -> str:
|
|
|
255
255
|
mem_type = r.get("type", "")
|
|
256
256
|
mentioned_at = r.get("mentioned_at", "")
|
|
257
257
|
type_str = f" [{mem_type}]" if mem_type else ""
|
|
258
|
-
|
|
258
|
+
# switchroom #tz-fix (recall side): mentioned_at arrives as a UTC ISO
|
|
259
|
+
# timestamp from the Hindsight server. Render it through the same
|
|
260
|
+
# SWITCHROOM_TIMEZONE→TZ→UTC zoneinfo conversion as format_current_time
|
|
261
|
+
# so recall lines never inject a UTC "when". Date-only / unparseable
|
|
262
|
+
# values are surfaced verbatim rather than crashing recall.
|
|
263
|
+
display_at = _format_local_timestamp(mentioned_at) if mentioned_at else ""
|
|
264
|
+
date_str = f" ({display_at})" if display_at else ""
|
|
259
265
|
lines.append(f"- {text}{type_str}{date_str}")
|
|
260
266
|
return "\n\n".join(lines)
|
|
261
267
|
|
|
@@ -271,6 +277,52 @@ def _resolve_agent_timezone() -> str:
|
|
|
271
277
|
return os.environ.get("SWITCHROOM_TIMEZONE") or os.environ.get("TZ") or "UTC"
|
|
272
278
|
|
|
273
279
|
|
|
280
|
+
def _format_local_timestamp(value: str) -> str:
|
|
281
|
+
"""Render a UTC ISO timestamp in the agent's LOCAL timezone (am/pm form).
|
|
282
|
+
|
|
283
|
+
Used by ``format_memories`` to convert the server-supplied ``mentioned_at``
|
|
284
|
+
(UTC ISO, e.g. ``2026-07-16T04:09:00Z``) through the same
|
|
285
|
+
SWITCHROOM_TIMEZONE→TZ→UTC zoneinfo cascade as ``format_current_time``,
|
|
286
|
+
so recalled memories never surface a UTC "when".
|
|
287
|
+
|
|
288
|
+
Guarding: only full ISO *datetime* values (those carrying a time component,
|
|
289
|
+
i.e. containing ``T``) are converted. Date-only strings (``2024-01-01``) and
|
|
290
|
+
anything unparseable are returned verbatim rather than crashing recall or
|
|
291
|
+
fabricating a midnight time.
|
|
292
|
+
"""
|
|
293
|
+
if not isinstance(value, str):
|
|
294
|
+
return value
|
|
295
|
+
raw = value.strip()
|
|
296
|
+
# Only convert full ISO datetimes — a bare date has no wall-clock to shift.
|
|
297
|
+
if "T" not in raw:
|
|
298
|
+
return value
|
|
299
|
+
parsed = None
|
|
300
|
+
try:
|
|
301
|
+
# Python <3.11 fromisoformat rejects a trailing 'Z'; normalise it.
|
|
302
|
+
iso = raw[:-1] + "+00:00" if raw.endswith("Z") else raw
|
|
303
|
+
parsed = datetime.fromisoformat(iso)
|
|
304
|
+
except (ValueError, TypeError):
|
|
305
|
+
return value
|
|
306
|
+
# Naive value → server sends UTC, so assume UTC before converting.
|
|
307
|
+
if parsed.tzinfo is None:
|
|
308
|
+
if ZoneInfo is not None:
|
|
309
|
+
try:
|
|
310
|
+
parsed = parsed.replace(tzinfo=ZoneInfo("UTC"))
|
|
311
|
+
except Exception:
|
|
312
|
+
return value
|
|
313
|
+
else:
|
|
314
|
+
return value
|
|
315
|
+
tz_name = _resolve_agent_timezone()
|
|
316
|
+
if ZoneInfo is not None:
|
|
317
|
+
try:
|
|
318
|
+
local = parsed.astimezone(ZoneInfo(tz_name))
|
|
319
|
+
except Exception: # unknown/invalid zone — degrade to process-local.
|
|
320
|
+
local = parsed.astimezone()
|
|
321
|
+
else:
|
|
322
|
+
local = parsed.astimezone()
|
|
323
|
+
return local.strftime("%Y-%m-%d %I:%M %p %Z")
|
|
324
|
+
|
|
325
|
+
|
|
274
326
|
def format_current_time() -> str:
|
|
275
327
|
"""Format the current time in the agent's LOCAL timezone for recall context.
|
|
276
328
|
|