alissa-tools-github-orcloop 0.4.0__tar.gz → 0.4.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {alissa_tools_github_orcloop-0.4.0/src/main/alissa_tools_github_orcloop.egg-info → alissa_tools_github_orcloop-0.4.2}/PKG-INFO +1 -1
- {alissa_tools_github_orcloop-0.4.0 → alissa_tools_github_orcloop-0.4.2}/src/main/alissa/tools/github/orcloop/console.py +105 -13
- {alissa_tools_github_orcloop-0.4.0 → alissa_tools_github_orcloop-0.4.2}/src/main/alissa/tools/github/orcloop/loop.py +416 -9
- {alissa_tools_github_orcloop-0.4.0 → alissa_tools_github_orcloop-0.4.2}/src/main/alissa/tools/github/orcloop/state.py +54 -11
- alissa_tools_github_orcloop-0.4.2/src/main/alissa/tools/github/orcloop/version +1 -0
- {alissa_tools_github_orcloop-0.4.0 → alissa_tools_github_orcloop-0.4.2}/src/main/alissa/tools/github/orcloop/workplan.py +26 -1
- {alissa_tools_github_orcloop-0.4.0 → alissa_tools_github_orcloop-0.4.2}/src/main/alissa/tools/github/orcloop/workplan_html.py +5 -3
- {alissa_tools_github_orcloop-0.4.0 → alissa_tools_github_orcloop-0.4.2/src/main/alissa_tools_github_orcloop.egg-info}/PKG-INFO +1 -1
- alissa_tools_github_orcloop-0.4.0/src/main/alissa/tools/github/orcloop/version +0 -1
- {alissa_tools_github_orcloop-0.4.0 → alissa_tools_github_orcloop-0.4.2}/LICENSE +0 -0
- {alissa_tools_github_orcloop-0.4.0 → alissa_tools_github_orcloop-0.4.2}/MANIFEST.in +0 -0
- {alissa_tools_github_orcloop-0.4.0 → alissa_tools_github_orcloop-0.4.2}/NOTICE +0 -0
- {alissa_tools_github_orcloop-0.4.0 → alissa_tools_github_orcloop-0.4.2}/README.md +0 -0
- {alissa_tools_github_orcloop-0.4.0 → alissa_tools_github_orcloop-0.4.2}/requirements.txt +0 -0
- {alissa_tools_github_orcloop-0.4.0 → alissa_tools_github_orcloop-0.4.2}/setup.cfg +0 -0
- {alissa_tools_github_orcloop-0.4.0 → alissa_tools_github_orcloop-0.4.2}/setup.py +0 -0
- {alissa_tools_github_orcloop-0.4.0 → alissa_tools_github_orcloop-0.4.2}/src/main/alissa/tools/github/orcloop/__init__.py +0 -0
- {alissa_tools_github_orcloop-0.4.0 → alissa_tools_github_orcloop-0.4.2}/src/main/alissa/tools/github/orcloop/__main__.py +0 -0
- {alissa_tools_github_orcloop-0.4.0 → alissa_tools_github_orcloop-0.4.2}/src/main/alissa/tools/github/orcloop/alissa_client.py +0 -0
- {alissa_tools_github_orcloop-0.4.0 → alissa_tools_github_orcloop-0.4.2}/src/main/alissa/tools/github/orcloop/config.py +0 -0
- {alissa_tools_github_orcloop-0.4.0 → alissa_tools_github_orcloop-0.4.2}/src/main/alissa/tools/github/orcloop/ghclient.py +0 -0
- {alissa_tools_github_orcloop-0.4.0 → alissa_tools_github_orcloop-0.4.2}/src/main/alissa/tools/github/orcloop/markers.py +0 -0
- {alissa_tools_github_orcloop-0.4.0 → alissa_tools_github_orcloop-0.4.2}/src/main/alissa/tools/github/orcloop/proc.py +0 -0
- {alissa_tools_github_orcloop-0.4.0 → alissa_tools_github_orcloop-0.4.2}/src/main/alissa/tools/github/orcloop/version.py +0 -0
- {alissa_tools_github_orcloop-0.4.0 → alissa_tools_github_orcloop-0.4.2}/src/main/alissa_tools_github_orcloop.egg-info/SOURCES.txt +0 -0
- {alissa_tools_github_orcloop-0.4.0 → alissa_tools_github_orcloop-0.4.2}/src/main/alissa_tools_github_orcloop.egg-info/dependency_links.txt +0 -0
- {alissa_tools_github_orcloop-0.4.0 → alissa_tools_github_orcloop-0.4.2}/src/main/alissa_tools_github_orcloop.egg-info/entry_points.txt +0 -0
- {alissa_tools_github_orcloop-0.4.0 → alissa_tools_github_orcloop-0.4.2}/src/main/alissa_tools_github_orcloop.egg-info/top_level.txt +0 -0
|
@@ -48,6 +48,8 @@ import hashlib
|
|
|
48
48
|
import hmac
|
|
49
49
|
import html
|
|
50
50
|
import logging
|
|
51
|
+
import socket
|
|
52
|
+
import sys
|
|
51
53
|
import threading
|
|
52
54
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
53
55
|
from typing import Any
|
|
@@ -68,6 +70,14 @@ WORKPLAN_PATH = "/workplan"
|
|
|
68
70
|
HEALTH_PATH = "/healthz"
|
|
69
71
|
|
|
70
72
|
|
|
73
|
+
def _static_prefix(fmt: str) -> str:
|
|
74
|
+
"""A log format's literal head, up to its first placeholder.
|
|
75
|
+
|
|
76
|
+
The one thing that can be taken from a logging call whose arguments are
|
|
77
|
+
untrusted: it is written in the source, not received from a socket."""
|
|
78
|
+
return fmt.split("%", 1)[0].strip() or "error"
|
|
79
|
+
|
|
80
|
+
|
|
71
81
|
def session_token(passcode: str) -> str:
|
|
72
82
|
"""The cookie value for a passcode. Never the passcode itself."""
|
|
73
83
|
return hmac.new(
|
|
@@ -137,29 +147,86 @@ class ConsoleHandler(BaseHTTPRequestHandler):
|
|
|
137
147
|
# -- plumbing ----------------------------------------------------------
|
|
138
148
|
|
|
139
149
|
def log_message(self, fmt: str, *args: Any) -> None:
|
|
140
|
-
"""
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
to
|
|
150
|
+
"""The base class's catch-all sink, kept as a backstop.
|
|
151
|
+
|
|
152
|
+
`log_request` and `log_error` below override the only two callers
|
|
153
|
+
`BaseHTTPRequestHandler` has for this method, so nothing in today's
|
|
154
|
+
flow reaches it -- but leaving it un-overridden would let any future
|
|
155
|
+
stdlib caller write the raw request line to stderr, which is the
|
|
156
|
+
behaviour both of those exist to prevent. It renders the same
|
|
157
|
+
stripped label and nothing from `args`; the reasoning for that is on
|
|
158
|
+
`_request_label` and `log_error`."""
|
|
159
|
+
log.debug("console %s", self._request_label())
|
|
160
|
+
|
|
161
|
+
def _request_label(self) -> str:
|
|
162
|
+
"""`<METHOD> <path>` with the query string removed, and safe to build
|
|
163
|
+
at any point in the request's life. Every logging path starts here, so
|
|
164
|
+
there is exactly one place that touches `self.path`.
|
|
146
165
|
|
|
147
166
|
`self.path` MUST be read defensively. It is set by `parse_request`, and
|
|
148
|
-
|
|
149
|
-
never parsed: a malformed request line, a bad HTTP version, a
|
|
150
|
-
line over 64KB (all three
|
|
151
|
-
`log_error`), and an idle connection hitting `timeout
|
|
167
|
+
the callers below run before that on every path where the request line
|
|
168
|
+
was never parsed: a malformed request line, a bad HTTP version, a
|
|
169
|
+
request line over 64KB (all three arrive through `send_error` ->
|
|
170
|
+
`log_error`), and an idle connection hitting `timeout`. An
|
|
152
171
|
`AttributeError` here escapes `handle()`, so the stdlib's own `400`
|
|
153
172
|
never gets sent AND `socketserver.handle_error` dumps a traceback --
|
|
154
173
|
unauthenticated, below the gate, once per connection. A daemon that
|
|
155
174
|
demotes `SKIPPED` to DEBUG for the sake of an unattended log must not
|
|
156
|
-
let anyone who can open a socket write a traceback into it.
|
|
175
|
+
let anyone who can open a socket write a traceback into it.
|
|
176
|
+
|
|
177
|
+
The query string is dropped for the other half of the same standard:
|
|
178
|
+
the passcode arrives as `?passcode=...` exactly once, on the unlock,
|
|
179
|
+
and the default implementation would write that whole line out."""
|
|
157
180
|
try:
|
|
158
181
|
raw = getattr(self, "path", None)
|
|
159
182
|
path = urlparse(raw).path if raw else "-"
|
|
160
183
|
except ValueError: # pragma: no cover - urlparse is total for str
|
|
161
184
|
path = "(unparseable)"
|
|
162
|
-
|
|
185
|
+
return f"{getattr(self, 'command', None) or '-'} {path}"
|
|
186
|
+
|
|
187
|
+
def log_request(self, code: "int | str" = "-", size: "int | str" = "-") -> None:
|
|
188
|
+
"""A served request, WITH its status code.
|
|
189
|
+
|
|
190
|
+
The code comes from this method's own argument, never from the
|
|
191
|
+
stdlib's `log_request` format -- that one is
|
|
192
|
+
`'"%s" %s %s' % (self.requestline, code, size)`, and `requestline`
|
|
193
|
+
carries the full path INCLUDING the query string, which is the one
|
|
194
|
+
thing this class exists to keep out of the log (the passcode arrives
|
|
195
|
+
there once, on the unlock). So the line is rebuilt from the stripped
|
|
196
|
+
label instead of formatted from the request line."""
|
|
197
|
+
log.debug("console %s -> %s", self._request_label(), code)
|
|
198
|
+
|
|
199
|
+
def log_error(self, fmt: str, *args: Any) -> None:
|
|
200
|
+
"""An error the stdlib is reporting: its STATUS CODE, never its text.
|
|
201
|
+
|
|
202
|
+
The code is what an operator who turned DEBUG on because the console is
|
|
203
|
+
answering 400s came for -- through 0.4.0 they got a line with no
|
|
204
|
+
method, no path and no code, because these paths run before
|
|
205
|
+
`parse_request` sets any of them.
|
|
206
|
+
|
|
207
|
+
Nothing from `args` is interpolated, and that rule is deliberately
|
|
208
|
+
stronger than scrubbing the query out of the rendered string. The first
|
|
209
|
+
version of this method rendered `fmt % args` on the reasoning that
|
|
210
|
+
neither sender's FORMAT carries a request line -- which is true, and
|
|
211
|
+
irrelevant: `parse_request` passes `"Bad request syntax (%r)" %
|
|
212
|
+
requestline` as `send_error`'s MESSAGE ARGUMENT, so the whole line,
|
|
213
|
+
query string and passcode included, arrived in `args`. The format is
|
|
214
|
+
not the payload. A rule that depends on having enumerated the senders
|
|
215
|
+
correctly is a rule that fails the next time one of them changes, so
|
|
216
|
+
this one does not enumerate them:
|
|
217
|
+
|
|
218
|
+
* an integer first argument is a status code (`send_error` passes one)
|
|
219
|
+
and is rendered as an integer;
|
|
220
|
+
* anything else falls back to the format's own static prefix, up to its
|
|
221
|
+
first placeholder -- `Request timed out:` for the timeout path.
|
|
222
|
+
|
|
223
|
+
The second reason to render nothing from `args`: the request line is
|
|
224
|
+
capped just under 64KB by `readline(65537)`, so rendering it let any
|
|
225
|
+
client that could open a socket write ~60KB of its own choosing into
|
|
226
|
+
the daemon's log, once per connection, below the gate."""
|
|
227
|
+
code = args[0] if args and isinstance(args[0], int) else None
|
|
228
|
+
detail = f"code {int(code)}" if code is not None else _static_prefix(fmt)
|
|
229
|
+
log.debug("console %s: %s", self._request_label(), detail)
|
|
163
230
|
|
|
164
231
|
def _send(self, status: int, body: str, ctype: str = "text/html; charset=utf-8",
|
|
165
232
|
extra: "list[tuple[str, str]] | None" = None) -> None:
|
|
@@ -288,6 +355,31 @@ class ConsoleHandler(BaseHTTPRequestHandler):
|
|
|
288
355
|
self._send(200, render_page(plan, refresh_seconds=self.context.poll_interval))
|
|
289
356
|
|
|
290
357
|
|
|
358
|
+
class _ConsoleServer(ThreadingHTTPServer):
|
|
359
|
+
"""`ThreadingHTTPServer` that reports a broken connection as one line.
|
|
360
|
+
|
|
361
|
+
The base class answers ANY exception escaping a handler with
|
|
362
|
+
`traceback.print_exc()` to stderr, and the commonest one here is not a bug:
|
|
363
|
+
a client that disconnects mid-response (a browser navigating away, a
|
|
364
|
+
platform TCP health probe, anything that reads a prefix and hangs up)
|
|
365
|
+
raises `ConnectionResetError`/`BrokenPipeError` out of `wfile.write`.
|
|
366
|
+
Unhandled, each one prints a multi-line traceback -- unauthenticated, below
|
|
367
|
+
the gate, once per occurrence, which is the same standard the request-line
|
|
368
|
+
guard on `_request_label` is written to.
|
|
369
|
+
|
|
370
|
+
Found while re-verifying the `log_error` fix against a live socket rather
|
|
371
|
+
than reported by a review; the two are one class of defect, so they are
|
|
372
|
+
fixed together. Anything that is NOT a disconnect keeps the base class's
|
|
373
|
+
traceback: a real bug in a handler must stay loud."""
|
|
374
|
+
|
|
375
|
+
def handle_error(self, request, client_address) -> None:
|
|
376
|
+
exc = sys.exc_info()[1]
|
|
377
|
+
if isinstance(exc, (ConnectionResetError, BrokenPipeError, socket.timeout)):
|
|
378
|
+
log.debug("console: client went away (%s)", type(exc).__name__)
|
|
379
|
+
return
|
|
380
|
+
super().handle_error(request, client_address)
|
|
381
|
+
|
|
382
|
+
|
|
291
383
|
def _handler_for(context: ConsoleContext) -> type:
|
|
292
384
|
"""A handler class bound to one context.
|
|
293
385
|
|
|
@@ -336,7 +428,7 @@ class Console:
|
|
|
336
428
|
return self._server.server_address[:2] # type: ignore[return-value]
|
|
337
429
|
|
|
338
430
|
def start(self) -> None:
|
|
339
|
-
server =
|
|
431
|
+
server = _ConsoleServer(
|
|
340
432
|
(self._host, self._port), _handler_for(self._context)
|
|
341
433
|
)
|
|
342
434
|
server.daemon_threads = True
|
|
@@ -319,6 +319,14 @@ released before a feed switch):
|
|
|
319
319
|
the merge already happened, so holding the close would only strand the task.
|
|
320
320
|
* Closed with NO merged closing PR -> the `closed-unmerged` escalation only;
|
|
321
321
|
the task is NOT mutated (a human closed it, or the work was abandoned).
|
|
322
|
+
* The origin task itself cannot be READ (404) on `ORIGIN_MISSING_MAX_ATTEMPTS`
|
|
323
|
+
distinct polls (O22, issue #58) -> the close loop for that item is ABANDONED:
|
|
324
|
+
a `closeloop-terminal:origin-archived` ledger row, exactly one comment on the
|
|
325
|
+
GitHub issue, and no Alissa read or write for it ever again. A task that
|
|
326
|
+
404s to the token that created it has been archived past the workspace's
|
|
327
|
+
retention window or deleted, and no retry recovers that -- while retrying
|
|
328
|
+
pages the one artifact guaranteed unreachable, which is how this daemon spent
|
|
329
|
+
four days generating a denied write per poll per item. See `_origin_missing`.
|
|
322
330
|
* The origin task already moved on (validated/cancelled/blocked/... -- the
|
|
323
331
|
operator drove it by hand) -> skip and log, never fight the operator.
|
|
324
332
|
|
|
@@ -334,7 +342,11 @@ longer than the same `release_blocked_floor_days` floor -- measured on the
|
|
|
334
342
|
FAILURE, not the task's age, so a transient rate limit on a long-queued task
|
|
335
343
|
never pages -- because a hold that could never page is how a task goes quiet
|
|
336
344
|
forever), and `closeloop-failed:a<j>` (a close-loop mutation failed; retried
|
|
337
|
-
with a floor between attempts, each attempt paging once).
|
|
345
|
+
with a floor between attempts, each attempt paging once). One escalation is
|
|
346
|
+
NOT posted on the task: `closeloop-terminal:origin-archived` is about a task
|
|
347
|
+
that cannot be read, so its only channel is the GitHub issue -- and there the
|
|
348
|
+
issue comment is PRIMARY, its delivery gating the ledger row, rather than the
|
|
349
|
+
best-effort second copy every other page treats it as.
|
|
338
350
|
|
|
339
351
|
NO SILENT STATES: every candidate and every released issue produces exactly one
|
|
340
352
|
logged `Decision` each pass -- a deferral (`held`) always leaves a per-poll log
|
|
@@ -387,6 +399,7 @@ from .alissa_client import (
|
|
|
387
399
|
VIEW_HEADLINE,
|
|
388
400
|
AlissaClient,
|
|
389
401
|
AlissaError,
|
|
402
|
+
AlissaNotFound,
|
|
390
403
|
AlissaTransient,
|
|
391
404
|
BodyOfWork,
|
|
392
405
|
BowMember,
|
|
@@ -522,6 +535,73 @@ ESCALATION_CLOSELOOP_FAILED = "closeloop-failed"
|
|
|
522
535
|
ESCALATION_ALREADY_MERGED = "already-merged"
|
|
523
536
|
ESCALATION_DEDUPE_UNREADABLE = "dedupe-unreadable"
|
|
524
537
|
|
|
538
|
+
# The close edge gave up on an item for good, with the REASON folded into the
|
|
539
|
+
# kind (`closeloop-terminal:<reason>`, the same "narrow the dedupe scope by
|
|
540
|
+
# folding more key into the string" the ledger's kind column is TEXT for). The
|
|
541
|
+
# row is read back with a prefix scan, so the reason an operator sees in the
|
|
542
|
+
# skip line every later poll comes out of the ledger rather than out of a
|
|
543
|
+
# constant this code happens to pick -- which is what makes the reason a
|
|
544
|
+
# recorded fact instead of a log-time guess.
|
|
545
|
+
#
|
|
546
|
+
# One reason today (O22, issue #58): the origin task's own read answers 404
|
|
547
|
+
# forever, because the task was ARCHIVED after validation (the platform's
|
|
548
|
+
# `autoArchiveDays` default) or deleted. Nothing about that self-heals, so
|
|
549
|
+
# every close-loop attempt after it is spent producing a failure -- and, worse,
|
|
550
|
+
# the failure's escalation targets that same unreachable task, so the daemon
|
|
551
|
+
# generates a denied write per poll per item, forever (the studio Convex
|
|
552
|
+
# `apiAddComment` NO_ACCESS storm this constant ends).
|
|
553
|
+
ESCALATION_CLOSELOOP_TERMINAL = "closeloop-terminal"
|
|
554
|
+
CLOSELOOP_TERMINAL_ORIGIN_ARCHIVED = "origin-archived"
|
|
555
|
+
|
|
556
|
+
# How many close-loop passes may see the origin task answer 404 before the item
|
|
557
|
+
# is given up on. A 404 is a PERMANENT status in the client's taxonomy, but
|
|
558
|
+
# "permanent" there is a classification and not an observation (the argument
|
|
559
|
+
# `_close_evidence_unwritable` makes at length): a task can 404 for a few
|
|
560
|
+
# seconds while the API is mid-failover, and giving up on the first one would
|
|
561
|
+
# spend a one-way remedy on a blip. Three DISTINCT polls is the same shape as
|
|
562
|
+
# the O14 context floor -- long enough that a blip cannot reach it, short
|
|
563
|
+
# enough that the item drains within a poll interval or three rather than
|
|
564
|
+
# staying in the ledger for hours.
|
|
565
|
+
#
|
|
566
|
+
# Counted in POLLS and not in seconds, unlike `CLOSELOOP_RETRY_FLOOR_SECONDS`:
|
|
567
|
+
# the point of that floor is to stop hammering an endpoint that might recover,
|
|
568
|
+
# and this path issues exactly one read per poll and then stops touching Alissa
|
|
569
|
+
# at all, so there is nothing to throttle -- while an hours-long clock would
|
|
570
|
+
# leave a known-dead item generating denied writes for those hours.
|
|
571
|
+
ORIGIN_MISSING_MAX_ATTEMPTS = 3
|
|
572
|
+
|
|
573
|
+
# NOT an operator page: one row per poll that saw the origin task 404, so the
|
|
574
|
+
# count of CONSECUTIVE failed reads survives a restart and, crucially, ADVANCES
|
|
575
|
+
# AT ALL.
|
|
576
|
+
#
|
|
577
|
+
# This is the ledger the incident was missing. The failure used to be counted
|
|
578
|
+
# by the `closeloop-failed:a<j>` page rows -- and those are recorded by
|
|
579
|
+
# `_escalate` only AFTER its primary comment lands, on the Alissa task. When the
|
|
580
|
+
# task is the artifact that 404'd, that comment is denied (NO_ACCESS), no row is
|
|
581
|
+
# written, and the next poll recomputes attempt 1 from an empty ledger: the
|
|
582
|
+
# production log showed `(attempt 1)` on every poll for four days. So this row
|
|
583
|
+
# is written by the loop itself, before anything can refuse it -- a counter
|
|
584
|
+
# whose advance depends on a remote write is not a counter.
|
|
585
|
+
#
|
|
586
|
+
# `marker:` for the reason the other markers carry it: `escalations_matching` is
|
|
587
|
+
# a prefix reader, and these rows must stay out of every prefix read of a page
|
|
588
|
+
# kind.
|
|
589
|
+
#
|
|
590
|
+
# CLEARED THE MOMENT A READ ANSWERS (`_origin_answered`), which is what makes
|
|
591
|
+
# the count CONSECUTIVE rather than lifetime-cumulative -- the distinction the
|
|
592
|
+
# round-1 `[major]` turned on. Cumulative counting reaches the bound on three
|
|
593
|
+
# unrelated blips spread over days, and the item it then abandons for good may
|
|
594
|
+
# be perfectly alive: an item whose close-loop WRITE keeps failing stays in the
|
|
595
|
+
# ledger for days on the retry floor, and every one of those retries is another
|
|
596
|
+
# chance to catch a 404 that means nothing. Abandonment is irreversible and
|
|
597
|
+
# self-silencing, so the evidence for it has to be an unbroken run -- and the
|
|
598
|
+
# page that announces it says "consecutive", which a lifetime count would make
|
|
599
|
+
# a false claim. The audit-trail argument for keeping spent rows loses to that:
|
|
600
|
+
# the same argument applies to `marker:dedupe-unreadable`, which is cleared for
|
|
601
|
+
# exactly this reason (a blip that clears must not make the next outage look
|
|
602
|
+
# days old).
|
|
603
|
+
ORIGIN_MISSING_ATTEMPT = "marker:origin-missing"
|
|
604
|
+
|
|
525
605
|
# The close edge could not write its evidence AT ALL -- not the enriched
|
|
526
606
|
# delivery record, not the bare confirmation line -- for longer than the
|
|
527
607
|
# close-loop retry floor, so the hand-off completed WITHOUT evidence (O17,
|
|
@@ -674,6 +754,43 @@ def closeloop_failed_kind(attempt: int) -> str:
|
|
|
674
754
|
return f"{ESCALATION_CLOSELOOP_FAILED}:a{attempt}"
|
|
675
755
|
|
|
676
756
|
|
|
757
|
+
def closeloop_terminal_kind(reason: str) -> str:
|
|
758
|
+
"""The ledger kind recording that the close edge gave up on an item, with
|
|
759
|
+
the REASON in the key (`closeloop-terminal:<reason>`).
|
|
760
|
+
|
|
761
|
+
Folding the reason in rather than storing it beside the row keeps the
|
|
762
|
+
ledger's two-column shape and makes the row self-describing: the terminal
|
|
763
|
+
short-circuit reads the reason straight back out of the kind it matched, so
|
|
764
|
+
the operator-facing skip line names the cause that was actually recorded."""
|
|
765
|
+
return f"{ESCALATION_CLOSELOOP_TERMINAL}:{reason}"
|
|
766
|
+
|
|
767
|
+
|
|
768
|
+
def closeloop_terminal_reason(kind: str) -> str:
|
|
769
|
+
"""The reason back out of a `closeloop-terminal:<reason>` kind. Empty for a
|
|
770
|
+
row with no reason at all -- which cannot be written by the helper above,
|
|
771
|
+
but a ledger is a file on disk and reading it defensively costs nothing."""
|
|
772
|
+
return kind.partition(":")[2]
|
|
773
|
+
|
|
774
|
+
|
|
775
|
+
def truncate_error(exc: object, limit: int) -> str:
|
|
776
|
+
"""An exception's text, capped at `limit` characters with an explicit
|
|
777
|
+
marker. The marker matters more than the cap: a silently shortened error
|
|
778
|
+
message reads like the endpoint's own words."""
|
|
779
|
+
text = str(exc)
|
|
780
|
+
if len(text) <= limit:
|
|
781
|
+
return text
|
|
782
|
+
return text[:limit].rstrip() + f" […truncated at {limit} characters]"
|
|
783
|
+
|
|
784
|
+
|
|
785
|
+
def origin_missing_attempt_kind(attempt: int) -> str:
|
|
786
|
+
"""The ledger row for ONE poll that found the origin task unreadable
|
|
787
|
+
(`marker:origin-missing:a<j>`). Per-attempt keying, like
|
|
788
|
+
`closeloop_failed_kind`, so the row count IS the attempt number under the
|
|
789
|
+
ledger's idempotent (task, kind) insert -- two writes of the same attempt
|
|
790
|
+
can never inflate it, and a restart cannot reset it."""
|
|
791
|
+
return f"{ORIGIN_MISSING_ATTEMPT}:a{attempt}"
|
|
792
|
+
|
|
793
|
+
|
|
677
794
|
# --- escalation copy -------------------------------------------------------
|
|
678
795
|
|
|
679
796
|
UNRELEASABLE_COMMENT = (
|
|
@@ -773,6 +890,53 @@ CLOSELOOP_FAILED_ISSUE_COMMENT = (
|
|
|
773
890
|
"Retrying; operator help may be needed."
|
|
774
891
|
)
|
|
775
892
|
|
|
893
|
+
# The GitHub-issue page that ENDS a close loop whose origin task is gone (O22,
|
|
894
|
+
# issue #58). It has no task-comment twin, and that absence is the whole design:
|
|
895
|
+
# the artifact this page is about is the one artifact that cannot receive it. A
|
|
896
|
+
# comment on an archived task is refused with NO_ACCESS, and a refusal whose
|
|
897
|
+
# target already answered 404 to a READ is permanent by construction -- so the
|
|
898
|
+
# GitHub issue, which this daemon can always write, is the PRIMARY channel here
|
|
899
|
+
# rather than the best-effort second copy it is everywhere else.
|
|
900
|
+
#
|
|
901
|
+
# Tense, by the O17 rule (a page posted before its act may not use the past
|
|
902
|
+
# tense): this is posted BEFORE the ledger row that makes the give-up stick, and
|
|
903
|
+
# the row is only written once the page has landed, so everything terminal is
|
|
904
|
+
# stated as what the orchestrator WILL do. The one past-tense claim it makes --
|
|
905
|
+
# the reads that already failed -- is the one thing that has already happened.
|
|
906
|
+
#
|
|
907
|
+
# It also states what is NOT touched. The trigger label may still be sitting on
|
|
908
|
+
# the issue, and leaving it is deliberate (parking is the operator's call, the
|
|
909
|
+
# same rule the park path holds), so the page says so rather than leaving a
|
|
910
|
+
# reader to wonder whether a labelled issue is still claimed.
|
|
911
|
+
#
|
|
912
|
+
# The `{error}` it embeds is TRUNCATED to `ORIGIN_MISSING_ERROR_MAX_CHARS`. This
|
|
913
|
+
# is the first API error text this daemon renders onto a GITHUB issue -- every
|
|
914
|
+
# other `{error}` goes to the Alissa task, and the one pre-existing issue-only
|
|
915
|
+
# page embeds no exception at all -- and its length and content are whatever the
|
|
916
|
+
# endpoint chose to return. In practice it is one benign line; the cap is there
|
|
917
|
+
# so a page on a durable, more widely readable surface stays predictable however
|
|
918
|
+
# the endpoint's error shape changes. Truncated rather than replaced by the
|
|
919
|
+
# status class: "Task not found or not accessible" is the sentence that tells an
|
|
920
|
+
# operator what happened, and dropping it to render `404` would cost the reader
|
|
921
|
+
# the one useful part.
|
|
922
|
+
ORIGIN_MISSING_ERROR_MAX_CHARS = 300
|
|
923
|
+
ORIGIN_MISSING_ISSUE_COMMENT = (
|
|
924
|
+
"**Orchestrator: close loop abandoned — origin task unreachable** — this "
|
|
925
|
+
"issue is closed by a merged pull request, but its origin Alissa task "
|
|
926
|
+
"{ref} could not be read on {attempts} consecutive polls: {error}. A task "
|
|
927
|
+
"that answers 404 to the orchestrator's own token is gone in practice — "
|
|
928
|
+
"archived after validation (the workspace's auto-archive window) or "
|
|
929
|
+
"deleted — and nothing about that recovers on a retry. So the orchestrator "
|
|
930
|
+
"is abandoning the close loop for this issue: it will make no further "
|
|
931
|
+
"reads or writes against {ref}, and this is the only comment it will post "
|
|
932
|
+
"about this failure. Nothing here is being changed — the merge stands, and "
|
|
933
|
+
"the `{label}` label, if it is still on this issue, is left exactly as it "
|
|
934
|
+
"is (parking is the operator's call, never the orchestrator's). If that "
|
|
935
|
+
"task still exists and still matters, close its loop by hand: satisfy its "
|
|
936
|
+
"validation criteria, attach this issue's merged pull request as evidence, "
|
|
937
|
+
"and move it to `pending_validation`."
|
|
938
|
+
)
|
|
939
|
+
|
|
776
940
|
# The TERMINAL page of the close edge (O17, issue #44). Everything a page has to
|
|
777
941
|
# say when the daemon stops: what it could not do, what it is doing anyway, that
|
|
778
942
|
# it will not try that again, and what is left for a human. The last part
|
|
@@ -1549,7 +1713,7 @@ class Orchestrator:
|
|
|
1549
1713
|
self,
|
|
1550
1714
|
task_number: int,
|
|
1551
1715
|
kind: str,
|
|
1552
|
-
task_comment: str,
|
|
1716
|
+
task_comment: str | None,
|
|
1553
1717
|
issue: Issue | None = None,
|
|
1554
1718
|
issue_comment: str | None = None,
|
|
1555
1719
|
) -> bool:
|
|
@@ -1570,27 +1734,67 @@ class Orchestrator:
|
|
|
1570
1734
|
every future attempt AND every future page) can gate on the receipt
|
|
1571
1735
|
instead of re-reading the ledger and hoping the two stay in step. Every
|
|
1572
1736
|
pre-existing caller ignores it, which is unchanged behaviour: a page
|
|
1573
|
-
that could not be posted was already retried next poll.
|
|
1737
|
+
that could not be posted was already retried next poll.
|
|
1738
|
+
|
|
1739
|
+
`task_comment=None` DECLARES THE TASK CHANNEL UNREACHABLE (O22, issue
|
|
1740
|
+
#58) and moves the primary role to the GitHub issue: the Alissa comment
|
|
1741
|
+
is not attempted at all, and the issue comment's delivery becomes the
|
|
1742
|
+
receipt the dedupe row is recorded behind. The one caller is the page
|
|
1743
|
+
about a task whose own READ answered 404 -- writing a comment to it
|
|
1744
|
+
would be refused with NO_ACCESS, which is exactly what the loop did for
|
|
1745
|
+
four days: a denied write per poll per item, no row recorded, and an
|
|
1746
|
+
attempt counter frozen at 1 forever. A refusal whose target already
|
|
1747
|
+
404'd on a read is permanent by construction, so the right number of
|
|
1748
|
+
attempts at it is zero. Carrying the decision in the copy argument
|
|
1749
|
+
rather than a separate flag is deliberate: there is then no way to
|
|
1750
|
+
declare the channel dead and still pass it a comment that would be
|
|
1751
|
+
silently dropped. With no issue to write either there is no channel at
|
|
1752
|
+
all, and the page cannot be delivered -- so nothing is recorded and the
|
|
1753
|
+
caller is told, rather than latching a page nobody will ever see."""
|
|
1574
1754
|
if self.state.escalated(task_number, kind):
|
|
1575
1755
|
return True
|
|
1576
1756
|
ref = f"TASK-{task_number}"
|
|
1577
1757
|
if self.config.dry_run:
|
|
1578
|
-
log.info(
|
|
1758
|
+
log.info(
|
|
1759
|
+
"[dry-run] would escalate %s on %s:\n%s",
|
|
1760
|
+
kind, ref, task_comment if task_comment is not None else issue_comment,
|
|
1761
|
+
)
|
|
1579
1762
|
return False
|
|
1580
1763
|
|
|
1581
|
-
|
|
1582
|
-
|
|
1583
|
-
except AlissaError as exc:
|
|
1764
|
+
issue_channel = issue is not None and issue_comment is not None
|
|
1765
|
+
if task_comment is None and not issue_channel:
|
|
1584
1766
|
log.error(
|
|
1585
|
-
"
|
|
1586
|
-
"
|
|
1767
|
+
"%s escalation on %s has no reachable channel (the task is "
|
|
1768
|
+
"unreadable and there is no issue to comment on) — not "
|
|
1769
|
+
"recording", kind, ref,
|
|
1587
1770
|
)
|
|
1588
1771
|
return False
|
|
1589
1772
|
|
|
1773
|
+
if task_comment is not None:
|
|
1774
|
+
try:
|
|
1775
|
+
self.alissa.add_comment(ref, task_comment)
|
|
1776
|
+
except AlissaError as exc:
|
|
1777
|
+
log.error(
|
|
1778
|
+
"could not post %s escalation on %s: %s — not recording; "
|
|
1779
|
+
"retries next poll", kind, ref, exc,
|
|
1780
|
+
)
|
|
1781
|
+
return False
|
|
1782
|
+
|
|
1590
1783
|
if issue is not None and issue_comment is not None:
|
|
1591
1784
|
try:
|
|
1592
1785
|
self.github.comment(issue.owner, issue.repo, issue.number, issue_comment)
|
|
1593
1786
|
except (CommandError, RateLimited) as exc:
|
|
1787
|
+
if task_comment is None:
|
|
1788
|
+
# PRIMARY here, not best-effort: with the task channel dead
|
|
1789
|
+
# this is the only place the operator can be told, so a
|
|
1790
|
+
# failure to post is a failure to page and must not record.
|
|
1791
|
+
log.error(
|
|
1792
|
+
"could not post %s escalation on %s (the task is "
|
|
1793
|
+
"unreadable, so the issue %s is the only channel): %s "
|
|
1794
|
+
"— not recording; retries next poll",
|
|
1795
|
+
kind, ref, issue.issue_slug, exc,
|
|
1796
|
+
)
|
|
1797
|
+
return False
|
|
1594
1798
|
log.warning(
|
|
1595
1799
|
"best-effort issue comment for %s failed on %s: %s",
|
|
1596
1800
|
kind, issue.issue_slug, exc,
|
|
@@ -3390,6 +3594,30 @@ class Orchestrator:
|
|
|
3390
3594
|
task_number,
|
|
3391
3595
|
))
|
|
3392
3596
|
|
|
3597
|
+
# The close edge gave up on this item for good (O22, issue #58). Same
|
|
3598
|
+
# short-circuit and the same reasoning as `closed-unmerged` above --
|
|
3599
|
+
# before the GitHub round-trips, because the decision can never change
|
|
3600
|
+
# -- and here it also carries the promise the terminal page made: no
|
|
3601
|
+
# further Alissa reads or writes for this item, on any pass. The reason
|
|
3602
|
+
# comes out of the ledger row rather than being re-derived, so the line
|
|
3603
|
+
# an operator reads names the cause that was actually recorded.
|
|
3604
|
+
terminal = self.state.escalations_matching(
|
|
3605
|
+
task_number, ESCALATION_CLOSELOOP_TERMINAL + ":"
|
|
3606
|
+
)
|
|
3607
|
+
if terminal:
|
|
3608
|
+
# `[0]` is the OLDEST row, which `escalations_matching` now
|
|
3609
|
+
# guarantees: only one reason can exist today, but the kind is a
|
|
3610
|
+
# prefix scan precisely so reasons can be added, and the day a
|
|
3611
|
+
# second one lands this line must name the one that actually ended
|
|
3612
|
+
# the loop rather than whichever row the database returned first.
|
|
3613
|
+
reason = closeloop_terminal_reason(str(terminal[0]["kind"])) or "unspecified"
|
|
3614
|
+
return self._logged(ref, Decision(
|
|
3615
|
+
Action.SKIPPED,
|
|
3616
|
+
f"{slug}: close loop abandoned ({reason}) — terminal, already "
|
|
3617
|
+
f"escalated",
|
|
3618
|
+
task_number,
|
|
3619
|
+
))
|
|
3620
|
+
|
|
3393
3621
|
try:
|
|
3394
3622
|
issue = self.github.issue(owner, repo, number)
|
|
3395
3623
|
except (CommandError, RateLimited) as exc:
|
|
@@ -3781,8 +4009,16 @@ class Orchestrator:
|
|
|
3781
4009
|
# silently untouched. A refused/partial answer arrives as the same
|
|
3782
4010
|
# `AlissaError` a failed read does and takes the same retry path.
|
|
3783
4011
|
task = self.alissa.get_task(ref, view=VIEW_FULL)
|
|
4012
|
+
except AlissaNotFound as exc:
|
|
4013
|
+
# The origin task is not merely refusing this write -- it cannot be
|
|
4014
|
+
# READ. Routed away from `_closeloop_failed` (O22, issue #58),
|
|
4015
|
+
# because that path's whole remedy is "page the task and retry",
|
|
4016
|
+
# and both halves are wrong here: the page targets the artifact
|
|
4017
|
+
# that just 404'd, and the retry has nothing left to succeed at.
|
|
4018
|
+
return self._origin_missing(task_number, ref, slug, issue, exc)
|
|
3784
4019
|
except AlissaError as exc:
|
|
3785
4020
|
return self._closeloop_failed(task_number, slug, issue, exc, prior)
|
|
4021
|
+
self._origin_answered(task_number, ref)
|
|
3786
4022
|
|
|
3787
4023
|
if task.status not in CLOSE_EXPECTED_STATES:
|
|
3788
4024
|
if task.status == CLOSE_HANDOFF_STATE:
|
|
@@ -4037,6 +4273,177 @@ class Orchestrator:
|
|
|
4037
4273
|
slug, ref, failing_for, exc,
|
|
4038
4274
|
)
|
|
4039
4275
|
|
|
4276
|
+
def _origin_answered(self, task_number: int, ref: str) -> None:
|
|
4277
|
+
"""The origin read answered, so any run of failed reads is OVER: drop
|
|
4278
|
+
the attempt rows.
|
|
4279
|
+
|
|
4280
|
+
This is what makes `_origin_missing`'s bound a count of CONSECUTIVE
|
|
4281
|
+
polls. Without it the rows are cumulative over the item's lifetime, and
|
|
4282
|
+
three unrelated 404 blips -- spread over days, on an item the close
|
|
4283
|
+
loop is retrying for an unrelated reason -- abandon a task that is
|
|
4284
|
+
alive and readable, permanently and silently. Same shape and same
|
|
4285
|
+
reasoning as `_dedupe_answered`, including the read-before-write guard
|
|
4286
|
+
so the ordinary path (no rows) does not write on every close pass.
|
|
4287
|
+
|
|
4288
|
+
Deliberately NOT gated on `dry_run`: a dry run cannot reach
|
|
4289
|
+
`_origin_missing` at all (`_close_loop` returns before the read), so
|
|
4290
|
+
there are no rows for it to clear and no side effect to suppress. The
|
|
4291
|
+
guard below means it does not even query."""
|
|
4292
|
+
if not self.state.escalations_matching(task_number, ORIGIN_MISSING_ATTEMPT + ":"):
|
|
4293
|
+
return
|
|
4294
|
+
cleared = self.state.clear_escalations_matching(
|
|
4295
|
+
task_number, ORIGIN_MISSING_ATTEMPT
|
|
4296
|
+
)
|
|
4297
|
+
log.info(
|
|
4298
|
+
"%s: the origin task reads again — clearing %d unreadable-attempt "
|
|
4299
|
+
"row(s); the give-up counter starts from zero", ref, cleared,
|
|
4300
|
+
)
|
|
4301
|
+
|
|
4302
|
+
def _origin_missing(
|
|
4303
|
+
self, task_number: int, ref: str, slug: str, issue: Issue, exc: AlissaNotFound
|
|
4304
|
+
) -> tuple[str, Decision]:
|
|
4305
|
+
"""The origin task's own read answered 404. Retry a bounded number of
|
|
4306
|
+
polls, then give up on the item for good -- ledgered, paged once on the
|
|
4307
|
+
GitHub issue, and never read or written again.
|
|
4308
|
+
|
|
4309
|
+
THE INCIDENT THIS ENDS (issue #58, root-caused 2026-08-04). Four
|
|
4310
|
+
released-and-merged items whose origin tasks had been validated and
|
|
4311
|
+
then auto-archived sat in the ledger for days. Each poll: `get_task`
|
|
4312
|
+
404s, `_closeloop_failed` pages `closeloop-failed:a<j>` -- whose PRIMARY
|
|
4313
|
+
channel is a comment on that same archived task -- the write is denied
|
|
4314
|
+
`NO_ACCESS`, `_escalate` therefore records nothing, and the next poll
|
|
4315
|
+
recomputes attempt 1 from an empty ledger. Neither the read nor the
|
|
4316
|
+
denied write was terminal, so the pair repeated every ~80s per item,
|
|
4317
|
+
forever, and the set grew as more closed loops aged past the archive
|
|
4318
|
+
window. Three separate things had to be true for that to happen, and
|
|
4319
|
+
this method contradicts all three:
|
|
4320
|
+
|
|
4321
|
+
* **The counter did not advance**, because the only thing counting was
|
|
4322
|
+
a row written behind a remote comment that could not land. The
|
|
4323
|
+
attempt rows here are written by the loop itself, first, before
|
|
4324
|
+
anything can refuse them -- so the count is a fact about polls, not
|
|
4325
|
+
about writes.
|
|
4326
|
+
* **The escalation targeted the unreachable artifact.** With the task
|
|
4327
|
+
channel declared dead (`task_comment=None`), the page goes to the
|
|
4328
|
+
GitHub issue, which this daemon can always write, and the daemon
|
|
4329
|
+
never attempts a write it knows will be denied.
|
|
4330
|
+
* **Nothing was terminal.** A permanently missing task cannot be
|
|
4331
|
+
closed by anyone, so the loop for it is over; the only open question
|
|
4332
|
+
is whether a human was told. The ledger row says so, and the
|
|
4333
|
+
short-circuit in `_evaluate_close` reads it before touching anything.
|
|
4334
|
+
|
|
4335
|
+
Why the bound is polls and not a single 404: see
|
|
4336
|
+
`ORIGIN_MISSING_MAX_ATTEMPTS`. Why only `AlissaNotFound` gets here: a
|
|
4337
|
+
403 is the OTHER shape, and it is operator-fixable (a role can be
|
|
4338
|
+
re-granted, and the existing paged-and-floored retry is the right
|
|
4339
|
+
answer for it); a 404 to the token that created the task is not
|
|
4340
|
+
something an operator can grant their way out of.
|
|
4341
|
+
|
|
4342
|
+
Why the give-up gates on the page having LANDED, exactly like
|
|
4343
|
+
`_close_evidence_unwritable`: the ledger row stops every future poll
|
|
4344
|
+
from looking at this item, so recording it behind an undelivered page
|
|
4345
|
+
would make the item disappear in silence -- which is the same class of
|
|
4346
|
+
bug as the storm, merely quieter. No row, no give-up, retry next poll.
|
|
4347
|
+
|
|
4348
|
+
Why the attempts have to be CONSECUTIVE, and are (`_origin_answered`
|
|
4349
|
+
clears the rows the moment a read answers): the remedy is one-way, so
|
|
4350
|
+
the evidence for it must be an unbroken run. Counting cumulatively
|
|
4351
|
+
reaches the bound on three unrelated blips days apart -- reachable
|
|
4352
|
+
through this file's own retry machinery, since an item whose close-loop
|
|
4353
|
+
WRITE keeps failing sits in the ledger for days on
|
|
4354
|
+
`CLOSELOOP_RETRY_FLOOR_SECONDS`, offering one more chance per hour to
|
|
4355
|
+
catch a meaningless 404 -- and would abandon a live, readable task
|
|
4356
|
+
under a page claiming it was gone.
|
|
4357
|
+
|
|
4358
|
+
Why the give-up RETRY is floored once the bound is reached: while the
|
|
4359
|
+
page cannot be delivered nothing settles, so without a floor this path
|
|
4360
|
+
would write a row and attempt a GitHub comment on every poll, forever
|
|
4361
|
+
-- an unbounded retry generating a failed remote write per poll, which
|
|
4362
|
+
is the shape issue #58 exists to kill, merely moved to the other
|
|
4363
|
+
channel. The floor is `CLOSELOOP_RETRY_FLOOR_SECONDS`, the same clock
|
|
4364
|
+
`_close_loop` puts between failed mutations, and it applies ONLY past
|
|
4365
|
+
the bound: the first attempts must stay poll-paced or a known-dead item
|
|
4366
|
+
would take hours to drain instead of the polls the issue asks for.
|
|
4367
|
+
|
|
4368
|
+
What is NOT done here: the trigger label is not touched (parking stays
|
|
4369
|
+
the operator's call, the rule the park path holds), the issue is not
|
|
4370
|
+
closed or re-opened, and no Alissa write of any kind is attempted."""
|
|
4371
|
+
prior = self.state.escalations_matching(task_number, ORIGIN_MISSING_ATTEMPT + ":")
|
|
4372
|
+
if len(prior) >= ORIGIN_MISSING_MAX_ATTEMPTS:
|
|
4373
|
+
last_at = max(int(row["escalated_at"]) for row in prior)
|
|
4374
|
+
waited = self._now() - last_at
|
|
4375
|
+
if waited < CLOSELOOP_RETRY_FLOOR_SECONDS:
|
|
4376
|
+
return self._logged(ref, Decision(
|
|
4377
|
+
Action.HELD,
|
|
4378
|
+
f"{slug}: origin task unreadable and the terminal page has "
|
|
4379
|
+
f"not been delivered; next delivery attempt floored "
|
|
4380
|
+
f"({len(prior)} attempt(s) so far)",
|
|
4381
|
+
task_number,
|
|
4382
|
+
))
|
|
4383
|
+
attempt = len(prior) + 1
|
|
4384
|
+
# `at=` the loop's own clock, not the ledger's `time.time()` default:
|
|
4385
|
+
# the floor above MEASURES against this timestamp, and the state
|
|
4386
|
+
# module's own docstring names that as the condition for passing one.
|
|
4387
|
+
self.state.record_escalation(
|
|
4388
|
+
task_number, origin_missing_attempt_kind(attempt), at=int(self._now()),
|
|
4389
|
+
)
|
|
4390
|
+
|
|
4391
|
+
if attempt < ORIGIN_MISSING_MAX_ATTEMPTS:
|
|
4392
|
+
log.warning(
|
|
4393
|
+
"%s: origin task %s could not be READ (%s) — attempt %d of %d; "
|
|
4394
|
+
"retrying next poll before giving up on this item",
|
|
4395
|
+
slug, ref, exc, attempt, ORIGIN_MISSING_MAX_ATTEMPTS,
|
|
4396
|
+
)
|
|
4397
|
+
return self._logged(ref, Decision(
|
|
4398
|
+
Action.HELD,
|
|
4399
|
+
f"{slug}: origin task unreadable (attempt {attempt} of "
|
|
4400
|
+
f"{ORIGIN_MISSING_MAX_ATTEMPTS}): {exc}",
|
|
4401
|
+
task_number,
|
|
4402
|
+
))
|
|
4403
|
+
|
|
4404
|
+
delivered = self._escalate(
|
|
4405
|
+
task_number,
|
|
4406
|
+
closeloop_terminal_kind(CLOSELOOP_TERMINAL_ORIGIN_ARCHIVED),
|
|
4407
|
+
None, # the task channel IS the artifact that 404'd
|
|
4408
|
+
issue,
|
|
4409
|
+
ORIGIN_MISSING_ISSUE_COMMENT.format(
|
|
4410
|
+
ref=ref, attempts=attempt, label=self.config.label,
|
|
4411
|
+
error=truncate_error(exc, ORIGIN_MISSING_ERROR_MAX_CHARS),
|
|
4412
|
+
),
|
|
4413
|
+
)
|
|
4414
|
+
if not delivered:
|
|
4415
|
+
log.error(
|
|
4416
|
+
"%s: giving up on the close loop for %s, but the terminal page "
|
|
4417
|
+
"could not be delivered to the issue — NOT recording the "
|
|
4418
|
+
"give-up; retries next poll", slug, ref,
|
|
4419
|
+
)
|
|
4420
|
+
return self._logged(ref, Decision(
|
|
4421
|
+
Action.HELD,
|
|
4422
|
+
f"{slug}: origin task unreadable and the terminal page could "
|
|
4423
|
+
f"not be posted — not settling",
|
|
4424
|
+
task_number,
|
|
4425
|
+
))
|
|
4426
|
+
|
|
4427
|
+
self._wp_closure(
|
|
4428
|
+
task_number=task_number, ref=ref, repo=issue.full_name,
|
|
4429
|
+
issue_number=issue.number,
|
|
4430
|
+
terminal_reason=CLOSELOOP_TERMINAL_ORIGIN_ARCHIVED,
|
|
4431
|
+
)
|
|
4432
|
+
log.error(
|
|
4433
|
+
"%s: GAVE UP on the close loop for %s after %d attempts — the "
|
|
4434
|
+
"origin task is permanently unreadable (%s); the issue was "
|
|
4435
|
+
"commented once and this item is now terminal (%s): no further "
|
|
4436
|
+
"Alissa reads or writes",
|
|
4437
|
+
slug, ref, attempt, exc, CLOSELOOP_TERMINAL_ORIGIN_ARCHIVED,
|
|
4438
|
+
)
|
|
4439
|
+
return self._logged(ref, Decision(
|
|
4440
|
+
Action.ESCALATED,
|
|
4441
|
+
f"{slug}: origin task permanently unreadable after {attempt} "
|
|
4442
|
+
f"attempts — close loop abandoned "
|
|
4443
|
+
f"({CLOSELOOP_TERMINAL_ORIGIN_ARCHIVED}), issue commented once",
|
|
4444
|
+
task_number,
|
|
4445
|
+
))
|
|
4446
|
+
|
|
4040
4447
|
def _closeloop_failed(
|
|
4041
4448
|
self, task_number: int, slug: str, issue: Issue, exc: Exception, prior: list
|
|
4042
4449
|
) -> tuple[str, Decision]:
|
|
@@ -67,15 +67,21 @@ CREATE TABLE IF NOT EXISTS meta (
|
|
|
67
67
|
"""
|
|
68
68
|
|
|
69
69
|
|
|
70
|
-
# The only `escalations` kinds
|
|
70
|
+
# The only `escalations` kinds the clear methods will remove: two outage clocks
|
|
71
71
|
# and the two pages they raise -- the O9 dedupe check and the O14 context read --
|
|
72
|
-
# all
|
|
73
|
-
# Spelled as literals here rather than
|
|
74
|
-
# module; `test_loop` pins that the
|
|
75
|
-
#
|
|
72
|
+
# plus the O22 origin-missing attempt counter, all of which describe a condition
|
|
73
|
+
# that can END (see `clear_escalation`). Spelled as literals here rather than
|
|
74
|
+
# imported from `loop`, which imports this module; `test_loop` pins that the
|
|
75
|
+
# loop's constants and this set stay the same five strings.
|
|
76
|
+
#
|
|
77
|
+
# `marker:origin-missing` is the one entry that names a PREFIX rather than a
|
|
78
|
+
# whole kind: its rows are `marker:origin-missing:a<j>`, one per poll, and they
|
|
79
|
+
# are cleared as a family by `clear_escalations_matching` -- a counter of
|
|
80
|
+
# CONSECUTIVE failures is only consecutive if a success resets it.
|
|
76
81
|
CLEARABLE_KINDS = frozenset({
|
|
77
82
|
"marker:dedupe-unreadable", "dedupe-unreadable",
|
|
78
83
|
"marker:context-unreadable", "context-unreadable",
|
|
84
|
+
"marker:origin-missing",
|
|
79
85
|
})
|
|
80
86
|
|
|
81
87
|
|
|
@@ -208,20 +214,57 @@ class State:
|
|
|
208
214
|
self._db.commit()
|
|
209
215
|
return cursor.rowcount > 0
|
|
210
216
|
|
|
217
|
+
@staticmethod
|
|
218
|
+
def _like_prefix(prefix: str) -> str:
|
|
219
|
+
"""`prefix` as a LIKE pattern, with `%`/`_`/`\\` escaped so a literal
|
|
220
|
+
wildcard in a kind cannot widen the match."""
|
|
221
|
+
escaped = prefix.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
|
|
222
|
+
return escaped + "%"
|
|
223
|
+
|
|
211
224
|
def escalations_matching(self, task_number: int, prefix: str) -> "list[sqlite3.Row]":
|
|
212
|
-
"""Escalation rows for this task whose `kind` starts with `prefix
|
|
225
|
+
"""Escalation rows for this task whose `kind` starts with `prefix`,
|
|
226
|
+
OLDEST FIRST.
|
|
213
227
|
|
|
214
228
|
The close-loop retry uses this to count prior `closeloop-failed:a<j>`
|
|
215
229
|
attempts (the row count IS the last attempt number) and to floor the
|
|
216
|
-
next retry off the newest `escalated_at`.
|
|
217
|
-
|
|
218
|
-
|
|
230
|
+
next retry off the newest `escalated_at`.
|
|
231
|
+
|
|
232
|
+
The ordering is part of the contract, not an implementation detail: a
|
|
233
|
+
caller that reads ONE row out of a multi-row family (the O22 terminal
|
|
234
|
+
reason, `rows[0]`) would otherwise name whichever row SQLite happened to
|
|
235
|
+
return first, and the failure would be a silently wrong log line rather
|
|
236
|
+
than an error. Oldest-first is the useful order for every caller here --
|
|
237
|
+
the first row is the observation that started the condition."""
|
|
219
238
|
return self._db.execute(
|
|
220
239
|
"SELECT task_number, kind, escalated_at FROM escalations "
|
|
221
|
-
"WHERE task_number=? AND kind LIKE ? ESCAPE '\\'"
|
|
222
|
-
|
|
240
|
+
"WHERE task_number=? AND kind LIKE ? ESCAPE '\\' "
|
|
241
|
+
"ORDER BY escalated_at ASC, kind ASC",
|
|
242
|
+
(task_number, self._like_prefix(prefix)),
|
|
223
243
|
).fetchall()
|
|
224
244
|
|
|
245
|
+
def clear_escalations_matching(self, task_number: int, prefix: str) -> int:
|
|
246
|
+
"""Forget every row of a clearable FAMILY at once; returns how many went.
|
|
247
|
+
|
|
248
|
+
The plural of `clear_escalation`, for a condition whose rows are keyed
|
|
249
|
+
per attempt (`marker:origin-missing:a<j>`) rather than by a single kind.
|
|
250
|
+
The allowlist check is on the PREFIX, so a family is clearable as a
|
|
251
|
+
whole or not at all -- a caller cannot reach a page kind by handing this
|
|
252
|
+
a prefix of it, and `CLEARABLE_KINDS` stays the one place that decides
|
|
253
|
+
what may be forgotten."""
|
|
254
|
+
if prefix not in CLEARABLE_KINDS:
|
|
255
|
+
raise ValueError(
|
|
256
|
+
f"{prefix!r} is not a clearable family: only "
|
|
257
|
+
f"{sorted(CLEARABLE_KINDS)} are. Escalation rows are an audit "
|
|
258
|
+
"trail; a condition that ends needs its own clearable kind, "
|
|
259
|
+
"not the deletion of a page."
|
|
260
|
+
)
|
|
261
|
+
cursor = self._db.execute(
|
|
262
|
+
"DELETE FROM escalations WHERE task_number=? AND kind LIKE ? ESCAPE '\\'",
|
|
263
|
+
(task_number, self._like_prefix(prefix)),
|
|
264
|
+
)
|
|
265
|
+
self._db.commit()
|
|
266
|
+
return cursor.rowcount
|
|
267
|
+
|
|
225
268
|
# --- loop-level scratch ------------------------------------------------
|
|
226
269
|
|
|
227
270
|
def get_meta(self, key: str) -> "str | None":
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
0.4.2
|
|
@@ -255,6 +255,15 @@ class Closure:
|
|
|
255
255
|
merged: bool = False
|
|
256
256
|
merged_at: float | None = None
|
|
257
257
|
closed_loop: bool = False # the walk reached pending_validation
|
|
258
|
+
# Why the close edge STOPPED without reaching `pending_validation` (O22,
|
|
259
|
+
# issue #58) -- currently only `origin-archived`, a task the orchestrator
|
|
260
|
+
# can no longer read. Empty on every ordinary closure, and mutually
|
|
261
|
+
# exclusive with `closed_loop` by construction: a loop is either handed off
|
|
262
|
+
# or abandoned, never both. Recorded rather than rendered: the board's job
|
|
263
|
+
# is showing work that still needs a human, and an abandoned item's page is
|
|
264
|
+
# already on its GitHub issue -- so this is the projection's record of the
|
|
265
|
+
# disposition, for a view that wants it, not a new coupling to the store.
|
|
266
|
+
terminal_reason: str = ""
|
|
258
267
|
delivery_notes: str = ""
|
|
259
268
|
at: float = 0.0
|
|
260
269
|
# Parsed ONCE, in `record_closure`, from `delivery_notes`. The body is
|
|
@@ -978,9 +987,25 @@ def _build_kpis(
|
|
|
978
987
|
c for c in closures
|
|
979
988
|
if c.merged and c.merged_at is not None and c.merged_at >= at - RECENT_SECONDS
|
|
980
989
|
]
|
|
990
|
+
# Counted off the SAME set the Validated column draws from: a status whose
|
|
991
|
+
# ref carries a `closed_loop` closure. The two share a label on one page,
|
|
992
|
+
# so they have to share a population as well as a clock (the clock was the
|
|
993
|
+
# other half, fixed in round 2).
|
|
994
|
+
#
|
|
995
|
+
# `_narrow` admits `depends_on` refs deliberately -- an upstream status
|
|
996
|
+
# riding inside a dependant's payload is the only free source of
|
|
997
|
+
# `validated` -- but such a ref has neither a member card nor a closure, so
|
|
998
|
+
# counting it here produced a tile the board could not account for. It
|
|
999
|
+
# still serves the narrowing it was added for; it just does not reach this
|
|
1000
|
+
# tile. Not lossy in practice: a task orcloop ever sees at `validated` is
|
|
1001
|
+
# one it drove, so it carries a closure until that closure ages out of the
|
|
1002
|
+
# window -- and once it has, the column could not show it either.
|
|
1003
|
+
settled = {c.ref for c in closures if c.closed_loop}
|
|
981
1004
|
validated = [
|
|
982
1005
|
s for s in statuses.values()
|
|
983
|
-
if s.status == VALIDATED_STATUS
|
|
1006
|
+
if s.status == VALIDATED_STATUS
|
|
1007
|
+
and s.at >= at - RECENT_SECONDS
|
|
1008
|
+
and s.ref in settled
|
|
984
1009
|
]
|
|
985
1010
|
return (
|
|
986
1011
|
Kpi(
|
|
@@ -54,9 +54,11 @@ from .workplan import (
|
|
|
54
54
|
# seen when some other task depends on it (see `workplan`'s module docstring).
|
|
55
55
|
VALIDATED_FOOTNOTE = (
|
|
56
56
|
"“Validated” counts tasks this daemon actually observed at `validated` "
|
|
57
|
-
"inside the window
|
|
58
|
-
"
|
|
59
|
-
"
|
|
57
|
+
"inside the window, and only those this loop itself closed — the tile and "
|
|
58
|
+
"the Validated column are drawn from one set. orcloop hands off at "
|
|
59
|
+
"`pending_validation` and stops polling a task, so a validation it never "
|
|
60
|
+
"had a reason to read is not counted here — this figure is a floor, not a "
|
|
61
|
+
"total."
|
|
60
62
|
)
|
|
61
63
|
|
|
62
64
|
LANE_GUIDE = (
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
0.4.0
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|