workproof 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- workproof/__init__.py +8 -0
- workproof/__main__.py +5 -0
- workproof/cli.py +96 -0
- workproof/core.py +303 -0
- workproof-0.1.0.dist-info/METADATA +136 -0
- workproof-0.1.0.dist-info/RECORD +10 -0
- workproof-0.1.0.dist-info/WHEEL +5 -0
- workproof-0.1.0.dist-info/entry_points.txt +2 -0
- workproof-0.1.0.dist-info/licenses/LICENSE +21 -0
- workproof-0.1.0.dist-info/top_level.txt +1 -0
workproof/__init__.py
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
from .core import (Finding, check, digest, is_empty, load_record, load_registry, proves_work,
|
|
2
|
+
read_evidence_file, record, scheduled_from_crontab, status, to_evidence, watch,
|
|
3
|
+
weekday_hours)
|
|
4
|
+
|
|
5
|
+
__version__ = '0.1.0'
|
|
6
|
+
__all__ = ['Finding', 'check', 'digest', 'is_empty', 'load_record', 'load_registry', 'proves_work',
|
|
7
|
+
'read_evidence_file', 'record', 'scheduled_from_crontab', 'status', 'to_evidence', 'watch',
|
|
8
|
+
'weekday_hours']
|
workproof/__main__.py
ADDED
workproof/cli.py
ADDED
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
"""Command line: `python -m workproof check|record`.
|
|
2
|
+
|
|
3
|
+
Exit codes: 0 healthy, 1 findings, 2 the checker itself failed. A monitor that
|
|
4
|
+
crashes quietly reproduces the bug it exists to catch, so crashes are loud and
|
|
5
|
+
distinct from findings.
|
|
6
|
+
"""
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import argparse
|
|
10
|
+
import json
|
|
11
|
+
import subprocess
|
|
12
|
+
import sys
|
|
13
|
+
|
|
14
|
+
from . import __version__, core
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def _scheduled(args):
|
|
18
|
+
if args.crontab_file:
|
|
19
|
+
with open(args.crontab_file, encoding='utf-8') as f:
|
|
20
|
+
return core.scheduled_from_crontab(f.read())
|
|
21
|
+
if args.crontab:
|
|
22
|
+
out = subprocess.run(['crontab', '-l'], capture_output=True, text=True, check=True).stdout
|
|
23
|
+
return core.scheduled_from_crontab(out)
|
|
24
|
+
return None
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def cmd_check(args) -> int:
|
|
28
|
+
registry = core.load_registry(args.registry)
|
|
29
|
+
findings = core.check(registry, args.store, scheduled=_scheduled(args))
|
|
30
|
+
if args.json:
|
|
31
|
+
print(json.dumps([f.as_dict() for f in findings], indent=2))
|
|
32
|
+
elif findings:
|
|
33
|
+
for f in findings:
|
|
34
|
+
print(f'{f.kind.upper():22} {f.agent}: {f.detail}')
|
|
35
|
+
else:
|
|
36
|
+
n = len(registry['agents'])
|
|
37
|
+
print(f'OK: {n} agent{"" if n == 1 else "s"}, all producing new work')
|
|
38
|
+
return 1 if findings else 0
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def cmd_status(args) -> int:
|
|
42
|
+
rows = core.status(core.load_registry(args.registry), args.store)
|
|
43
|
+
if args.json:
|
|
44
|
+
print(json.dumps(rows, indent=2))
|
|
45
|
+
return 0
|
|
46
|
+
fmt = lambda v: '-' if v is None else f'{v:g}h' # noqa: E731
|
|
47
|
+
print(f'{"AGENT":28} {"RUNS":>5} {"SINCE RUN":>10} {"SINCE NEW WORK":>15} {"LIMIT":>7}')
|
|
48
|
+
for r in rows:
|
|
49
|
+
print(f'{r["agent"]:28} {r["runs"]:>5} {fmt(r["last_run_h"]):>10} {fmt(r["last_change_h"]):>15} '
|
|
50
|
+
f'{r["limit_h"]:g}h'.rstrip())
|
|
51
|
+
return 0
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def cmd_record(args) -> int:
|
|
55
|
+
if args.file:
|
|
56
|
+
raw = core.read_evidence_file(args.file)
|
|
57
|
+
elif args.error:
|
|
58
|
+
raw = b'' # a failed run has no evidence; do not block waiting on stdin
|
|
59
|
+
else:
|
|
60
|
+
raw = sys.stdin.buffer.read()
|
|
61
|
+
core.record(args.agent, raw, args.store, error=args.error)
|
|
62
|
+
return 0
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def main(argv=None) -> int:
|
|
66
|
+
p = argparse.ArgumentParser(prog='workproof', description=__doc__.splitlines()[0])
|
|
67
|
+
p.add_argument('--version', action='version', version=f'workproof {__version__}')
|
|
68
|
+
sub = p.add_subparsers(dest='cmd', required=True)
|
|
69
|
+
|
|
70
|
+
c = sub.add_parser('check', help='compare recorded evidence with the registry')
|
|
71
|
+
c.add_argument('--registry', required=True)
|
|
72
|
+
c.add_argument('--store', default='.workproof')
|
|
73
|
+
c.add_argument('--json', action='store_true')
|
|
74
|
+
c.add_argument('--crontab', action='store_true', help='also check coverage against `crontab -l`')
|
|
75
|
+
c.add_argument('--crontab-file', help='also check coverage against a crontab text file')
|
|
76
|
+
c.set_defaults(fn=cmd_check)
|
|
77
|
+
|
|
78
|
+
s = sub.add_parser('status', help='show how long each agent has been quiet')
|
|
79
|
+
s.add_argument('--registry', required=True)
|
|
80
|
+
s.add_argument('--store', default='.workproof')
|
|
81
|
+
s.add_argument('--json', action='store_true')
|
|
82
|
+
s.set_defaults(fn=cmd_status)
|
|
83
|
+
|
|
84
|
+
r = sub.add_parser('record', help='record one run from the shell (for non-Python agents)')
|
|
85
|
+
r.add_argument('agent')
|
|
86
|
+
r.add_argument('--store', default='.workproof')
|
|
87
|
+
r.add_argument('--file', help='evidence file (tail is hashed); default reads stdin')
|
|
88
|
+
r.add_argument('--error', help='record this run as failed with this message')
|
|
89
|
+
r.set_defaults(fn=cmd_record)
|
|
90
|
+
|
|
91
|
+
args = p.parse_args(argv)
|
|
92
|
+
try:
|
|
93
|
+
return args.fn(args)
|
|
94
|
+
except Exception as e: # noqa: BLE001
|
|
95
|
+
print(f'workproof: {args.cmd} failed: {type(e).__name__}: {e}', file=sys.stderr)
|
|
96
|
+
return 2
|
workproof/core.py
ADDED
|
@@ -0,0 +1,303 @@
|
|
|
1
|
+
"""Proof-of-work monitoring for scheduled agents. Standard library only.
|
|
2
|
+
|
|
3
|
+
An agent records evidence of what it produced after every run. A separate
|
|
4
|
+
checker (cron, CI, anything) compares the evidence against a registry and
|
|
5
|
+
reports agents that stopped running, kept running without producing anything
|
|
6
|
+
new, or produce empty output.
|
|
7
|
+
|
|
8
|
+
What this catches: agents that are dead or idle while looking healthy.
|
|
9
|
+
What it does NOT catch: agents that produce changing but wrong output.
|
|
10
|
+
"""
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import functools
|
|
14
|
+
import hashlib
|
|
15
|
+
import json
|
|
16
|
+
import os
|
|
17
|
+
import re
|
|
18
|
+
import sys
|
|
19
|
+
import tempfile
|
|
20
|
+
from dataclasses import asdict, dataclass
|
|
21
|
+
from datetime import datetime, timedelta, timezone
|
|
22
|
+
from pathlib import Path
|
|
23
|
+
|
|
24
|
+
TAIL_BYTES = 65536 # hash only the tail of large evidence files
|
|
25
|
+
EMPTY_FORMS = (b'', b'null', b'[]', b'{}', b'""')
|
|
26
|
+
_SLUG = re.compile(r'[^A-Za-z0-9_.-]+')
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _utcnow() -> datetime:
|
|
30
|
+
return datetime.now(timezone.utc)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
# ── evidence ────────────────────────────────────────────────────────────────
|
|
34
|
+
|
|
35
|
+
def to_evidence(obj) -> bytes:
|
|
36
|
+
"""Canonical bytes for whatever an agent returned, so equal output hashes equal."""
|
|
37
|
+
if obj is None:
|
|
38
|
+
return b''
|
|
39
|
+
if isinstance(obj, bytes):
|
|
40
|
+
return obj
|
|
41
|
+
if isinstance(obj, str):
|
|
42
|
+
return obj.encode('utf-8')
|
|
43
|
+
try:
|
|
44
|
+
return json.dumps(obj, sort_keys=True, default=str, ensure_ascii=False).encode('utf-8')
|
|
45
|
+
except (TypeError, ValueError):
|
|
46
|
+
return repr(obj).encode('utf-8')
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def is_empty(raw: bytes) -> bool:
|
|
50
|
+
return raw.strip() in EMPTY_FORMS
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def read_evidence_file(path) -> bytes:
|
|
54
|
+
"""Tail of a file (large logs only need their newest bytes). b'' if missing."""
|
|
55
|
+
try:
|
|
56
|
+
size = os.path.getsize(path)
|
|
57
|
+
with open(path, 'rb') as f:
|
|
58
|
+
if size > TAIL_BYTES:
|
|
59
|
+
f.seek(size - TAIL_BYTES)
|
|
60
|
+
return f.read()
|
|
61
|
+
except OSError:
|
|
62
|
+
return b''
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def digest(raw: bytes) -> str:
|
|
66
|
+
return hashlib.sha256(raw).hexdigest()[:16]
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
# ── store ───────────────────────────────────────────────────────────────────
|
|
70
|
+
|
|
71
|
+
def _path(store, agent: str) -> Path:
|
|
72
|
+
return Path(store) / (_SLUG.sub('_', agent).strip('_') + '.json')
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def load_record(store, agent: str):
|
|
76
|
+
p = _path(store, agent)
|
|
77
|
+
try:
|
|
78
|
+
rec = json.loads(p.read_text(encoding='utf-8'))
|
|
79
|
+
except FileNotFoundError:
|
|
80
|
+
return None
|
|
81
|
+
if rec.get('agent') != agent:
|
|
82
|
+
raise ValueError(f'store name collision: {p.name} belongs to {rec.get("agent")!r}, not {agent!r}')
|
|
83
|
+
return rec
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _write_atomic(p: Path, data: dict) -> None:
|
|
87
|
+
p.parent.mkdir(parents=True, exist_ok=True)
|
|
88
|
+
fd, tmp = tempfile.mkstemp(dir=str(p.parent), suffix='.tmp')
|
|
89
|
+
try:
|
|
90
|
+
with os.fdopen(fd, 'w', encoding='utf-8') as f:
|
|
91
|
+
json.dump(data, f, indent=2)
|
|
92
|
+
os.replace(tmp, p)
|
|
93
|
+
except BaseException:
|
|
94
|
+
try:
|
|
95
|
+
os.unlink(tmp)
|
|
96
|
+
except OSError:
|
|
97
|
+
pass
|
|
98
|
+
raise
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def record(agent: str, evidence=b'', store='.workproof', *, error=None, now=None) -> dict:
|
|
102
|
+
"""Record one run. Call at the end of every run, success or failure.
|
|
103
|
+
|
|
104
|
+
Evidence counts as new work only if it is non-empty AND differs from the last
|
|
105
|
+
non-empty evidence. A failed run (error set) updates last_run but never counts
|
|
106
|
+
as work, and empty output never counts as work.
|
|
107
|
+
"""
|
|
108
|
+
raw = evidence if isinstance(evidence, bytes) else to_evidence(evidence)
|
|
109
|
+
now = now or _utcnow()
|
|
110
|
+
iso = now.isoformat()
|
|
111
|
+
prev = load_record(store, agent) or {}
|
|
112
|
+
h, empty = digest(raw), is_empty(raw)
|
|
113
|
+
|
|
114
|
+
if error is not None:
|
|
115
|
+
rec = {**prev, 'agent': agent, 'last_run': iso, 'last_error': str(error)[:500],
|
|
116
|
+
'consecutive_errors': prev.get('consecutive_errors', 0) + 1,
|
|
117
|
+
'runs': prev.get('runs', 0) + 1}
|
|
118
|
+
rec.setdefault('hash', h)
|
|
119
|
+
rec.setdefault('last_change', iso)
|
|
120
|
+
rec.setdefault('empty', empty)
|
|
121
|
+
else:
|
|
122
|
+
changed = (not empty) and h != prev.get('hash')
|
|
123
|
+
rec = {'agent': agent, 'hash': h if not empty else prev.get('hash', h), 'empty': empty,
|
|
124
|
+
'last_run': iso, 'runs': prev.get('runs', 0) + 1,
|
|
125
|
+
'last_change': iso if changed or 'last_change' not in prev and not empty
|
|
126
|
+
else prev.get('last_change', iso),
|
|
127
|
+
'last_error': None, 'consecutive_errors': 0}
|
|
128
|
+
_write_atomic(_path(store, agent), rec)
|
|
129
|
+
return rec
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def _safe_record(*args, **kwargs) -> None:
|
|
133
|
+
"""A monitor must never break the thing it monitors: report and carry on.
|
|
134
|
+
The checker will flag the agent as not running if recording keeps failing."""
|
|
135
|
+
try:
|
|
136
|
+
record(*args, **kwargs)
|
|
137
|
+
except Exception as e: # noqa: BLE001
|
|
138
|
+
print(f'workproof: could not record heartbeat: {type(e).__name__}: {e}', file=sys.stderr)
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def proves_work(agent: str, *, store='.workproof', evidence=None, evidence_file=None):
|
|
142
|
+
"""Decorator: record the return value (or evidence(result), or a file's tail)
|
|
143
|
+
after each run. Exceptions are recorded, then re-raised."""
|
|
144
|
+
def deco(fn):
|
|
145
|
+
@functools.wraps(fn)
|
|
146
|
+
def wrapper(*a, **k):
|
|
147
|
+
try:
|
|
148
|
+
result = fn(*a, **k)
|
|
149
|
+
except Exception as e:
|
|
150
|
+
_safe_record(agent, b'', store, error=f'{type(e).__name__}: {e}')
|
|
151
|
+
raise
|
|
152
|
+
if evidence_file is not None:
|
|
153
|
+
raw = read_evidence_file(evidence_file)
|
|
154
|
+
elif evidence is not None:
|
|
155
|
+
raw = to_evidence(evidence(result))
|
|
156
|
+
else:
|
|
157
|
+
raw = to_evidence(result)
|
|
158
|
+
_safe_record(agent, raw, store)
|
|
159
|
+
return result
|
|
160
|
+
return wrapper
|
|
161
|
+
return deco
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
class watch:
|
|
165
|
+
"""Context manager form: `with watch('name') as w: ...; w.evidence = output`."""
|
|
166
|
+
|
|
167
|
+
def __init__(self, agent: str, store='.workproof'):
|
|
168
|
+
self.agent, self.store, self.evidence = agent, store, None
|
|
169
|
+
|
|
170
|
+
def __enter__(self):
|
|
171
|
+
return self
|
|
172
|
+
|
|
173
|
+
def __exit__(self, etype, exc, tb):
|
|
174
|
+
if etype is not None and issubclass(etype, Exception):
|
|
175
|
+
_safe_record(self.agent, b'', self.store, error=f'{etype.__name__}: {exc}')
|
|
176
|
+
else:
|
|
177
|
+
_safe_record(self.agent, to_evidence(self.evidence), self.store)
|
|
178
|
+
return False
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
# ── checking ────────────────────────────────────────────────────────────────
|
|
182
|
+
|
|
183
|
+
def weekday_hours(start: datetime, end: datetime) -> float:
|
|
184
|
+
"""Hours between start and end counting Mon-Fri only (UTC).
|
|
185
|
+
|
|
186
|
+
A weekday-only agent idle over a weekend has missed nothing, but wall-clock age
|
|
187
|
+
says 48h+. Without this, every Monday raised false alarms for healthy agents.
|
|
188
|
+
"""
|
|
189
|
+
if end <= start:
|
|
190
|
+
return 0.0
|
|
191
|
+
total, cur = 0.0, start
|
|
192
|
+
while cur < end:
|
|
193
|
+
midnight = (cur + timedelta(days=1)).replace(hour=0, minute=0, second=0, microsecond=0)
|
|
194
|
+
nxt = min(end, midnight)
|
|
195
|
+
if cur.weekday() < 5:
|
|
196
|
+
total += (nxt - cur).total_seconds() / 3600
|
|
197
|
+
cur = nxt
|
|
198
|
+
return total
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
def _age_hours(since_iso: str, now: datetime, weekdays_only: bool) -> float:
|
|
202
|
+
since = datetime.fromisoformat(since_iso)
|
|
203
|
+
if since.tzinfo is None:
|
|
204
|
+
since = since.replace(tzinfo=timezone.utc)
|
|
205
|
+
if weekdays_only:
|
|
206
|
+
return weekday_hours(since, now)
|
|
207
|
+
return max(0.0, (now - since).total_seconds() / 3600)
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
@dataclass
|
|
211
|
+
class Finding:
|
|
212
|
+
kind: str # never_seen | no_run | no_new_work | empty_output | last_run_failed
|
|
213
|
+
agent: str # | unmonitored | retired_but_scheduled
|
|
214
|
+
detail: str
|
|
215
|
+
|
|
216
|
+
def as_dict(self) -> dict:
|
|
217
|
+
return asdict(self)
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
def load_registry(path) -> dict:
|
|
221
|
+
with open(path, encoding='utf-8') as f:
|
|
222
|
+
reg = json.load(f)
|
|
223
|
+
agents = reg.get('agents')
|
|
224
|
+
if not isinstance(agents, dict) or not agents:
|
|
225
|
+
raise ValueError('registry needs a non-empty "agents" object')
|
|
226
|
+
for name, cfg in agents.items():
|
|
227
|
+
gap = cfg.get('max_gap_hours')
|
|
228
|
+
if not isinstance(gap, (int, float)) or gap <= 0:
|
|
229
|
+
raise ValueError(f'agent {name!r}: max_gap_hours must be a positive number')
|
|
230
|
+
return reg
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
def scheduled_from_crontab(text: str) -> set:
|
|
234
|
+
"""Script names in the ACTIVE (uncommented) lines of a crontab."""
|
|
235
|
+
found = set()
|
|
236
|
+
for line in text.splitlines():
|
|
237
|
+
s = line.strip()
|
|
238
|
+
if not s or s.startswith('#'):
|
|
239
|
+
continue
|
|
240
|
+
found.update(re.findall(r'([A-Za-z0-9_\-]+\.(?:py|sh|js))', s))
|
|
241
|
+
return found
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
def check(registry: dict, store='.workproof', *, scheduled=None, now=None) -> list:
|
|
245
|
+
"""Compare recorded evidence with the registry. Returns a list of Finding."""
|
|
246
|
+
now = now or _utcnow()
|
|
247
|
+
findings = []
|
|
248
|
+
agents = registry['agents']
|
|
249
|
+
silent = registry.get('no_evidence_expected', {})
|
|
250
|
+
retired = registry.get('retired', {})
|
|
251
|
+
|
|
252
|
+
for name, cfg in agents.items():
|
|
253
|
+
wk = bool(cfg.get('weekdays_only'))
|
|
254
|
+
gap = float(cfg['max_gap_hours'])
|
|
255
|
+
run_gap = float(cfg.get('max_run_gap_hours', gap))
|
|
256
|
+
rec = load_record(store, name)
|
|
257
|
+
if rec is None:
|
|
258
|
+
findings.append(Finding('never_seen', name, 'declared in the registry but has never recorded a run'))
|
|
259
|
+
continue
|
|
260
|
+
|
|
261
|
+
run_age = _age_hours(rec['last_run'], now, wk)
|
|
262
|
+
if run_age > run_gap:
|
|
263
|
+
findings.append(Finding('no_run', name, f'has not run in {run_age:.0f}h (limit {run_gap:g}h)'))
|
|
264
|
+
continue
|
|
265
|
+
|
|
266
|
+
if rec.get('consecutive_errors', 0) > 0:
|
|
267
|
+
findings.append(Finding('last_run_failed', name,
|
|
268
|
+
f'{rec["consecutive_errors"]} consecutive failed run(s): {rec.get("last_error")}'))
|
|
269
|
+
|
|
270
|
+
if rec.get('empty'):
|
|
271
|
+
findings.append(Finding('empty_output', name, 'last run produced empty output'))
|
|
272
|
+
continue
|
|
273
|
+
|
|
274
|
+
change_age = _age_hours(rec['last_change'], now, wk)
|
|
275
|
+
if change_age > gap:
|
|
276
|
+
findings.append(Finding('no_new_work', name,
|
|
277
|
+
f'running, but output unchanged for {change_age:.0f}h (limit {gap:g}h)'))
|
|
278
|
+
|
|
279
|
+
if scheduled is not None:
|
|
280
|
+
declared = {cfg.get('script') for cfg in agents.values() if cfg.get('script')}
|
|
281
|
+
declared |= {k.split()[0] for k in silent} | set(retired)
|
|
282
|
+
for script in sorted(set(scheduled) - declared):
|
|
283
|
+
findings.append(Finding('unmonitored', script, 'scheduled but not declared in the registry'))
|
|
284
|
+
for script in sorted(set(retired) & set(scheduled)):
|
|
285
|
+
findings.append(Finding('retired_but_scheduled', script,
|
|
286
|
+
f'marked retired ({retired[script]}) but still scheduled'))
|
|
287
|
+
return findings
|
|
288
|
+
|
|
289
|
+
|
|
290
|
+
def status(registry: dict, store='.workproof', *, now=None) -> list:
|
|
291
|
+
"""One row per declared agent: how long since it last ran and last produced new work."""
|
|
292
|
+
now = now or _utcnow()
|
|
293
|
+
rows = []
|
|
294
|
+
for name, cfg in registry['agents'].items():
|
|
295
|
+
wk = bool(cfg.get('weekdays_only'))
|
|
296
|
+
rec = load_record(store, name)
|
|
297
|
+
row = {'agent': name, 'limit_h': cfg['max_gap_hours'], 'runs': 0, 'last_run_h': None, 'last_change_h': None}
|
|
298
|
+
if rec is not None:
|
|
299
|
+
row.update(runs=rec.get('runs', 0),
|
|
300
|
+
last_run_h=round(_age_hours(rec['last_run'], now, wk), 1),
|
|
301
|
+
last_change_h=round(_age_hours(rec['last_change'], now, wk), 1))
|
|
302
|
+
rows.append(row)
|
|
303
|
+
return rows
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: workproof
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Proof-of-work monitoring for scheduled and AI agents: catches agents that are dead or idle while every health check is green.
|
|
5
|
+
Author: workproof contributors
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/DreamsElectricSheep/workproof
|
|
8
|
+
Project-URL: Issues, https://github.com/DreamsElectricSheep/workproof/issues
|
|
9
|
+
Keywords: agents,monitoring,heartbeat,observability,cron,llm,reliability,dead-man-switch
|
|
10
|
+
Classifier: Development Status :: 3 - Alpha
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: Topic :: System :: Monitoring
|
|
14
|
+
Classifier: Topic :: Software Development :: Quality Assurance
|
|
15
|
+
Requires-Python: >=3.9
|
|
16
|
+
Description-Content-Type: text/markdown
|
|
17
|
+
License-File: LICENSE
|
|
18
|
+
Dynamic: license-file
|
|
19
|
+
|
|
20
|
+
# workproof
|
|
21
|
+
|
|
22
|
+
Proof-of-work monitoring for scheduled and AI agents. Standard library only, Python 3.9+.
|
|
23
|
+
|
|
24
|
+
Normal health checks look at artifacts: the process is up, the file exists, the file is recent. An agent can pass every one of those and still do nothing. It can crash and rewrite an empty file, swallow a fatal error and exit 0, or have a cron entry that was never added. workproof checks that new work was actually produced.
|
|
25
|
+
|
|
26
|
+
## Why this exists
|
|
27
|
+
|
|
28
|
+
It was extracted from a monitor written after an audit of a fleet of scheduled agents found four of them dead for between 5 days and 6 weeks while every existing check reported them healthy:
|
|
29
|
+
|
|
30
|
+
- one had not run for six weeks and was in no monitor at all;
|
|
31
|
+
- one had been dead for nine days, rewriting an empty output file every day;
|
|
32
|
+
- one had its logic removed by an automated code edit, still compiled and imported, and did nothing for three weeks;
|
|
33
|
+
- one hit a fatal error and exited 0, so the restart policy never fired.
|
|
34
|
+
|
|
35
|
+
Each check looked at an artifact. None looked at whether work happened. The longer write-up is in [docs/five-failures.md](docs/five-failures.md).
|
|
36
|
+
|
|
37
|
+
## How it works
|
|
38
|
+
|
|
39
|
+
Two parts, deliberately separate:
|
|
40
|
+
|
|
41
|
+
1. The agent records evidence after every run (decorator, context manager, or a shell command).
|
|
42
|
+
2. A checker you run from cron or CI compares the evidence with a registry and exits non-zero on findings. Because the checker is separate, it also catches an agent that never starts, which a decorator alone cannot see.
|
|
43
|
+
|
|
44
|
+
Evidence is tracked by content hash, not file age. Output identical to the last run does not count as new work, and empty output never does.
|
|
45
|
+
|
|
46
|
+
## Quick start
|
|
47
|
+
|
|
48
|
+
```bash
|
|
49
|
+
pip install workproof
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
Record evidence from Python:
|
|
53
|
+
|
|
54
|
+
```python
|
|
55
|
+
from workproof import proves_work
|
|
56
|
+
|
|
57
|
+
@proves_work("daily-summarizer", store=".workproof")
|
|
58
|
+
def run():
|
|
59
|
+
...
|
|
60
|
+
return summary # hashed; the same output twice is not new work
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
Or as a context manager, or from the shell for agents in any language:
|
|
64
|
+
|
|
65
|
+
```python
|
|
66
|
+
from workproof import watch
|
|
67
|
+
|
|
68
|
+
with watch("nightly-sync") as w:
|
|
69
|
+
rows = sync()
|
|
70
|
+
w.evidence = rows
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
```bash
|
|
74
|
+
./my_agent.sh > out.json && workproof record my-agent --file out.json
|
|
75
|
+
./my_agent.sh || workproof record my-agent --error "exit code $?"
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
Declare what you expect in a registry:
|
|
79
|
+
|
|
80
|
+
```json
|
|
81
|
+
{
|
|
82
|
+
"agents": {
|
|
83
|
+
"daily-summarizer": {"script": "summarize.py", "max_gap_hours": 30, "weekdays_only": true},
|
|
84
|
+
"nightly-sync": {"script": "sync.py", "max_gap_hours": 30}
|
|
85
|
+
},
|
|
86
|
+
"no_evidence_expected": {"cleanup.py": "legitimately produces nothing"},
|
|
87
|
+
"retired": {"old_bot.py": "replaced by daily-summarizer"}
|
|
88
|
+
}
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
Run the checker from cron or CI:
|
|
92
|
+
|
|
93
|
+
```bash
|
|
94
|
+
workproof check --registry agents.json --store .workproof --crontab
|
|
95
|
+
workproof status --registry agents.json --store .workproof
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
Exit code `0` healthy, `1` findings, `2` the checker itself failed. A monitor that crashes quietly reproduces the bug it exists to catch, so a crash is loud and distinct from a finding.
|
|
99
|
+
|
|
100
|
+
## What it reports
|
|
101
|
+
|
|
102
|
+
| Finding | Meaning |
|
|
103
|
+
|---|---|
|
|
104
|
+
| `never_seen` | declared in the registry but has never recorded a run |
|
|
105
|
+
| `no_run` | has not run within its limit (the job is not firing) |
|
|
106
|
+
| `no_new_work` | it runs, but its output has been identical for too long |
|
|
107
|
+
| `empty_output` | the last run produced nothing |
|
|
108
|
+
| `last_run_failed` | consecutive failed runs (a failed run never counts as work) |
|
|
109
|
+
| `unmonitored` | scheduled in cron but not declared in the registry |
|
|
110
|
+
| `retired_but_scheduled` | marked retired but still scheduled |
|
|
111
|
+
|
|
112
|
+
Registry options per agent: `max_gap_hours` (required), `max_run_gap_hours`, `weekdays_only`, `script`. Weekday-only agents are measured in weekday hours, so a quiet weekend does not raise a false alarm every Monday.
|
|
113
|
+
|
|
114
|
+
## What it does not do
|
|
115
|
+
|
|
116
|
+
- It detects dead or idle agents, not wrong ones. An agent emitting changing garbage passes.
|
|
117
|
+
- Output that is legitimately constant will be flagged. Declare it under `no_evidence_expected` or give it a longer limit.
|
|
118
|
+
- Weekdays are evaluated in UTC.
|
|
119
|
+
- One writer per agent name is assumed (no file locking).
|
|
120
|
+
- There is no dashboard and no built-in alerting. Wire the exit code or `--json` output into whatever you already use.
|
|
121
|
+
|
|
122
|
+
## Is a hosted version useful?
|
|
123
|
+
|
|
124
|
+
I am deciding whether to build a hosted version with alerting, a dashboard and multi-agent history. If your team would use it, or if something here does not fit how you run agents, please [open an issue](https://github.com/DreamsElectricSheep/workproof/issues) and describe your setup. That is the most useful thing you can send.
|
|
125
|
+
|
|
126
|
+
## Tests
|
|
127
|
+
|
|
128
|
+
```bash
|
|
129
|
+
python -m unittest discover -s tests -v
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
The suite reproduces the four incidents above.
|
|
133
|
+
|
|
134
|
+
## License
|
|
135
|
+
|
|
136
|
+
MIT
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
workproof/__init__.py,sha256=AD78-5xh5U8LV6n3AWHwQuLcp8lbEKJfAIcrFOcGHqQ,481
|
|
2
|
+
workproof/__main__.py,sha256=E6Gls0DNz8GQK2K-kOUIx8cYhgANW_CH54VKrfCfs14,52
|
|
3
|
+
workproof/cli.py,sha256=szBj3HUEJCGE9ZZNDs5tpsXS4WJmAwH0NnbyZ6f4RLU,3657
|
|
4
|
+
workproof/core.py,sha256=RnbLe5GWT763oWhqpILAeKxluX4MsUS_dI3ybGAzw30,11641
|
|
5
|
+
workproof-0.1.0.dist-info/licenses/LICENSE,sha256=0YQWvfEY2Qk0FmnhtLQ8xKN8lGpbnQ2r6aPyX5uRBFA,1079
|
|
6
|
+
workproof-0.1.0.dist-info/METADATA,sha256=qca_zjdjLN5IXuN0xf9JWMD0TvOLSBAhv4W2SQ_KUKE,5518
|
|
7
|
+
workproof-0.1.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
|
|
8
|
+
workproof-0.1.0.dist-info/entry_points.txt,sha256=H6fKs5KER5T9OF-yu6-ABwOCxRAzib1rqo-5Bolnkyk,49
|
|
9
|
+
workproof-0.1.0.dist-info/top_level.txt,sha256=UfDE_y59EN2XBZ_mb-wpxqWdTDPs9oiGq-_rGJeEPqE,10
|
|
10
|
+
workproof-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 workproof contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
workproof
|