ecadlib 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ecadlib/__init__.py +3 -0
- ecadlib/__main__.py +5 -0
- ecadlib/a365_sync.py +637 -0
- ecadlib/agent.py +1407 -0
- ecadlib/agent_queue.py +780 -0
- ecadlib/agent_watch.py +732 -0
- ecadlib/altium365.py +385 -0
- ecadlib/altium_dblib_sync.py +530 -0
- ecadlib/altium_export.py +209 -0
- ecadlib/altium_forms.py +768 -0
- ecadlib/altium_lib.py +1089 -0
- ecadlib/altium_menu.py +1751 -0
- ecadlib/altium_pkg.py +396 -0
- ecadlib/altium_project.py +1095 -0
- ecadlib/app.py +1458 -0
- ecadlib/approvals.py +395 -0
- ecadlib/assets/demo_bom.csv +8 -0
- ecadlib/assets/kicad_action_plugin.py +607 -0
- ecadlib/assets/kicad_reread.py +60 -0
- ecadlib/assets/pcblib_template.json.gz +0 -0
- ecadlib/bom_inputs.py +339 -0
- ecadlib/bom_triage.py +519 -0
- ecadlib/cad_model.py +528 -0
- ecadlib/cfb.py +171 -0
- ecadlib/cli.py +185 -0
- ecadlib/clihelp.py +149 -0
- ecadlib/cliutil.py +119 -0
- ecadlib/completion.py +173 -0
- ecadlib/costs.py +276 -0
- ecadlib/crosscheck.py +412 -0
- ecadlib/datasheet.py +2168 -0
- ecadlib/dedupe.py +572 -0
- ecadlib/detect.py +170 -0
- ecadlib/doctor.py +450 -0
- ecadlib/ds_route.py +541 -0
- ecadlib/eagle_export.py +1409 -0
- ecadlib/easy_setup.py +338 -0
- ecadlib/easyeda_convert.py +386 -0
- ecadlib/easyeda_pro_export.py +972 -0
- ecadlib/eval/__init__.py +0 -0
- ecadlib/eval/baselines/kicad-v1.json +404 -0
- ecadlib/eval/build_set.py +299 -0
- ecadlib/eval/cli.py +119 -0
- ecadlib/eval/gate.py +358 -0
- ecadlib/eval/golden/demo.txt +379 -0
- ecadlib/eval/golden/specs.txt +3120 -0
- ecadlib/eval/golden/testpack.txt +463 -0
- ecadlib/eval/golden.py +446 -0
- ecadlib/eval/replay/kicad-v1.json.gz +0 -0
- ecadlib/eval/runner.py +456 -0
- ecadlib/eval/score.py +179 -0
- ecadlib/eval/sets/kicad-v1.json +630 -0
- ecadlib/firstrun.py +400 -0
- ecadlib/footprint_adv.py +1006 -0
- ecadlib/footprint_bench.py +218 -0
- ecadlib/footprint_check.py +562 -0
- ecadlib/footprint_gen.py +582 -0
- ecadlib/footprint_rules.py +1092 -0
- ecadlib/forge.py +267 -0
- ecadlib/fp_share.py +95 -0
- ecadlib/fp_tidy.py +389 -0
- ecadlib/fsutil.py +62 -0
- ecadlib/house_style.py +1051 -0
- ecadlib/httpcache.py +124 -0
- ecadlib/hub.py +1490 -0
- ecadlib/hub_copy.py +161 -0
- ecadlib/hub_tools.py +79 -0
- ecadlib/kicad_dblib.py +257 -0
- ecadlib/kicad_export.py +837 -0
- ecadlib/kicad_import.py +913 -0
- ecadlib/kicad_index.py +160 -0
- ecadlib/kicad_menu.py +193 -0
- ecadlib/kicad_pcm.py +475 -0
- ecadlib/launcher.py +156 -0
- ecadlib/librarian.py +963 -0
- ecadlib/lifecycle.py +691 -0
- ecadlib/llm/__init__.py +276 -0
- ecadlib/llm/cli.py +70 -0
- ecadlib/llm/openai_compat.py +214 -0
- ecadlib/llm/profiles.py +245 -0
- ecadlib/llm/rails.py +570 -0
- ecadlib/llm/run.py +126 -0
- ecadlib/llm/sampling.py +49 -0
- ecadlib/mcp_server.py +723 -0
- ecadlib/messages.py +419 -0
- ecadlib/migrate.py +407 -0
- ecadlib/notify.py +329 -0
- ecadlib/package_schema.json +1238 -0
- ecadlib/package_specs/CP_EIA-3216-18_Kemet-A.json +27 -0
- ecadlib/package_specs/C_0402_1005Metric.json +22 -0
- ecadlib/package_specs/C_0805_2012Metric.json +22 -0
- ecadlib/package_specs/DFN-8-1EP_3x3mm_P0.5mm_EP1.66x2.38mm.json +40 -0
- ecadlib/package_specs/D_SMA.json +27 -0
- ecadlib/package_specs/D_SMB.json +27 -0
- ecadlib/package_specs/D_SOD-123.json +43 -0
- ecadlib/package_specs/D_SOD-323.json +43 -0
- ecadlib/package_specs/LED_0603_1608Metric.json +23 -0
- ecadlib/package_specs/LED_0805_2012Metric.json +23 -0
- ecadlib/package_specs/LQFP-48_7x7mm_P0.5mm.json +31 -0
- ecadlib/package_specs/L_0805_2012Metric.json +22 -0
- ecadlib/package_specs/MSOP-8_3x3mm_P0.65mm.json +31 -0
- ecadlib/package_specs/QFN-16-1EP_3x3mm_P0.5mm_EP1.75x1.75mm.json +39 -0
- ecadlib/package_specs/QFN-32-1EP_5x5mm_P0.5mm_EP3.45x3.45mm.json +37 -0
- ecadlib/package_specs/R_0402_1005Metric.json +22 -0
- ecadlib/package_specs/R_0603_1608Metric.json +22 -0
- ecadlib/package_specs/R_0805_2012Metric.json +22 -0
- ecadlib/package_specs/R_1206_3216Metric.json +22 -0
- ecadlib/package_specs/SOIC-16_3.9x9.9mm_P1.27mm.json +32 -0
- ecadlib/package_specs/SOIC-8_3.9x4.9mm_P1.27mm.json +32 -0
- ecadlib/package_specs/SOT-223-3_TabPin2.json +56 -0
- ecadlib/package_specs/SOT-23-5.json +57 -0
- ecadlib/package_specs/SOT-23-6.json +62 -0
- ecadlib/package_specs/SOT-23.json +48 -0
- ecadlib/package_specs/TSSOP-16_4.4x5mm_P0.65mm.json +31 -0
- ecadlib/package_specs/VQFN-16-1EP_3x3mm_P0.5mm_EP1.68x1.68mm.json +39 -0
- ecadlib/package_specs/advanced/BGA-100_11.0x11.0mm_Layout10x10_P1.0mm_Ball0.5mm_Pad0.4mm_NSMD.json +26 -0
- ecadlib/package_specs/advanced/BGA-121_9.0x9.0mm_Layout11x11_P0.8mm_Ball0.4mm_Pad0.35mm_NSMD.json +26 -0
- ecadlib/package_specs/advanced/BGA-144_7.0x7.0mm_Layout13x13_P0.5mm_Ball0.3mm_Pad0.25mm_NSMD.json +29 -0
- ecadlib/package_specs/advanced/BGA-169_11.0x11.0mm_Layout13x13_P0.8mm_Ball0.5mm_Pad0.4mm_NSMD.json +23 -0
- ecadlib/package_specs/advanced/BGA-256_11.0x11.0mm_Layout20x20_P0.5mm_Ball0.3mm_Pad0.25mm_NSMD.json +26 -0
- ecadlib/package_specs/advanced/BGA-256_17.0x17.0mm_Layout16x16_P1.0mm_Ball0.5mm_Pad0.4mm_NSMD.json +26 -0
- ecadlib/package_specs/advanced/BGA-324_15.0x15.0mm_Layout18x18_P0.8mm_Ball0.5mm_Pad0.4mm_NSMD.json +26 -0
- ecadlib/package_specs/advanced/BGA-36_3.396x3.466mm_Layout6x6_P0.4mm_Ball0.25mm_Pad0.2mm_NSMD.json +24 -0
- ecadlib/package_specs/advanced/BGA-672_27.0x27.0mm_Layout26x26_P1.0mm_Ball0.6mm_Pad0.5mm_NSMD.json +32 -0
- ecadlib/package_specs/advanced/BGA-68_5.0x5.0mm_Layout9x9_P0.5mm_Ball0.3mm_Pad0.25mm_NSMD.json +30 -0
- ecadlib/package_specs/advanced/BGA-90_8.0x13.0mm_Layout2x3x15_P0.8mm.json +26 -0
- ecadlib/package_specs/advanced/DFN-8-1EP_3x3mm_P0.5mm_EP1.65x2.38mm_ThermalVias.json +60 -0
- ecadlib/package_specs/advanced/Example_Custom_LPad_Slot.json +24 -0
- ecadlib/package_specs/advanced/JST_GH_SM04B-GHS-TB_1x04-1MP_P1.25mm_Horizontal.json +49 -0
- ecadlib/package_specs/advanced/JST_PH_B4B-PH-K_1x04_P2.00mm_Vertical.json +33 -0
- ecadlib/package_specs/advanced/JST_PH_S4B-PH-SM4-TB_1x04-1MP_P2.00mm_Horizontal.json +55 -0
- ecadlib/package_specs/advanced/JST_SH_SM04B-SRSS-TB_1x04-1MP_P1.00mm_Horizontal.json +49 -0
- ecadlib/package_specs/advanced/JST_XH_B2B-XH-A_1x02_P2.50mm_Vertical.json +34 -0
- ecadlib/package_specs/advanced/JST_XH_B4B-XH-A_1x04_P2.50mm_Vertical.json +34 -0
- ecadlib/package_specs/advanced/JST_XH_B6B-XH-A_1x06_P2.50mm_Vertical.json +33 -0
- ecadlib/package_specs/advanced/LFBGA-144_10x10mm_Layout12x12_P0.8mm.json +23 -0
- ecadlib/package_specs/advanced/Molex_KK-254_AE-6410-04A_1x04_P2.54mm_Vertical.json +34 -0
- ecadlib/package_specs/advanced/Molex_PicoBlade_53398-0471_1x04-1MP_P1.25mm_Vertical.json +54 -0
- ecadlib/package_specs/advanced/PinHeader_1x05_P1.00mm_Vertical_SMD_Pin1Left.json +14 -0
- ecadlib/package_specs/advanced/PinHeader_1x05_P1.27mm_Vertical_SMD_Pin1Left.json +14 -0
- ecadlib/package_specs/advanced/PinHeader_1x05_P2.00mm_Vertical.json +12 -0
- ecadlib/package_specs/advanced/PinHeader_1x05_P2.00mm_Vertical_SMD_Pin1Left.json +14 -0
- ecadlib/package_specs/advanced/PinHeader_1x05_P2.54mm_Vertical.json +12 -0
- ecadlib/package_specs/advanced/PinHeader_1x05_P2.54mm_Vertical_SMD_Pin1Left.json +25 -0
- ecadlib/package_specs/advanced/PinHeader_1x05_P2.54mm_Vertical_SMD_Pin1Right.json +14 -0
- ecadlib/package_specs/advanced/PinHeader_1x10_P1.00mm_Vertical.json +15 -0
- ecadlib/package_specs/advanced/PinHeader_2x05_P1.00mm_Vertical_SMD.json +13 -0
- ecadlib/package_specs/advanced/PinHeader_2x05_P1.27mm_Vertical.json +18 -0
- ecadlib/package_specs/advanced/PinHeader_2x05_P1.27mm_Vertical_SMD.json +24 -0
- ecadlib/package_specs/advanced/PinHeader_2x05_P2.00mm_Vertical_SMD.json +13 -0
- ecadlib/package_specs/advanced/PinHeader_2x05_P2.54mm_Vertical.json +12 -0
- ecadlib/package_specs/advanced/PinHeader_2x05_P2.54mm_Vertical_SMD.json +24 -0
- ecadlib/package_specs/advanced/PinHeader_2x20_P2.54mm_Vertical.json +12 -0
- ecadlib/package_specs/advanced/QFN-16-1EP_3x3mm_P0.5mm_EP1.75x1.75mm_ThermalVias.json +87 -0
- ecadlib/package_specs/advanced/UFBGA-132_7x7mm_Layout12x12_P0.5mm.json +43 -0
- ecadlib/package_specs/advanced/USB_C_Receptacle_HRO_TYPE-C-31-M-12.json +218 -0
- ecadlib/package_specs/standard.json.gz +0 -0
- ecadlib/parity.py +1204 -0
- ecadlib/parts_export.py +154 -0
- ecadlib/partview.py +215 -0
- ecadlib/plugins.py +231 -0
- ecadlib/providers.py +226 -0
- ecadlib/rebuild.py +471 -0
- ecadlib/release_check.py +437 -0
- ecadlib/reports.py +274 -0
- ecadlib/resolve.py +556 -0
- ecadlib/review_cli.py +292 -0
- ecadlib/rules.py +979 -0
- ecadlib/rules_infer.py +1205 -0
- ecadlib/sexpr.py +149 -0
- ecadlib/sheetsafe.py +65 -0
- ecadlib/sim_models.py +1087 -0
- ecadlib/sourcing.py +1161 -0
- ecadlib/stdparts.py +755 -0
- ecadlib/step_gen.py +929 -0
- ecadlib/symbol_layout.py +798 -0
- ecadlib/team.py +558 -0
- ecadlib/team_setup.py +288 -0
- ecadlib/term.py +420 -0
- ecadlib/term_views.py +64 -0
- ecadlib/testpack.py +334 -0
- ecadlib/ui/__init__.py +1 -0
- ecadlib/ui/__main__.py +3 -0
- ecadlib/ui/server.py +1029 -0
- ecadlib/ui/static/hairline.js +2 -0
- ecadlib/ui/static/index.html +2098 -0
- ecadlib/ui/static/setup.html +294 -0
- ecadlib/upkeep.py +717 -0
- ecadlib/upstream.py +573 -0
- ecadlib/userconfig.py +257 -0
- ecadlib/visual_check.py +1603 -0
- ecadlib/ws_status.py +203 -0
- ecadlib-0.3.0.dist-info/METADATA +2245 -0
- ecadlib-0.3.0.dist-info/RECORD +199 -0
- ecadlib-0.3.0.dist-info/WHEEL +5 -0
- ecadlib-0.3.0.dist-info/entry_points.txt +2 -0
- ecadlib-0.3.0.dist-info/licenses/LICENSE +21 -0
- ecadlib-0.3.0.dist-info/licenses/NOTICE +53 -0
- ecadlib-0.3.0.dist-info/top_level.txt +1 -0
ecadlib/agent.py
ADDED
|
@@ -0,0 +1,1407 @@
|
|
|
1
|
+
"""Library agent: the orchestrator that keeps the library right on its own.
|
|
2
|
+
|
|
3
|
+
ecadlib agent run [--workspace W] [--dry-run] [--upkeep] [--lib DIR] one pass of the loop below
|
|
4
|
+
ecadlib agent serve [--workspace W] [--interval S] run it every [agent] interval_minutes
|
|
5
|
+
ecadlib agent status [--workspace W] [--healthy SECONDS] what the agent did last, queue counts
|
|
6
|
+
ecadlib agent approve ID [--user U] [-m NOTE] | reject ID -m NOTE sign off a part in the review queue
|
|
7
|
+
ecadlib agent push [--dry-run] push approved parts to Altium 365 now
|
|
8
|
+
|
|
9
|
+
One pass (run_once):
|
|
10
|
+
WATCH agent_watch.scan() finds new / changed / removed parts in the watched projects and BOMs
|
|
11
|
+
QUEUE each change is upserted into agent_queue (one open item per part key, never duplicated)
|
|
12
|
+
BUILD every detected item goes through the librarian (own libraries, KiCad, EasyEDA, then the datasheet reader;
|
|
13
|
+
as one Message Batch when [agent] datasheet_batch and the rules' sources allow it). The agent keeps one
|
|
14
|
+
cumulative BOM, <ws>/agent/library_bom.csv, and its librarian plan is the agent's library. Before and after
|
|
15
|
+
pictures (visual_check renderers): before = the part's current version in that library (or in the library
|
|
16
|
+
repo's git history), after = the new build.
|
|
17
|
+
PROVE visual_check.prove() compares our footprint/symbol with the datasheet drawing; "fail" -> failed with the
|
|
18
|
+
reason, anything else -> review (doubts listed). A part the librarian could not build -> failed with why.
|
|
19
|
+
REVIEW review items are grouped into batches of [agent] batch_size; each batch is ONE pull request: a branch
|
|
20
|
+
ecadlib/agent-<batch> in the library git repo with the changed library files and the pictures (under
|
|
21
|
+
[agent] pictures_path), body with before/after pictures, proof findings and doubts. The commit is made with
|
|
22
|
+
git plumbing (a temporary index), so the working tree and HEAD of the repo are never touched; pushes are
|
|
23
|
+
never forced, and an open batch PR is updated, never duplicated. On GitLab (origin on a gitlab host, or
|
|
24
|
+
[agent] git_host = "gitlab") the same happens with merge requests (ecadlib/forge.py). No git repo, no remote or no token: the PR
|
|
25
|
+
body and pictures are written to <ws>/agent/batches/<id>/ only, and the report says why.
|
|
26
|
+
PUSH approved items go to Altium 365 through altium365.push (release check and approvals still gate it), when
|
|
27
|
+
[agent] auto_push is true (before BUILD, in the same pass) or with `ecadlib agent push`. Only the build a
|
|
28
|
+
person approved is sent: an item whose library files changed since, or that has nothing built (a removal,
|
|
29
|
+
an upkeep finding), is not. A change seen after the approval becomes a new item once it is pushed.
|
|
30
|
+
--dry-run sends nothing.
|
|
31
|
+
MAINTAIN on the [agent] upkeep schedule, upkeep.run() over every librarian plan in the workspace; new findings
|
|
32
|
+
(obsolete / NRND, stock-out, price jump, KiCad upstream change, Altium 365 drift) become queue items with
|
|
33
|
+
origin "upkeep". A KiCad source that changed is rebuilt; the others are findings for a person.
|
|
34
|
+
|
|
35
|
+
State lives in <ws>/agent/: status.json (for the web UI), agent_state.json (last upkeep), run.lock (one run at a time),
|
|
36
|
+
plan.lock (held while the librarian rewrites the plan: approve/reject wait for it), library_bom.json/.csv,
|
|
37
|
+
pictures/<item>/, batches/<id>/, and watch_pending.json / upkeep_pending.json while what a scan saw is not queued yet.
|
|
38
|
+
A run that dies anywhere loses nothing: the next one rescans from before it, re-proves items left in `proving` and
|
|
39
|
+
rebuilds items left in `building` (an item that stopped the agent 3 times is failed instead). A dry run writes none of
|
|
40
|
+
it and restores the watch state."""
|
|
41
|
+
import argparse, contextlib, copy, csv, datetime as _dt, hashlib, json, os, re, shutil, signal, subprocess, sys, tempfile, threading, time
|
|
42
|
+
from pathlib import Path
|
|
43
|
+
|
|
44
|
+
from . import rules as RU, sheetsafe as SS
|
|
45
|
+
|
|
46
|
+
AGENT = "ecadlib-agent"
|
|
47
|
+
BOM_NAME = "library_bom.csv"
|
|
48
|
+
BOM_COLS = (("designator", "Designator"), ("qty", "Qty"), ("value", "Value"), ("footprint", "Footprint"),
|
|
49
|
+
("manufacturer", "Manufacturer"), ("mpn", "Mfr. Part #"), ("lcsc", "LCSC Part Number"),
|
|
50
|
+
("description", "Description"), ("datasheet", "Datasheet"))
|
|
51
|
+
SCHEDULES = {"off": None, "daily": 86400, "weekly": 7 * 86400, "monthly": 30 * 86400}
|
|
52
|
+
SKIP_UPKEEP = ("recovered", "back in stock", "resolved", "reconverted") # good news is not work
|
|
53
|
+
BODY_MAX = 60000
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
# ------------------------------------------------------------------ small helpers
|
|
57
|
+
def stamp(now=None):
|
|
58
|
+
"""UTC ISO-8601 with Z. now: None, a datetime, an epoch number or an ISO string."""
|
|
59
|
+
if isinstance(now, str):
|
|
60
|
+
return now
|
|
61
|
+
if isinstance(now, (int, float)):
|
|
62
|
+
now = _dt.datetime.fromtimestamp(now, _dt.timezone.utc)
|
|
63
|
+
now = now or _dt.datetime.now(_dt.timezone.utc)
|
|
64
|
+
if now.tzinfo is None:
|
|
65
|
+
now = now.replace(tzinfo=_dt.timezone.utc)
|
|
66
|
+
return now.astimezone(_dt.timezone.utc).replace(microsecond=0).strftime("%Y-%m-%dT%H:%M:%SZ")
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _epoch(s):
|
|
70
|
+
try:
|
|
71
|
+
return _dt.datetime.strptime(s, "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=_dt.timezone.utc).timestamp()
|
|
72
|
+
except (TypeError, ValueError):
|
|
73
|
+
return None
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def part_key(manufacturer, mpn):
|
|
77
|
+
from .agent_queue import norm_manufacturer # "ST" and "STMicroelectronics" are one maker
|
|
78
|
+
return f"{norm_manufacturer(manufacturer)}|{(mpn or '').strip().upper()}"
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def write_json(path, data):
|
|
82
|
+
path = Path(path)
|
|
83
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
84
|
+
tmp = path.with_name(path.name + f".{os.getpid()}.{threading.get_ident()}.tmp")
|
|
85
|
+
with open(tmp, "w") as f:
|
|
86
|
+
f.write(json.dumps(data, indent=2, default=str))
|
|
87
|
+
f.flush()
|
|
88
|
+
os.fsync(f.fileno()) # a power cut must not leave an empty file behind the rename
|
|
89
|
+
os.replace(tmp, path)
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def write_json_text(path, text):
|
|
93
|
+
"""write_json for text that already is JSON (a saved copy of a state file)."""
|
|
94
|
+
path = Path(path)
|
|
95
|
+
tmp = path.with_name(path.name + f".{os.getpid()}.{threading.get_ident()}.tmp")
|
|
96
|
+
with open(tmp, "w") as f:
|
|
97
|
+
f.write(text)
|
|
98
|
+
f.flush()
|
|
99
|
+
os.fsync(f.fileno())
|
|
100
|
+
os.replace(tmp, path)
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def read_json(path, default):
|
|
104
|
+
try:
|
|
105
|
+
return json.loads(Path(path).read_text())
|
|
106
|
+
except (FileNotFoundError, ValueError):
|
|
107
|
+
return copy.deepcopy(default)
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def settings(rules):
|
|
111
|
+
a = dict(RU.DEFAULTS["agent"])
|
|
112
|
+
a.update((rules or {}).get("agent") or {})
|
|
113
|
+
return a
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def _safe(s):
|
|
117
|
+
return re.sub(r"[^A-Za-z0-9._-]+", "_", str(s)).strip("._")[:80] or "item"
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def _par(item):
|
|
121
|
+
return {p["name"]: p["value"] for p in item.get("parameters", [])}
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
class _NoQueue:
|
|
125
|
+
"""A dry run on a workspace without a queue yet: an empty queue that makes no folders."""
|
|
126
|
+
def list(self, status=None, batch=None):
|
|
127
|
+
return []
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def _modules(watcher=None, queue=None, prover=None, workspace=None, dry_run=False):
|
|
131
|
+
"""The three sibling modules, imported only when the caller did not pass its own (tests pass fakes)."""
|
|
132
|
+
if watcher is None:
|
|
133
|
+
from . import agent_watch as watcher
|
|
134
|
+
if queue is None:
|
|
135
|
+
from . import agent_queue as AQ
|
|
136
|
+
queue = _NoQueue() if dry_run and not (Path(workspace) / "agent" / "queue" / "items").is_dir() else AQ.Queue(workspace)
|
|
137
|
+
if prover is None:
|
|
138
|
+
from . import visual_check as prover
|
|
139
|
+
return watcher, queue, prover
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def _queue(ws):
|
|
143
|
+
from . import agent_queue as AQ
|
|
144
|
+
return AQ.Queue(ws)
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
@contextlib.contextmanager
|
|
148
|
+
def run_lock(ws, create=True):
|
|
149
|
+
"""One run at a time per workspace (serve, a CLI run and the web UI share it). Yields False when held.
|
|
150
|
+
create=False (a dry run): no folder or file is made; without a lock file no run ever happened, so none runs."""
|
|
151
|
+
d = Path(ws) / "agent"
|
|
152
|
+
if not create and not (d / "run.lock").exists():
|
|
153
|
+
yield True
|
|
154
|
+
return
|
|
155
|
+
if create:
|
|
156
|
+
d.mkdir(parents=True, exist_ok=True)
|
|
157
|
+
f = open(d / "run.lock", "a+" if create else "r")
|
|
158
|
+
got = False
|
|
159
|
+
try:
|
|
160
|
+
try:
|
|
161
|
+
try:
|
|
162
|
+
import fcntl
|
|
163
|
+
fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
164
|
+
except ImportError: # Windows: locks a byte range from the position, so byte 0 for all
|
|
165
|
+
import msvcrt
|
|
166
|
+
f.seek(0) # msvcrt locks bytes from the file position: "a+" starts at the end,
|
|
167
|
+
msvcrt.locking(f.fileno(), msvcrt.LK_NBLCK, 1) # which moves once the pid is written: always byte 0
|
|
168
|
+
except OSError:
|
|
169
|
+
got = False
|
|
170
|
+
else:
|
|
171
|
+
got = True
|
|
172
|
+
if create:
|
|
173
|
+
f.seek(0); f.truncate(); f.write(f"{os.getpid()} {stamp()}\n"); f.flush()
|
|
174
|
+
yield got
|
|
175
|
+
finally:
|
|
176
|
+
if got and os.name == "nt":
|
|
177
|
+
import msvcrt
|
|
178
|
+
try:
|
|
179
|
+
f.seek(0)
|
|
180
|
+
msvcrt.locking(f.fileno(), msvcrt.LK_UNLCK, 1)
|
|
181
|
+
except OSError:
|
|
182
|
+
pass
|
|
183
|
+
f.close()
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def _librarian_out(ws, bom):
|
|
187
|
+
"""The same folder Workspace uses for this BOM, so the review queue's sign-off finds the agent's plan."""
|
|
188
|
+
from .app import Workspace
|
|
189
|
+
w = Workspace.__new__(Workspace) # no __init__: it makes the workspace folders (a dry run must not)
|
|
190
|
+
w.root = Path(ws).resolve()
|
|
191
|
+
return w._librarian_dir(Path(bom))
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
# ------------------------------------------------------------------ status file
|
|
195
|
+
def status_path(ws):
|
|
196
|
+
return Path(ws) / "agent" / "status.json"
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def write_status(ws, **fields):
|
|
200
|
+
st = read_json(status_path(ws), {})
|
|
201
|
+
st.update(fields, heartbeat=stamp(), pid=os.getpid())
|
|
202
|
+
write_json(status_path(ws), st)
|
|
203
|
+
return st
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
def status(workspace, queue=None):
|
|
207
|
+
"""status.json plus the queue's counts per state (what `ecadlib agent status` and the web UI show)."""
|
|
208
|
+
ws = Path(workspace).resolve()
|
|
209
|
+
st = read_json(status_path(ws), {"state": "never run"})
|
|
210
|
+
try:
|
|
211
|
+
q = queue if queue is not None else _queue(ws)
|
|
212
|
+
counts = {}
|
|
213
|
+
for it in q.list():
|
|
214
|
+
counts[it["status"]] = counts.get(it["status"], 0) + 1
|
|
215
|
+
st["queue"] = counts
|
|
216
|
+
except Exception as e: # queue module not installed / unreadable: status still answers
|
|
217
|
+
st["queue_error"] = f"{type(e).__name__}: {e}"
|
|
218
|
+
st["batches"] = [read_json(b, {}) for b in sorted((ws / "agent" / "batches").glob("*/batch.json"))[-20:]]
|
|
219
|
+
return st
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
# ------------------------------------------------------------------ git
|
|
223
|
+
class GitError(Exception):
|
|
224
|
+
pass
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def git(repo, *args, env=None, input=None, ok=(0,)):
|
|
228
|
+
e = dict(os.environ, **(env or {}))
|
|
229
|
+
e.setdefault("GIT_TERMINAL_PROMPT", "0")
|
|
230
|
+
try:
|
|
231
|
+
p = subprocess.run(["git", *args], cwd=repo, capture_output=True, text=True, env=e, input=input)
|
|
232
|
+
except FileNotFoundError:
|
|
233
|
+
raise GitError("git is not installed") from None
|
|
234
|
+
if p.returncode not in ok:
|
|
235
|
+
raise GitError(f"git {' '.join(args[:3])}: {(p.stderr or p.stdout).strip()[:300]}")
|
|
236
|
+
return p.stdout.strip()
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
def repo_root(path):
|
|
240
|
+
try:
|
|
241
|
+
return Path(git(path, "rev-parse", "--show-toplevel")).resolve()
|
|
242
|
+
except (GitError, OSError):
|
|
243
|
+
return None
|
|
244
|
+
|
|
245
|
+
|
|
246
|
+
def github_slug(url):
|
|
247
|
+
m = re.search(r"github[^/:]*[/:]([^/]+)/([^/]+?)(?:\.git)?/?$", url or "")
|
|
248
|
+
return f"{m.group(1)}/{m.group(2)}" if m else None
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
def commit_files(repo, branch, base, files, message, remove=()):
|
|
252
|
+
"""Commit {repo path: local file} on top of `branch` (or `base` when the branch does not exist) WITHOUT touching
|
|
253
|
+
the working tree, the index or HEAD: a temporary index, write-tree, commit-tree, update-ref. `remove`: repo paths
|
|
254
|
+
taken out of the tree. -> (sha, changed)."""
|
|
255
|
+
parent = git(repo, "rev-parse", "--verify", "-q", f"refs/heads/{branch}", ok=(0, 1)) or git(repo, "rev-parse", base)
|
|
256
|
+
with tempfile.TemporaryDirectory() as td:
|
|
257
|
+
env = {"GIT_INDEX_FILE": str(Path(td) / "index")}
|
|
258
|
+
git(repo, "read-tree", parent, env=env)
|
|
259
|
+
for path, src in sorted(files.items()):
|
|
260
|
+
blob = git(repo, "hash-object", "-w", "--", str(src))
|
|
261
|
+
git(repo, "update-index", "--add", "--cacheinfo", f"100644,{blob},{path}", env=env)
|
|
262
|
+
gone = sorted(set(remove) - set(files))
|
|
263
|
+
if gone:
|
|
264
|
+
git(repo, "update-index", "--force-remove", "--", *gone, env=env)
|
|
265
|
+
tree = git(repo, "write-tree", env=env)
|
|
266
|
+
if tree == git(repo, "rev-parse", f"{parent}^{{tree}}"):
|
|
267
|
+
if not git(repo, "rev-parse", "--verify", "-q", f"refs/heads/{branch}", ok=(0, 1)):
|
|
268
|
+
git(repo, "update-ref", f"refs/heads/{branch}", parent, "")
|
|
269
|
+
return parent, False
|
|
270
|
+
ident = {}
|
|
271
|
+
if not git(repo, "config", "user.email", ok=(0, 1)):
|
|
272
|
+
ident = {"GIT_AUTHOR_NAME": AGENT, "GIT_AUTHOR_EMAIL": f"{AGENT}@users.noreply.github.com",
|
|
273
|
+
"GIT_COMMITTER_NAME": AGENT, "GIT_COMMITTER_EMAIL": f"{AGENT}@users.noreply.github.com"}
|
|
274
|
+
sha = git(repo, "commit-tree", tree, "-p", parent, "-F", "-", env=ident, input=message)
|
|
275
|
+
old = git(repo, "rev-parse", "--verify", "-q", f"refs/heads/{branch}", ok=(0, 1))
|
|
276
|
+
git(repo, "update-ref", f"refs/heads/{branch}", sha, old or "") # refuses if someone moved it meanwhile
|
|
277
|
+
return sha, True
|
|
278
|
+
|
|
279
|
+
|
|
280
|
+
# ------------------------------------------------------------------ the agent's BOM and builder
|
|
281
|
+
def _line_of(item, plan_items=None):
|
|
282
|
+
"""BOM line for a queue item: the watched line, else (upkeep) built from the library plan entry."""
|
|
283
|
+
ch = item.get("change") or {}
|
|
284
|
+
ln = dict(ch.get("line") or {})
|
|
285
|
+
if not ln and plan_items:
|
|
286
|
+
pi = _find_plan_item(plan_items, item)
|
|
287
|
+
if pi:
|
|
288
|
+
p = _par(pi)
|
|
289
|
+
ln = {"mpn": p.get("Manufacturer Part Number", ""), "manufacturer": p.get("Manufacturer", ""),
|
|
290
|
+
"value": p.get("Value", ""), "footprint": (pi.get("footprint") or {}).get("bom_ref") or (pi.get("footprint") or {}).get("ref", ""),
|
|
291
|
+
"description": pi.get("description", ""), "datasheet": p.get("ComponentLink2URL", ""),
|
|
292
|
+
"lcsc": p.get("Supplier Part Number 1", "") if p.get("Supplier 1", "").upper() in ("LCSC", "JLCPCB") else ""}
|
|
293
|
+
ln.setdefault("mpn", item.get("mpn", ""))
|
|
294
|
+
ln.setdefault("manufacturer", item.get("manufacturer", ""))
|
|
295
|
+
ln.setdefault("qty", "1")
|
|
296
|
+
if not ln.get("designator"):
|
|
297
|
+
ln["designator"] = "U1"
|
|
298
|
+
return ln
|
|
299
|
+
|
|
300
|
+
|
|
301
|
+
def _find_plan_item(plan_items, item):
|
|
302
|
+
name = (item.get("build") or {}).get("part")
|
|
303
|
+
from .agent_queue import norm_manufacturer
|
|
304
|
+
mpn, mfr = (item.get("mpn") or "").strip().upper(), norm_manufacturer(item.get("manufacturer"))
|
|
305
|
+
hits = []
|
|
306
|
+
for pi in plan_items:
|
|
307
|
+
if name and pi.get("name") == name:
|
|
308
|
+
return pi
|
|
309
|
+
p = _par(pi)
|
|
310
|
+
if mpn and (p.get("Manufacturer Part Number", "").strip().upper() == mpn or pi.get("name", "").upper() == mpn):
|
|
311
|
+
hits.append(pi)
|
|
312
|
+
ln = (item.get("change") or {}).get("line") or {}
|
|
313
|
+
if not hits and not ln.get("mpn") and ln.get("value"): # a bare passive: matched by value like the librarian
|
|
314
|
+
same = [pi for pi in plan_items if not _par(pi).get("Manufacturer Part Number") and _par(pi).get("Value") == ln["value"]]
|
|
315
|
+
hits = [pi for pi in same if ln.get("footprint") in ((pi.get("footprint") or {}).get("ref"),
|
|
316
|
+
(pi.get("footprint") or {}).get("bom_ref"))] or same
|
|
317
|
+
return next((h for h in hits if norm_manufacturer(_par(h).get("Manufacturer")) == mfr), hits[0] if hits else None)
|
|
318
|
+
|
|
319
|
+
|
|
320
|
+
def write_bom(ws, lines):
|
|
321
|
+
"""<ws>/agent/library_bom.csv: one line per part the agent ever built (removed parts stay in the library)."""
|
|
322
|
+
f = Path(ws) / "agent" / BOM_NAME
|
|
323
|
+
f.parent.mkdir(parents=True, exist_ok=True)
|
|
324
|
+
tmp = f.with_name(f.name + ".tmp")
|
|
325
|
+
with open(tmp, "w", newline="") as fh:
|
|
326
|
+
w = SS.writer(fh) # read_bom undoes the formula guard
|
|
327
|
+
w.writerow([h for _, h in BOM_COLS])
|
|
328
|
+
for key in sorted(lines):
|
|
329
|
+
w.writerow([str(lines[key].get(k) or "") for k, _ in BOM_COLS])
|
|
330
|
+
os.replace(tmp, f)
|
|
331
|
+
return f
|
|
332
|
+
|
|
333
|
+
|
|
334
|
+
def librarian_builder(ws, bom, out, refresh, rules, log=None, libs=(), http=None, offline=False):
|
|
335
|
+
"""Default builder: the existing librarian on the agent's BOM. refresh: plan names to redo. The datasheet reader is
|
|
336
|
+
used when the rules' sources include it; with [agent] datasheet_batch it is one Message Batch that is submitted
|
|
337
|
+
without waiting (a later run collects it; parts waiting for it stay `building`)."""
|
|
338
|
+
from . import librarian as L
|
|
339
|
+
cfg, reader = settings(rules), None
|
|
340
|
+
cache = Path(ws) / "http_cache"
|
|
341
|
+
if http is None:
|
|
342
|
+
from .httpcache import CachedHTTP
|
|
343
|
+
http = CachedHTTP(cache, min_interval=0.5, offline=offline, timeout=60)
|
|
344
|
+
if cfg["datasheet_batch"] and "datasheet" in rules["sources"]["order"]:
|
|
345
|
+
from . import datasheet as D
|
|
346
|
+
reader = D.Reader(out, cache, None, rules, http, batch=True, wait=False, log=log or (lambda *a: None))
|
|
347
|
+
libs = [str(l) for l in libs] + [str(l) for l in cfg["libs"]] + [str(Path(ws) / "footprints")]
|
|
348
|
+
return L.run(str(bom), libs, out, None, list(refresh) if refresh else None, False, [], offline, cache, 0.5,
|
|
349
|
+
None, http=http, log=log or (lambda *a: None), rules=rules,
|
|
350
|
+
datasheet_batch=bool(reader), reader=reader)
|
|
351
|
+
|
|
352
|
+
|
|
353
|
+
# ------------------------------------------------------------------ pictures
|
|
354
|
+
def _render(prover, kind, src, png, notes):
|
|
355
|
+
try:
|
|
356
|
+
fn = prover.render_footprint if kind == "footprint" else prover.render_symbol
|
|
357
|
+
return str(fn(str(src), str(png)))
|
|
358
|
+
except Exception as e: # a picture is a help for the reviewer, never a reason to fail
|
|
359
|
+
notes.append(f"{kind} picture not drawn: {type(e).__name__}: {e}")
|
|
360
|
+
return None
|
|
361
|
+
|
|
362
|
+
|
|
363
|
+
def _model_files(pi, out):
|
|
364
|
+
"""{'footprint': kicad_mod or PcbLib, 'symbol': SchLib} that exist on disk for a plan item."""
|
|
365
|
+
from .release_check import _path
|
|
366
|
+
got = {}
|
|
367
|
+
fp = pi.get("footprint") or {}
|
|
368
|
+
for kind, cands in (("footprint", [fp.get("kicad_mod"), fp.get("file")]), ("symbol", [(pi.get("symbol") or {}).get("file")])):
|
|
369
|
+
for c in cands:
|
|
370
|
+
if c and _path(out, c).is_file():
|
|
371
|
+
got[kind] = _path(out, c)
|
|
372
|
+
break
|
|
373
|
+
return got
|
|
374
|
+
|
|
375
|
+
|
|
376
|
+
def _git_before(repo, out, pi, tmpdir):
|
|
377
|
+
"""The plan item's files as committed in the library repo's HEAD, when they are no longer on disk."""
|
|
378
|
+
if not repo:
|
|
379
|
+
return {}
|
|
380
|
+
got = {}
|
|
381
|
+
fp = pi.get("footprint") or {}
|
|
382
|
+
for kind, rel in (("footprint", fp.get("kicad_mod") or fp.get("file")), ("symbol", (pi.get("symbol") or {}).get("file"))):
|
|
383
|
+
if not rel:
|
|
384
|
+
continue
|
|
385
|
+
try:
|
|
386
|
+
p = (Path(out) / rel).resolve().relative_to(repo).as_posix()
|
|
387
|
+
data = subprocess.run(["git", "show", f"HEAD:{p}"], cwd=repo, capture_output=True).stdout
|
|
388
|
+
except (ValueError, OSError):
|
|
389
|
+
continue
|
|
390
|
+
if data:
|
|
391
|
+
f = Path(tmpdir) / f"before_{kind}{Path(rel).suffix}"
|
|
392
|
+
f.write_bytes(data)
|
|
393
|
+
got[kind] = f
|
|
394
|
+
return got
|
|
395
|
+
|
|
396
|
+
|
|
397
|
+
def pictures_for(ws, prover, item_id, files, which, notes):
|
|
398
|
+
d = Path(ws) / "agent" / "pictures" / _safe(item_id)
|
|
399
|
+
d.mkdir(parents=True, exist_ok=True)
|
|
400
|
+
out = []
|
|
401
|
+
for kind in ("footprint", "symbol"):
|
|
402
|
+
if kind in files:
|
|
403
|
+
p = _render(prover, kind, files[kind], d / f"{which}_{kind}.png", notes)
|
|
404
|
+
if p and Path(p).is_file():
|
|
405
|
+
out.append(Path(p).resolve().relative_to(Path(ws).resolve()).as_posix()
|
|
406
|
+
if Path(ws).resolve() in Path(p).resolve().parents else str(p))
|
|
407
|
+
return out
|
|
408
|
+
|
|
409
|
+
|
|
410
|
+
def _published(it):
|
|
411
|
+
"""What a batch PR shows of an item: its files and when it was built (a rebuild brings a new proof)."""
|
|
412
|
+
b = it.get("build") or {}
|
|
413
|
+
return f"{b.get('hash')}@{b.get('built')}"
|
|
414
|
+
|
|
415
|
+
|
|
416
|
+
def _hash_files(files):
|
|
417
|
+
h = hashlib.sha256()
|
|
418
|
+
for k in sorted(files):
|
|
419
|
+
h.update(k.encode()); h.update(Path(files[k]).read_bytes())
|
|
420
|
+
return h.hexdigest()[:16]
|
|
421
|
+
|
|
422
|
+
|
|
423
|
+
# ------------------------------------------------------------------ upkeep -> findings
|
|
424
|
+
def upkeep_findings(ws, rules, now=None, log=None):
|
|
425
|
+
"""Default MAINTAIN step: upkeep.run on every librarian plan in the workspace; -> list of finding dicts."""
|
|
426
|
+
from . import upkeep as UK
|
|
427
|
+
ws = Path(ws)
|
|
428
|
+
found, runs = [], []
|
|
429
|
+
for plan in sorted((ws / "librarian").glob("*/plan.json")):
|
|
430
|
+
base = ws / "agent" / "upkeep" / plan.parent.name
|
|
431
|
+
rep = UK.run(str(plan), None, base / "state", base / "upkeep", rules=rules,
|
|
432
|
+
ctx=UK.Ctx(cache=ws / "http_cache"), now=stamp(now), save=False)
|
|
433
|
+
runs.append((rep, base / "state"))
|
|
434
|
+
items = read_json(plan, [])
|
|
435
|
+
for sec in rep["checks"]:
|
|
436
|
+
for c in sec["changes"]:
|
|
437
|
+
found.append(dict(c, check=sec["check"], plan=plan.relative_to(ws).as_posix(), _items=items))
|
|
438
|
+
# the state (what upkeep remembers as reported) is saved only once every plan was checked: a later plan that fails
|
|
439
|
+
# must not leave the earlier ones' findings remembered as seen but never returned (never queued, never reported)
|
|
440
|
+
for rep, state in runs:
|
|
441
|
+
UK.save_state(rep, state)
|
|
442
|
+
return found
|
|
443
|
+
|
|
444
|
+
|
|
445
|
+
def finding_item(f, now):
|
|
446
|
+
"""A queue item (fields form) for one upkeep finding; the key is the plan part's manufacturer|MPN when known."""
|
|
447
|
+
items, name = f.get("_items") or [], str(f.get("item") or "")
|
|
448
|
+
pi = next((i for i in items if i.get("name") == name or _par(i).get("Manufacturer Part Number", "").upper() == name.upper()
|
|
449
|
+
or (i.get("footprint") or {}).get("ref") == name), None)
|
|
450
|
+
mpn, mfr = (_par(pi).get("Manufacturer Part Number", "") or pi.get("name", ""), _par(pi).get("Manufacturer", "")) if pi else (name, "")
|
|
451
|
+
change = {k: v for k, v in f.items() if not k.startswith("_")}
|
|
452
|
+
change.update(detected=stamp(now), part=pi.get("name") if pi else None)
|
|
453
|
+
return {"key": part_key(mfr, mpn), "mpn": mpn, "manufacturer": mfr, "origin": "upkeep", "change": change}
|
|
454
|
+
|
|
455
|
+
|
|
456
|
+
def upkeep_due(ws, cfg, now=None):
|
|
457
|
+
period = SCHEDULES.get(cfg["upkeep_schedule"])
|
|
458
|
+
if not period:
|
|
459
|
+
return False
|
|
460
|
+
last, t = _epoch(read_json(Path(ws) / "agent" / "agent_state.json", {}).get("last_upkeep")), _epoch(stamp(now))
|
|
461
|
+
# last in the future: the clock was ahead when it was stamped (then corrected); waiting for it would stop upkeep
|
|
462
|
+
return last is None or t is None or t - last >= period or last > t
|
|
463
|
+
|
|
464
|
+
|
|
465
|
+
# ------------------------------------------------------------------ PR body
|
|
466
|
+
_MD_SPECIAL = re.compile(r"([\\`*_{}\[\]()!|~])")
|
|
467
|
+
|
|
468
|
+
|
|
469
|
+
def _md(v):
|
|
470
|
+
"""Untrusted text (MPNs, BOM fields, notes, model answers) as one inert line of GitHub markdown: no HTML, links,
|
|
471
|
+
images, tables breaks, @mentions or #issue references (so no "fixes #12" closing an issue on merge)."""
|
|
472
|
+
s = " ".join(str(v if v is not None else "").split())
|
|
473
|
+
s = s.replace("&", "&").replace("<", "<").replace(">", ">")
|
|
474
|
+
s = _MD_SPECIAL.sub(r"\\\1", s)
|
|
475
|
+
return s.replace("@", "@").replace("#", "#")
|
|
476
|
+
|
|
477
|
+
|
|
478
|
+
_esc = _md
|
|
479
|
+
|
|
480
|
+
|
|
481
|
+
def _url(v):
|
|
482
|
+
"""A datasheet link for <...>: http(s) only, with nothing that could close the autolink."""
|
|
483
|
+
from urllib.parse import quote
|
|
484
|
+
v = str(v or "").strip()
|
|
485
|
+
return quote(v, safe=":/?#[]@!$&'()*+,;=%~.-_") if re.match(r"https?://[^\s]+$", v) else None
|
|
486
|
+
|
|
487
|
+
|
|
488
|
+
def pr_body(batch_id, items, link, t, removed=()):
|
|
489
|
+
from . import messages as M
|
|
490
|
+
sts = [((it.get("build") or {}).get("status")) for it in items if (it.get("build") or {}).get("status")]
|
|
491
|
+
md = [f"# Library agent batch {_md(batch_id)}", "",
|
|
492
|
+
f"**{M.plural(len(items), 'part')} to review**" + (f" ({_md(M.counts(sts))})" if sts else "") + f", built {_md(t)}.", "",
|
|
493
|
+
"**Next:** compare each part's pictures with its datasheet, then **Approve** or **Reject** it in the review queue "
|
|
494
|
+
"(`ecadlib ui`, Review queue, or `ecadlib agent approve ID`). Approved parts are released to Altium 365 by the "
|
|
495
|
+
"agent; merging this PR only keeps the library files in git.", ""]
|
|
496
|
+
for it in items:
|
|
497
|
+
ch, b, pr = it.get("change") or {}, it.get("build") or {}, it.get("proof") or {}
|
|
498
|
+
src = ch.get("source") or {}
|
|
499
|
+
what = ch.get("kind") or ch.get("check") or ""
|
|
500
|
+
md += [f"## {_md(it.get('mpn') or it['key'])}" + (f" ({_md(it['manufacturer'])})" if it.get("manufacturer") else ""), "",
|
|
501
|
+
f"Queue item `{_safe(it['id'])}`, {_md(it.get('origin', 'watch'))}: **{_md(what)}**"
|
|
502
|
+
+ (f" in {_md(src.get('type'))} {_md(src.get('name') or src.get('ref'))}" if src else "")
|
|
503
|
+
+ (f": {_md(ch.get('detail'))}" if it.get("origin") == "upkeep" and ch.get("detail") else ""), ""]
|
|
504
|
+
if b.get("part"):
|
|
505
|
+
md.append(f"Library part {_md(b['part'])}: **{_md(M.status_word(b.get('status', '?')))}**; symbol from "
|
|
506
|
+
f"{_md(M.source_word((b.get('source') or {}).get('symbol')))}, footprint from "
|
|
507
|
+
f"{_md(M.source_word((b.get('source') or {}).get('footprint')))}.")
|
|
508
|
+
md.append("")
|
|
509
|
+
elif b.get("action") == "none":
|
|
510
|
+
md += [f"Nothing rebuilt: {_md(b.get('reason', ''))}", ""]
|
|
511
|
+
ds = _url((ch.get("line") or {}).get("datasheet"))
|
|
512
|
+
if ds:
|
|
513
|
+
md += [f"Datasheet: <{ds}>", ""]
|
|
514
|
+
pics = it.get("pictures") or {}
|
|
515
|
+
before, after = pics.get("before") or [], pics.get("after") or []
|
|
516
|
+
if before or after:
|
|
517
|
+
md += ["| Before | After |", "|---|---|"]
|
|
518
|
+
for i in range(max(len(before), len(after))):
|
|
519
|
+
cell = lambda lst: f"![{_safe(Path(lst[i]).stem)}]({link(it, lst[i])})" if i < len(lst) else ("(new part)" if not lst else "")
|
|
520
|
+
md.append(f"| {cell(before)} | {cell(after)} |")
|
|
521
|
+
md.append("")
|
|
522
|
+
if pr:
|
|
523
|
+
md.append(f"**Independent check:** {_md(pr.get('verdict', '?'))}" + (f" (score {pr['score']:.2f})" if isinstance(pr.get("score"), (int, float)) else "")
|
|
524
|
+
+ (f": {_md(pr.get('reason'))}" if pr.get("reason") else "")
|
|
525
|
+
+ (f" ({_md(pr['model'])})" if pr.get("model") and pr.get("verdict") != "skipped" else ""))
|
|
526
|
+
if pr.get("findings"):
|
|
527
|
+
md += ["", "| Severity | What | Expected | Found | Page |", "|---|---|---|---|---|"]
|
|
528
|
+
md += [f"| {_md(f.get('severity'))} | {_md(f.get('what'))} | {_md(f.get('expected'))} | {_md(f.get('found'))} | "
|
|
529
|
+
f"{_md(f.get('page'))} |" for f in pr["findings"]]
|
|
530
|
+
md.append("")
|
|
531
|
+
doubts = list(b.get("hold_reasons") or []) + list(b.get("verify") or []) + list(b.get("notes") or [])
|
|
532
|
+
cc = b.get("cross_check") or {} # ecadlib.crosscheck: datasheet read against a second source
|
|
533
|
+
if cc.get("status") in ("disagreed", "unchecked"):
|
|
534
|
+
doubts.insert(0, f"cross-check {cc['status']}: {cc.get('summary', '')}")
|
|
535
|
+
doubts += [f"proof: {f.get('what')}" for f in pr.get("findings") or [] if f.get("severity") in ("error", "warning")]
|
|
536
|
+
if pr.get("verdict") in ("doubt", "skipped"):
|
|
537
|
+
doubts.insert(0, f"proof {pr['verdict']}: {pr.get('reason', '')}")
|
|
538
|
+
if doubts:
|
|
539
|
+
md += ["**Check before you approve:**", ""] + [f"- {_md(e['text'])} Next: {_md(e['fix'])}"
|
|
540
|
+
for e in M.explain_all(dict.fromkeys(map(str, doubts)))] + [""]
|
|
541
|
+
if removed:
|
|
542
|
+
md += ["## Taken out of this PR", "", "These parts were rejected, failed or moved to another batch; their files "
|
|
543
|
+
"and pictures were removed from this branch:", ""]
|
|
544
|
+
md += [f"- {_md(r.get('mpn') or r.get('id'))}: {_md(r.get('status'))}" for r in removed] + [""]
|
|
545
|
+
md.append("<sub>Opened by ecadlib agent. It never force-pushes; re-runs update this PR.</sub>")
|
|
546
|
+
text = "\n".join(md) + "\n"
|
|
547
|
+
return text if len(text) <= BODY_MAX else text[:BODY_MAX - 80] + "\n\n(cut: the full text is in the batch folder)\n"
|
|
548
|
+
|
|
549
|
+
|
|
550
|
+
# ------------------------------------------------------------------ batch PR
|
|
551
|
+
def _departed(it, batch_id):
|
|
552
|
+
"""An item whose files must leave a batch branch: gone, rejected, failed, or now in another batch."""
|
|
553
|
+
return it is None or it.get("batch") != batch_id or it.get("status") in ("rejected", "failed")
|
|
554
|
+
|
|
555
|
+
|
|
556
|
+
def publish_batch(ws, rules, batch_id, items, out, now=None, env=None, session=None, log=None, lookup=None):
|
|
557
|
+
"""Write <ws>/agent/batches/<id>/ (PR.md, pictures, batch.json) and, with a git repo + remote + token, commit the
|
|
558
|
+
library files and pictures to ecadlib/agent-<id> and open or update its PR. Files of items that left the batch
|
|
559
|
+
(lookup(id) -> the item now, or None: rejected, failed, moved) are removed from the branch and the PR body says so.
|
|
560
|
+
-> the batch record."""
|
|
561
|
+
ws, cfg, env = Path(ws), settings(rules), os.environ if env is None else env
|
|
562
|
+
log = log or (lambda *a: None)
|
|
563
|
+
t = stamp(now)
|
|
564
|
+
bdir = ws / "agent" / "batches" / _safe(batch_id)
|
|
565
|
+
rec = read_json(bdir / "batch.json", {"id": batch_id, "branch": f"ecadlib/agent-{_safe(batch_id)}", "pr_url": None,
|
|
566
|
+
"pr_number": None, "published": {}})
|
|
567
|
+
rec.update(items=[it["id"] for it in items], updated=t)
|
|
568
|
+
by_id = {it["id"]: it for it in items}
|
|
569
|
+
lookup = lookup or by_id.get
|
|
570
|
+
rec.setdefault("paths", {})
|
|
571
|
+
departed = [i for i in rec["paths"] if i not in by_id and _departed(lookup(i), batch_id)]
|
|
572
|
+
removed = list(rec.get("removed") or []) + [
|
|
573
|
+
{"id": i, "mpn": (lookup(i) or {}).get("mpn"), "status": (lookup(i) or {}).get("status") or "gone"}
|
|
574
|
+
for i in departed]
|
|
575
|
+
bdir.mkdir(parents=True, exist_ok=True)
|
|
576
|
+
pic_dir = bdir / "pictures"
|
|
577
|
+
for it in items: # pictures next to the PR text, for the folder fallback and the web UI
|
|
578
|
+
for p in (it.get("pictures") or {}).get("before", []) + (it.get("pictures") or {}).get("after", []):
|
|
579
|
+
src = ws / p
|
|
580
|
+
if src.is_file():
|
|
581
|
+
(pic_dir / _safe(it["id"])).mkdir(parents=True, exist_ok=True)
|
|
582
|
+
shutil.copy2(src, pic_dir / _safe(it["id"]) / src.name)
|
|
583
|
+
local_link = lambda it, p: f"pictures/{_safe(it['id'])}/{Path(p).name}"
|
|
584
|
+
(bdir / "PR.md").write_text(pr_body(batch_id, items, local_link, t, removed))
|
|
585
|
+
|
|
586
|
+
repo_cfg = cfg["library_repo"]
|
|
587
|
+
repo = repo_root(Path(repo_cfg) if repo_cfg else ws)
|
|
588
|
+
reason = None
|
|
589
|
+
if repo is None:
|
|
590
|
+
reason = "no git repository (set [agent] library_repo or make the workspace a git repo)"
|
|
591
|
+
files = {}
|
|
592
|
+
if repo is not None:
|
|
593
|
+
out = Path(out).resolve()
|
|
594
|
+
inside = repo in out.parents
|
|
595
|
+
libdir = cfg["library_path"].strip("/") or "library"
|
|
596
|
+
rel = lambda p: (Path(p).resolve().relative_to(repo).as_posix() if inside
|
|
597
|
+
else f"{libdir}/{Path(p).resolve().relative_to(out).as_posix()}")
|
|
598
|
+
from .release_check import _path
|
|
599
|
+
for it in items:
|
|
600
|
+
b = it.get("build") or {}
|
|
601
|
+
for f in b.get("files", {}).values():
|
|
602
|
+
if f and _path(out, f).is_file():
|
|
603
|
+
files[rel(_path(out, f))] = _path(out, f)
|
|
604
|
+
own = {it["id"]: set() for it in items}
|
|
605
|
+
for it in items:
|
|
606
|
+
for f in ((it.get("build") or {}).get("files") or {}).values():
|
|
607
|
+
if f and _path(out, f).is_file():
|
|
608
|
+
own[it["id"]].add(rel(_path(out, f)))
|
|
609
|
+
if (out / "plan.json").is_file():
|
|
610
|
+
files[rel(out / "plan.json")] = out / "plan.json"
|
|
611
|
+
for it in items:
|
|
612
|
+
for p in (it.get("pictures") or {}).get("before", []) + (it.get("pictures") or {}).get("after", []):
|
|
613
|
+
if (ws / p).is_file():
|
|
614
|
+
key = f"{cfg['pictures_path'].strip('/')}/{_safe(batch_id)}/{_safe(it['id'])}/{Path(p).name}"
|
|
615
|
+
files[key] = ws / p
|
|
616
|
+
own[it["id"]].add(key)
|
|
617
|
+
# files an item had on the branch from an earlier build and no longer has (a rebuild with fewer pictures; a
|
|
618
|
+
# part removed from its BOM: nothing built, no AFTER pictures). Its old pictures (only ever on this branch)
|
|
619
|
+
# leave now; other files stay recorded as its own, so they leave with it if it departs, never forgotten
|
|
620
|
+
pics = lambda i: f"{cfg['pictures_path'].strip('/')}/{_safe(batch_id)}/{_safe(i)}/"
|
|
621
|
+
stale = {i: set(rec["paths"][i]) - own[i] for i in own if i in rec["paths"]}
|
|
622
|
+
for i, ps in stale.items():
|
|
623
|
+
own[i] |= {x for x in ps if not x.startswith(pics(i))}
|
|
624
|
+
keep = set(files) | {x for i, ps in rec["paths"].items() if i not in departed and i not in own for x in ps} \
|
|
625
|
+
| {x for ps in own.values() for x in ps}
|
|
626
|
+
drop = ({x for i in departed for x in rec["paths"][i]} | {x for ps in stale.values() for x in ps}) - keep
|
|
627
|
+
try:
|
|
628
|
+
base = cfg["base_branch"] or git(repo, "symbolic-ref", "--short", "HEAD")
|
|
629
|
+
remote = git(repo, "remote", "get-url", "origin", ok=(0, 2, 128))
|
|
630
|
+
from . import forge
|
|
631
|
+
kind = forge.host_kind(cfg, remote)
|
|
632
|
+
if kind == "gitlab":
|
|
633
|
+
slug = cfg["gitlab_project"] or forge.project_from_remote(remote, cfg["gitlab_api"])
|
|
634
|
+
tok_env, host = cfg["gitlab_token_env"], "GitLab"
|
|
635
|
+
else:
|
|
636
|
+
slug, tok_env, host = cfg["github_repo"] or github_slug(remote), cfg["github_token_env"], "GitHub"
|
|
637
|
+
token = env.get(tok_env) if tok_env else None
|
|
638
|
+
start = base
|
|
639
|
+
if remote:
|
|
640
|
+
# start from the remote's base, never the local one: local commits nobody pushed (work in progress,
|
|
641
|
+
# private files) must not reach origin inside the agent's branch. FETCH_HEAD only: no ref, index or
|
|
642
|
+
# working tree of the repo changes.
|
|
643
|
+
git(repo, "fetch", "--no-tags", "origin", f"refs/heads/{base}")
|
|
644
|
+
start = git(repo, "rev-parse", "--verify", "FETCH_HEAD^{commit}")
|
|
645
|
+
# build on what is on the remote branch (a person may have pushed a fix)
|
|
646
|
+
rs = git(repo, "ls-remote", "origin", f"refs/heads/{rec['branch']}")
|
|
647
|
+
if rs:
|
|
648
|
+
git(repo, "fetch", "--no-tags", "origin", f"+refs/heads/{rec['branch']}:refs/heads/{rec['branch']}")
|
|
649
|
+
msg = f"ecadlib agent batch {batch_id}: {len(items)} part(s)\n\n" + "\n".join(
|
|
650
|
+
f"- {it.get('mpn') or it['key']} ({(it.get('change') or {}).get('kind') or (it.get('change') or {}).get('check') or ''})" for it in items)
|
|
651
|
+
if departed:
|
|
652
|
+
msg += "\n\nTaken out: " + ", ".join(str(r["mpn"] or r["id"]) for r in removed[-len(departed):])
|
|
653
|
+
sha, changed = commit_files(repo, rec["branch"], start, files, msg, remove=drop)
|
|
654
|
+
rec.update(repo=str(repo), base=base, commit=sha)
|
|
655
|
+
for i in departed: # committed: their files are out of the branch now
|
|
656
|
+
rec["paths"].pop(i, None)
|
|
657
|
+
rec["removed"] = removed
|
|
658
|
+
rec["paths"].update({i: sorted(ps) for i, ps in own.items()})
|
|
659
|
+
if not remote:
|
|
660
|
+
reason = f"branch {rec['branch']} committed in {repo}, but the repo has no origin remote: no PR"
|
|
661
|
+
elif not token:
|
|
662
|
+
reason = (f"branch {rec['branch']} committed locally; no {host} token in ${tok_env}: "
|
|
663
|
+
"not pushed, no PR")
|
|
664
|
+
elif not slug:
|
|
665
|
+
reason = (f"origin is not a {host} remote and [agent] {'gitlab_project' if kind == 'gitlab' else 'github_repo'}"
|
|
666
|
+
" is empty: not pushed, no PR")
|
|
667
|
+
elif kind == "gitlab" and not cfg["gitlab_project"] and forge.wrong_api_host(cfg["gitlab_api"], remote):
|
|
668
|
+
reason = f"branch {rec['branch']} committed locally; {forge.wrong_api_host(cfg['gitlab_api'], remote)}: no MR"
|
|
669
|
+
elif kind == "gitlab":
|
|
670
|
+
git(repo, "push", "origin", f"refs/heads/{rec['branch']}:refs/heads/{rec['branch']}") # never --force
|
|
671
|
+
gl = forge.GitLab(slug, token, session, cfg["gitlab_api"])
|
|
672
|
+
link = lambda it, p: gl.raw_url(sha, f"{cfg['pictures_path'].strip('/')}/{_safe(batch_id)}/"
|
|
673
|
+
f"{_safe(it['id'])}/{Path(p).name}")
|
|
674
|
+
body = pr_body(batch_id, items, link, t, removed)
|
|
675
|
+
forge.upsert_mr(gl, rec, f"Library agent: {len(items)} part(s) to review (batch {batch_id})", body, base)
|
|
676
|
+
log(f"agent: MR {rec['pr_action']} {rec['pr_url']}")
|
|
677
|
+
else:
|
|
678
|
+
git(repo, "push", "origin", f"refs/heads/{rec['branch']}:refs/heads/{rec['branch']}") # never --force
|
|
679
|
+
from .upkeep import GitHub
|
|
680
|
+
gh = GitHub(slug, token, session)
|
|
681
|
+
gh.API = cfg["github_api"].rstrip("/")
|
|
682
|
+
web = "https://github.com" if gh.API == "https://api.github.com" else re.sub(r"/api/v3$", "", gh.API)
|
|
683
|
+
link = lambda it, p: (f"{web}/{slug}/blob/{sha}/{cfg['pictures_path'].strip('/')}/{_safe(batch_id)}/"
|
|
684
|
+
f"{_safe(it['id'])}/{Path(p).name}?raw=true")
|
|
685
|
+
body, title = pr_body(batch_id, items, link, t, removed), f"Library agent: {len(items)} part(s) to review (batch {batch_id})"
|
|
686
|
+
num = rec.get("pr_number")
|
|
687
|
+
if num: # merged or closed meanwhile: a closed PR is not a review; open a new one
|
|
688
|
+
cur = gh.api("GET", f"pulls/{num}")
|
|
689
|
+
if isinstance(cur, dict) and cur.get("state") == "closed":
|
|
690
|
+
rec.setdefault("closed_prs", []).append(rec.get("pr_url"))
|
|
691
|
+
num, rec["pr_number"], rec["pr_url"] = None, None, None
|
|
692
|
+
if not num:
|
|
693
|
+
owner = slug.split("/")[0]
|
|
694
|
+
open_ = gh.api("GET", f"pulls?state=open&head={owner}:{rec['branch']}&per_page=10") or []
|
|
695
|
+
num = open_[0]["number"] if open_ else None
|
|
696
|
+
rec["pr_url"] = open_[0].get("html_url") if open_ else rec.get("pr_url")
|
|
697
|
+
if num:
|
|
698
|
+
r = gh.api("PATCH", f"pulls/{num}", {"title": title, "body": body})
|
|
699
|
+
rec.update(pr_action="updated")
|
|
700
|
+
else:
|
|
701
|
+
r = gh.api("POST", "pulls", {"title": title, "head": rec["branch"], "base": base, "body": body})
|
|
702
|
+
rec.update(pr_action="opened")
|
|
703
|
+
rec.update(pr_number=(r or {}).get("number") or num, pr_url=(r or {}).get("html_url") or rec.get("pr_url"))
|
|
704
|
+
log(f"agent: PR {rec['pr_action']} {rec['pr_url']}")
|
|
705
|
+
except Exception as e: # git or GitHub failed: the folder is still there
|
|
706
|
+
reason = f"{type(e).__name__}: {e}"
|
|
707
|
+
rec["fallback"] = reason
|
|
708
|
+
rec["folder"] = bdir.relative_to(ws).as_posix()
|
|
709
|
+
rec["published"] = {it["id"]: _published(it) for it in items}
|
|
710
|
+
write_json(bdir / "batch.json", rec)
|
|
711
|
+
if reason:
|
|
712
|
+
log(f"agent: batch {batch_id} written to {bdir} ({reason})")
|
|
713
|
+
return rec
|
|
714
|
+
|
|
715
|
+
|
|
716
|
+
def _github(ws, cfg, env, session):
|
|
717
|
+
"""(GitHub or GitLab client, None) for the library repo's PRs / MRs, or (None, why not)."""
|
|
718
|
+
repo = repo_root(Path(cfg["library_repo"]) if cfg["library_repo"] else ws)
|
|
719
|
+
if repo is None:
|
|
720
|
+
return None, "no git repository"
|
|
721
|
+
remote = git(repo, "remote", "get-url", "origin", ok=(0, 2, 128))
|
|
722
|
+
from . import forge
|
|
723
|
+
kind, gl, why = forge.client(cfg, remote, env, session)
|
|
724
|
+
if kind == "gitlab":
|
|
725
|
+
return gl, why
|
|
726
|
+
slug = cfg["github_repo"] or github_slug(remote)
|
|
727
|
+
token = env.get(cfg["github_token_env"]) if cfg["github_token_env"] else None
|
|
728
|
+
if not (remote and slug and token):
|
|
729
|
+
return None, "no GitHub remote or token"
|
|
730
|
+
from .upkeep import GitHub
|
|
731
|
+
gh = GitHub(slug, token, session)
|
|
732
|
+
gh.API = cfg["github_api"].rstrip("/")
|
|
733
|
+
return gh, None
|
|
734
|
+
|
|
735
|
+
|
|
736
|
+
def close_batch(ws, rules, batch_id, members, now=None, env=None, session=None, log=None):
|
|
737
|
+
"""Every item of the batch is decided (pushed, rejected or failed) and none waits for its push. With a pushed part
|
|
738
|
+
the PR is how the library repo gets its files: one "all parts decided; ready to merge" comment, and it stays
|
|
739
|
+
open (rec["ready"]). With nothing pushed: an outcome comment and the PR is closed (rec["closed"]), so PRs never
|
|
740
|
+
pile up. Nothing happens twice. -> rec."""
|
|
741
|
+
ws, cfg, env = Path(ws), settings(rules), os.environ if env is None else env
|
|
742
|
+
bdir = ws / "agent" / "batches" / _safe(batch_id)
|
|
743
|
+
rec = read_json(bdir / "batch.json", {"id": batch_id})
|
|
744
|
+
if rec.get("closed") or rec.get("ready"):
|
|
745
|
+
return rec
|
|
746
|
+
pushed = [m for m in members if m["status"] == "pushed"]
|
|
747
|
+
lines = "\n".join([f"- {_md(m.get('mpn') or m['key'])}: {_md(m['status'])}" for m in members] or ["- (no parts left)"])
|
|
748
|
+
if pushed:
|
|
749
|
+
text = (f"All parts decided; ready to merge.\n\n{lines}\n\nThe pushed parts are in Altium 365; merging this PR "
|
|
750
|
+
"adds their library files to the library repo. Files of rejected or failed parts were taken out.")
|
|
751
|
+
else:
|
|
752
|
+
text = f"Every part of this batch is decided and none was pushed, so the library agent closes this PR:\n\n{lines}"
|
|
753
|
+
if rec.get("pr_number") and rec.get("pr_state") != "closed":
|
|
754
|
+
try:
|
|
755
|
+
gh, why = _github(ws, cfg, env, session)
|
|
756
|
+
if gh is None:
|
|
757
|
+
raise RuntimeError(f"PR not updated: {why}")
|
|
758
|
+
gitlab = getattr(gh, "kind", "") == "gitlab"
|
|
759
|
+
if not rec.get("commented"): # a close that fails after it is retried without a 2nd comment
|
|
760
|
+
if gitlab:
|
|
761
|
+
gh.comment_mr(rec["pr_number"], text)
|
|
762
|
+
else:
|
|
763
|
+
gh.api("POST", f"issues/{rec['pr_number']}/comments", {"body": text})
|
|
764
|
+
rec["commented"] = stamp(now)
|
|
765
|
+
if not pushed: # a PR holding pushed parts is never closed by the agent
|
|
766
|
+
if gitlab:
|
|
767
|
+
gh.close_mr(rec["pr_number"])
|
|
768
|
+
else:
|
|
769
|
+
gh.api("PATCH", f"pulls/{rec['pr_number']}", {"state": "closed"})
|
|
770
|
+
rec["pr_state"] = "closed"
|
|
771
|
+
except Exception as e: # tried again next pass
|
|
772
|
+
rec["close_note"] = f"{type(e).__name__}: {e}"
|
|
773
|
+
if bdir.is_dir():
|
|
774
|
+
write_json(bdir / "batch.json", rec)
|
|
775
|
+
return rec
|
|
776
|
+
rec.update({"ready" if pushed else "closed": stamp(now)}, outcome={m["id"]: m["status"] for m in members})
|
|
777
|
+
rec.pop("close_note", None)
|
|
778
|
+
bdir.mkdir(parents=True, exist_ok=True)
|
|
779
|
+
write_json(bdir / "batch.json", rec)
|
|
780
|
+
(log or (lambda *a: None))(f"agent: batch {batch_id} {'ready to merge' if pushed else 'closed'}")
|
|
781
|
+
return rec
|
|
782
|
+
|
|
783
|
+
|
|
784
|
+
# ------------------------------------------------------------------ push
|
|
785
|
+
def push_approved(workspace, rules=None, dry_run=False, queue=None, a365=None, env=None, now=None):
|
|
786
|
+
"""Approved queue items -> Altium 365 through altium365.push (release check and approvals still gate each part).
|
|
787
|
+
-> {"results": [...], "pushed": [ids], "skipped": reason or None}."""
|
|
788
|
+
from . import altium365 as A
|
|
789
|
+
from .approvals import load_policy
|
|
790
|
+
ws, env = Path(workspace).resolve(), os.environ if env is None else env
|
|
791
|
+
rules = RU.resolve(rules, ws)
|
|
792
|
+
q = queue if queue is not None else _queue(ws)
|
|
793
|
+
approved = q.list(status="approved")
|
|
794
|
+
out = _librarian_out(ws, ws / "agent" / BOM_NAME)
|
|
795
|
+
plan = read_json(out / "plan.json", [])
|
|
796
|
+
rep = {"results": [], "pushed": [], "skipped": None, "dry_run": dry_run, "approved": [i["id"] for i in approved]}
|
|
797
|
+
if not approved:
|
|
798
|
+
return rep
|
|
799
|
+
url = (rules.get("altium365") or {}).get("workspace") or ""
|
|
800
|
+
if not dry_run and not url:
|
|
801
|
+
rep["skipped"] = "no Altium 365 workspace: set [altium365] workspace in ecadlib.toml"
|
|
802
|
+
return rep
|
|
803
|
+
client = a365
|
|
804
|
+
if client is None and not dry_run:
|
|
805
|
+
tok = env.get("NEXAR_TOKEN")
|
|
806
|
+
if not tok:
|
|
807
|
+
rep["skipped"] = "NEXAR_TOKEN is not set: approved parts wait for the push"
|
|
808
|
+
return rep
|
|
809
|
+
client = A.Nexar(tok)
|
|
810
|
+
pairs = []
|
|
811
|
+
for it in approved:
|
|
812
|
+
if (it.get("build") or {}).get("action") == "none":
|
|
813
|
+
# a removal or an upkeep finding: what was approved is "nothing to build", not the library files (which
|
|
814
|
+
# may never have been approved): nothing is sent; the item is closed
|
|
815
|
+
rep["results"].append({"id": it["id"], "action": "none", "detail": "nothing was built: nothing sent"})
|
|
816
|
+
if not dry_run:
|
|
817
|
+
try:
|
|
818
|
+
q.transition(it["id"], "pushed", user=AGENT, note="nothing was built: nothing sent to Altium 365")
|
|
819
|
+
_followup(q, it)
|
|
820
|
+
except (ValueError, KeyError) as e:
|
|
821
|
+
rep["results"][-1]["queue"] = f"{type(e).__name__}: {e}"
|
|
822
|
+
continue
|
|
823
|
+
pi = _find_plan_item(plan, it)
|
|
824
|
+
if pi is None:
|
|
825
|
+
rep["results"].append({"id": it["id"], "action": "skip", "detail": "not in the agent's library plan"})
|
|
826
|
+
continue
|
|
827
|
+
why = _not_as_approved(it, pi, out)
|
|
828
|
+
if why:
|
|
829
|
+
rep["results"].append({"id": it["id"], "action": "skip", "detail": why})
|
|
830
|
+
continue
|
|
831
|
+
pi = copy.deepcopy(pi)
|
|
832
|
+
for k in ("symbol", "footprint"):
|
|
833
|
+
if pi[k].get("file"):
|
|
834
|
+
pi[k]["file"] = str((out / pi[k]["file"]).resolve())
|
|
835
|
+
pairs.append((it, pi))
|
|
836
|
+
if not dry_run: # a person may have rejected one while the plan was read
|
|
837
|
+
live = []
|
|
838
|
+
for it, pi in pairs:
|
|
839
|
+
try:
|
|
840
|
+
cur = q.get(it["id"])
|
|
841
|
+
except (KeyError, ValueError):
|
|
842
|
+
continue
|
|
843
|
+
if cur.get("status") == "approved" and _published(cur) == _published(it):
|
|
844
|
+
live.append((cur, pi))
|
|
845
|
+
else:
|
|
846
|
+
rep["results"].append({"id": it["id"], "action": "skip", "detail": f"now {cur.get('status')}: not pushed"})
|
|
847
|
+
pairs = live
|
|
848
|
+
results = A.push([p for _, p in pairs], client, url or "https://<workspace>", dry_run=dry_run,
|
|
849
|
+
policy=load_policy(out), rules=rules) if pairs else []
|
|
850
|
+
for (it, _), res in zip(pairs, results):
|
|
851
|
+
res = dict(res, id=it["id"])
|
|
852
|
+
rep["results"].append(res)
|
|
853
|
+
if dry_run:
|
|
854
|
+
continue
|
|
855
|
+
try:
|
|
856
|
+
q.update(it["id"], push=dict(res, at=stamp(now)))
|
|
857
|
+
if res["action"] in ("release", "update_parameters", "up_to_date"):
|
|
858
|
+
q.transition(it["id"], "pushed", user=AGENT, note=f"Altium 365: {res['action']}")
|
|
859
|
+
rep["pushed"].append(it["id"])
|
|
860
|
+
_followup(q, it)
|
|
861
|
+
except (ValueError, KeyError) as e: # changed by a person meanwhile: the push result is still in the report
|
|
862
|
+
res["queue"] = f"{type(e).__name__}: {e}"
|
|
863
|
+
return rep
|
|
864
|
+
|
|
865
|
+
|
|
866
|
+
def _followup(q, it):
|
|
867
|
+
"""Its source changed after the approval: that change is new work (a new item, now that this one is closed)."""
|
|
868
|
+
nxt = it.get("followup")
|
|
869
|
+
if nxt:
|
|
870
|
+
q.upsert({"key": it["key"], "mpn": it.get("mpn") or nxt.get("mpn", ""), "manufacturer": it.get("manufacturer", ""),
|
|
871
|
+
"origin": it.get("origin") or "watch", "change": nxt})
|
|
872
|
+
|
|
873
|
+
|
|
874
|
+
def _not_as_approved(it, pi, out):
|
|
875
|
+
"""Why an approved item must not be pushed: its library files are no longer the build that was approved (rebuilt
|
|
876
|
+
since, or the approval was given on another build). None when they are."""
|
|
877
|
+
b = it.get("build") or {}
|
|
878
|
+
ap = it.get("approval") or {}
|
|
879
|
+
if ap.get("build") and ap["build"] != _published(it):
|
|
880
|
+
return "approved on another build than the current one: review it again"
|
|
881
|
+
if b.get("hash"):
|
|
882
|
+
files = _model_files(pi, out)
|
|
883
|
+
try:
|
|
884
|
+
now_hash = _hash_files(files) if files else None
|
|
885
|
+
except OSError:
|
|
886
|
+
now_hash = None
|
|
887
|
+
if now_hash != b["hash"]:
|
|
888
|
+
return "its library files changed since the approved build: reject and retry it to review the new build"
|
|
889
|
+
return None
|
|
890
|
+
|
|
891
|
+
|
|
892
|
+
# ------------------------------------------------------------------ one pass
|
|
893
|
+
def run_once(workspace, rules=None, dry_run=False, client=None, now=None, *, watcher=None, queue=None, prover=None,
|
|
894
|
+
builder=None, upkeep=None, do_upkeep=None, github_session=None, a365=None, env=None, libs=(), log=None,
|
|
895
|
+
notifier=None):
|
|
896
|
+
"""One pass: scan -> queue -> build -> pictures -> prove -> review/failed -> batch PR -> push (auto_push) ->
|
|
897
|
+
upkeep (when due). client goes to the watcher and the prover (their API clients). Returns a report dict.
|
|
898
|
+
Injectable for tests: watcher/queue/prover (the sibling modules), builder(ws, bom, out, refresh, rules, log),
|
|
899
|
+
upkeep(ws, rules, now) -> findings, github_session (requests-like), a365 (Nexar-like client), env,
|
|
900
|
+
notifier(url, body, headers, timeout) -> HTTP status (the [notify] webhook sender; notify.http_sender by default)."""
|
|
901
|
+
ws = Path(workspace).resolve()
|
|
902
|
+
rules = RU.resolve(rules, ws)
|
|
903
|
+
from . import migrate as MG # agent/ written by a newer ecadlib: stop here; older: stamped
|
|
904
|
+
MG.open_folder(ws / "agent", ["agent"], write=not dry_run and (ws / "agent").is_dir())
|
|
905
|
+
from .agent_queue import use_rules
|
|
906
|
+
use_rules(rules) # [agent] manufacturer_aliases for part keys
|
|
907
|
+
cfg, t, log = settings(rules), stamp(now), log or (lambda *a: None)
|
|
908
|
+
env = os.environ if env is None else env
|
|
909
|
+
rep = {"started": t, "dry_run": dry_run, "detected": [], "built": [], "review": [], "failed": [], "waiting": [],
|
|
910
|
+
"batches": [], "push": None, "upkeep": None, "notes": []}
|
|
911
|
+
with run_lock(ws, create=not dry_run) as got:
|
|
912
|
+
if not got:
|
|
913
|
+
rep["skipped"] = "another agent run is in progress"
|
|
914
|
+
return rep
|
|
915
|
+
watcher, q, prover = _modules(watcher, queue, prover, ws, dry_run)
|
|
916
|
+
if not dry_run and hasattr(q, "migrate_keys"):
|
|
917
|
+
q.migrate_keys() # once: items keyed before manufacturer names were normalised
|
|
918
|
+
if not dry_run:
|
|
919
|
+
write_status(ws, state="running", run_started=t, dry_run=False)
|
|
920
|
+
try:
|
|
921
|
+
_pass(ws, rules, cfg, t, dry_run, client, now, watcher, q, prover, builder, upkeep, do_upkeep,
|
|
922
|
+
github_session, a365, env, libs, log, rep)
|
|
923
|
+
except Exception as e:
|
|
924
|
+
rep["error"] = f"{type(e).__name__}: {e}"
|
|
925
|
+
if not dry_run:
|
|
926
|
+
write_status(ws, state="error", last_error=rep["error"], last_run=_summary(rep))
|
|
927
|
+
raise
|
|
928
|
+
if not dry_run and (rules.get("notify") or {}).get("targets"):
|
|
929
|
+
rep["notify"] = _notify(rules, rep, q, env, log, notifier)
|
|
930
|
+
rep["finished"] = stamp() if now is None else t
|
|
931
|
+
if not dry_run:
|
|
932
|
+
write_status(ws, state="idle", last_run=_summary(rep), enabled=cfg["enabled"])
|
|
933
|
+
return rep
|
|
934
|
+
|
|
935
|
+
|
|
936
|
+
def _notify(rules, rep, q, env, log, sender=None):
|
|
937
|
+
"""[notify] webhooks for this pass (batch ready, parts held, upkeep findings, lifecycle). Never raises: a failed
|
|
938
|
+
post is in the report, the run goes on."""
|
|
939
|
+
try:
|
|
940
|
+
from . import notify as N
|
|
941
|
+
return N.send(rules, N.agent_events(rep, lambda i: _get_or_none(q, i)), env=env, sender=sender, log=log)
|
|
942
|
+
except Exception as e:
|
|
943
|
+
return [{"target": "-", "event": "-", "result": f"error: {type(e).__name__}: {e}"}]
|
|
944
|
+
|
|
945
|
+
|
|
946
|
+
def _summary(rep):
|
|
947
|
+
return {k: (len(v) if isinstance(v, list) else v) for k, v in rep.items()
|
|
948
|
+
if k in ("started", "finished", "detected", "built", "review", "failed", "waiting", "error", "skipped")} | {
|
|
949
|
+
"batches": [{k: b.get(k) for k in ("id", "pr_url", "fallback", "folder")} for b in rep["batches"]],
|
|
950
|
+
"pushed": len((rep.get("push") or {}).get("pushed", [])), "upkeep": (rep.get("upkeep") or {}).get("findings")}
|
|
951
|
+
|
|
952
|
+
|
|
953
|
+
def _pass(ws, rules, cfg, t, dry_run, client, now, watcher, q, prover, builder, upkeep, do_upkeep, session, a365, env,
|
|
954
|
+
libs, log, rep):
|
|
955
|
+
# WATCH (a dry run puts the watch state back, so the next real run still sees these changes). A real run keeps
|
|
956
|
+
# the state from before the scan in watch_pending.json until every change is in the queue: a run that dies in
|
|
957
|
+
# between (killed, power cut) gets the old state back next time and sees the same changes again, never loses them.
|
|
958
|
+
wfile, pend = ws / "agent" / "watch_state.json", ws / "agent" / "watch_pending.json"
|
|
959
|
+
saved = wfile.read_bytes() if dry_run and wfile.exists() else None
|
|
960
|
+
if not dry_run:
|
|
961
|
+
old = read_json(pend, None)
|
|
962
|
+
if isinstance(old, dict) and "state" in old:
|
|
963
|
+
if old["state"] is None:
|
|
964
|
+
wfile.unlink(missing_ok=True)
|
|
965
|
+
else:
|
|
966
|
+
write_json_text(wfile, old["state"])
|
|
967
|
+
rep["notes"].append("the last run stopped before queueing what it saw: scanning again from before it")
|
|
968
|
+
else:
|
|
969
|
+
write_json(pend, {"state": wfile.read_text() if wfile.exists() else None, "at": t})
|
|
970
|
+
try:
|
|
971
|
+
import inspect
|
|
972
|
+
kw = {"dry_run": True} if dry_run and "dry_run" in inspect.signature(watcher.scan).parameters else {}
|
|
973
|
+
changes = watcher.scan(ws, rules=rules, client=client, now=None if now is None else t, **kw) or [] # t: ISO
|
|
974
|
+
finally:
|
|
975
|
+
if dry_run:
|
|
976
|
+
if saved is not None:
|
|
977
|
+
wfile.write_bytes(saved)
|
|
978
|
+
elif wfile.exists():
|
|
979
|
+
wfile.unlink()
|
|
980
|
+
rep["detected"] = [{k: c.get(k) for k in ("key", "mpn", "kind")} for c in changes]
|
|
981
|
+
|
|
982
|
+
# MAINTAIN: upkeep findings become queue items (on its schedule, or when asked)
|
|
983
|
+
due = upkeep_due(ws, cfg, now) if do_upkeep is None else do_upkeep
|
|
984
|
+
upend = ws / "agent" / "upkeep_pending.json"
|
|
985
|
+
findings = [] if dry_run else read_json(upend, []) # findings a dead run computed but did not queue
|
|
986
|
+
findings = findings if isinstance(findings, list) else []
|
|
987
|
+
if due and not dry_run:
|
|
988
|
+
try:
|
|
989
|
+
raw = (upkeep or upkeep_findings)(ws, rules, now)
|
|
990
|
+
new = [finding_item(f, now) for f in raw if str(f.get("kind", "")).lower() not in SKIP_UPKEEP]
|
|
991
|
+
findings += new
|
|
992
|
+
write_json(upend, findings) # upkeep remembers them as seen: keep them until they are queued
|
|
993
|
+
rep["upkeep"] = {"ran": t, "findings": len(new), "items": [
|
|
994
|
+
{k: (f.get("change") or {}).get(k) for k in ("check", "kind", "item", "was", "now")} for f in new]}
|
|
995
|
+
st = read_json(ws / "agent" / "agent_state.json", {})
|
|
996
|
+
write_json(ws / "agent" / "agent_state.json", dict(st, last_upkeep=t))
|
|
997
|
+
except Exception as e:
|
|
998
|
+
rep["upkeep"] = {"ran": t, "error": f"{type(e).__name__}: {e}"}
|
|
999
|
+
elif due:
|
|
1000
|
+
rep["upkeep"] = {"due": True, "note": "dry run: upkeep not run"}
|
|
1001
|
+
|
|
1002
|
+
if dry_run:
|
|
1003
|
+
rep["notes"].append("dry run: nothing queued, built, committed or pushed")
|
|
1004
|
+
rep["would_build"] = [i["id"] for i in q.list(status="detected")] + [c.get("key") for c in changes]
|
|
1005
|
+
rep["would_batch"] = [i["id"] for i in q.list(status="review") if not i.get("batch")]
|
|
1006
|
+
if cfg["auto_push"]:
|
|
1007
|
+
rep["push"] = push_approved(ws, rules, True, q, a365, env, now)
|
|
1008
|
+
return
|
|
1009
|
+
|
|
1010
|
+
with getattr(q, "lock", None) or contextlib.nullcontext(): # one lock hold: the folder is read once, not per part
|
|
1011
|
+
for c in changes:
|
|
1012
|
+
try:
|
|
1013
|
+
it = q.upsert({"key": c.get("key") or part_key(c.get("manufacturer"), c.get("mpn")), "mpn": c.get("mpn", ""),
|
|
1014
|
+
"manufacturer": c.get("manufacturer", ""), "origin": "watch", "change": c})
|
|
1015
|
+
if it.get("status") == "review" and c.get("kind") in ("new", "changed"):
|
|
1016
|
+
# the source changed while the part waits for review: rebuild it (its batch PR is then updated)
|
|
1017
|
+
q.update(it["id"], build=dict(it.get("build") or {}, requested_at=t, request_note=f"source {c.get('kind')}"))
|
|
1018
|
+
q.transition(it["id"], "building", user=AGENT, note=f"rebuild: {c.get('kind')} in a watched source")
|
|
1019
|
+
elif it.get("status") == "approved" and c.get("kind") == "changed":
|
|
1020
|
+
# the approved build is still what a person signed: it is pushed as it is, and this change becomes
|
|
1021
|
+
# a new item once it is (else it would be lost: the watch never reports it again)
|
|
1022
|
+
q.update(it["id"], followup=c)
|
|
1023
|
+
rep["notes"].append(f"{it.get('mpn')}: changed in a watched source after approval; rebuilt after "
|
|
1024
|
+
"its push (or reject and retry it to rebuild now)")
|
|
1025
|
+
except Exception as e: # one bad change must not lose the others
|
|
1026
|
+
rep["notes"].append(f"{c.get('key') or c.get('mpn')}: not queued: {type(e).__name__}: {e}")
|
|
1027
|
+
for f in findings:
|
|
1028
|
+
try:
|
|
1029
|
+
q.upsert(f)
|
|
1030
|
+
except Exception as e:
|
|
1031
|
+
rep["notes"].append(f"upkeep finding {f.get('key')}: not queued: {type(e).__name__}: {e}")
|
|
1032
|
+
pend.unlink(missing_ok=True)
|
|
1033
|
+
upend.unlink(missing_ok=True)
|
|
1034
|
+
|
|
1035
|
+
# PUSH (before the build: what was approved is sent before the librarian runs again, and a change that came in
|
|
1036
|
+
# after an approval becomes a new item that this same run builds)
|
|
1037
|
+
if cfg["auto_push"]:
|
|
1038
|
+
rep["push"] = push_approved(ws, rules, False, q, a365, env, now)
|
|
1039
|
+
|
|
1040
|
+
# BUILD
|
|
1041
|
+
_build(ws, rules, cfg, t, client, q, prover, builder, libs, log, rep)
|
|
1042
|
+
|
|
1043
|
+
# REVIEW: batches and their PRs
|
|
1044
|
+
out = _librarian_out(ws, ws / "agent" / BOM_NAME)
|
|
1045
|
+
review = q.list(status="review")
|
|
1046
|
+
fresh = [i for i in review if not i.get("batch")]
|
|
1047
|
+
size = max(1, int(cfg["batch_size"]))
|
|
1048
|
+
touched = {}
|
|
1049
|
+
for n in range(0, len(fresh), size):
|
|
1050
|
+
b = q.batch([i["id"] for i in fresh[n:n + size]])
|
|
1051
|
+
touched[b["id"]] = True
|
|
1052
|
+
for bid in {i["batch"] for i in review if i.get("batch")}: # a rebuilt item: update its batch's PR
|
|
1053
|
+
rec = read_json(ws / "agent" / "batches" / _safe(bid) / "batch.json", {})
|
|
1054
|
+
if any(rec.get("published", {}).get(i["id"]) != _published(i)
|
|
1055
|
+
for i in review if i.get("batch") == bid):
|
|
1056
|
+
touched[bid] = True
|
|
1057
|
+
lookup = lambda i: _get_or_none(q, i)
|
|
1058
|
+
open_batches = [b["id"] for b in q.batches() if b.get("status", "open") == "open"] if callable(getattr(q, "batches", None)) else \
|
|
1059
|
+
sorted({i["batch"] for i in q.list() if i.get("batch")})
|
|
1060
|
+
for bid in open_batches:
|
|
1061
|
+
members = q.list(batch=bid)
|
|
1062
|
+
rec = read_json(ws / "agent" / "batches" / _safe(bid) / "batch.json", {})
|
|
1063
|
+
if members and all(m["status"] in ("pushed", "rejected", "failed") for m in members) or (not members and rec):
|
|
1064
|
+
try: # all decided, nothing waits for a push: close or ready
|
|
1065
|
+
pushed = [m for m in members if m["status"] == "pushed"]
|
|
1066
|
+
if pushed and any(_departed(lookup(i), bid) for i in rec.get("paths", {})):
|
|
1067
|
+
publish_batch(ws, rules, bid, pushed, out, now, env, session, log, lookup=lookup) # take out first
|
|
1068
|
+
rec = close_batch(ws, rules, bid, members, now, env, session, log)
|
|
1069
|
+
if (rec.get("closed") or rec.get("ready")) and hasattr(q, "update_batch"):
|
|
1070
|
+
q.update_batch(bid, status="closed" if rec.get("closed") else "ready")
|
|
1071
|
+
except (KeyError, ValueError, OSError) as e:
|
|
1072
|
+
rep["notes"].append(f"batch {bid}: not closed: {type(e).__name__}: {e}")
|
|
1073
|
+
touched.pop(bid, None)
|
|
1074
|
+
elif any(_departed(lookup(i), bid) for i in rec.get("paths", {})):
|
|
1075
|
+
touched[bid] = True # a rejected / failed / moved part's files leave its branch
|
|
1076
|
+
for bid in touched:
|
|
1077
|
+
items = [i for i in q.list(batch=bid) if i["status"] in ("review", "approved")]
|
|
1078
|
+
if not items:
|
|
1079
|
+
continue
|
|
1080
|
+
rec = publish_batch(ws, rules, bid, items, out, now, env, session, log, lookup=lookup)
|
|
1081
|
+
try:
|
|
1082
|
+
if rec.get("pr_url") and hasattr(q, "update_batch"): # the queue's batch record (web UI batch view) gets the PR too
|
|
1083
|
+
q.update_batch(bid, pr_url=rec["pr_url"], branch=rec.get("branch"))
|
|
1084
|
+
for i in items:
|
|
1085
|
+
if rec.get("pr_url") and i.get("pr_url") != rec["pr_url"]:
|
|
1086
|
+
q.update(i["id"], pr_url=rec["pr_url"])
|
|
1087
|
+
except (KeyError, ValueError) as e:
|
|
1088
|
+
rep["notes"].append(f"batch {bid}: PR link not stored: {type(e).__name__}: {e}")
|
|
1089
|
+
rep["batches"].append(rec)
|
|
1090
|
+
|
|
1091
|
+
if not cfg["auto_push"]:
|
|
1092
|
+
n = len(q.list(status="approved"))
|
|
1093
|
+
if n:
|
|
1094
|
+
rep["notes"].append(f"{n} approved part(s) wait for `ecadlib agent push` ([agent] auto_push is off)")
|
|
1095
|
+
|
|
1096
|
+
|
|
1097
|
+
MAX_ATTEMPTS = 3 # runs that died while building / proving one item before that item is failed
|
|
1098
|
+
|
|
1099
|
+
|
|
1100
|
+
def _get_or_none(q, i):
|
|
1101
|
+
try:
|
|
1102
|
+
return q.get(i)
|
|
1103
|
+
except (KeyError, ValueError):
|
|
1104
|
+
return None
|
|
1105
|
+
|
|
1106
|
+
|
|
1107
|
+
def _wants_build(it):
|
|
1108
|
+
"""Items to build now: detected and building ones. Runs never overlap (run_lock), so an item found `building` at
|
|
1109
|
+
the start of a run is one a person asked to rebuild, one waiting for a datasheet batch, or one a run died on: all
|
|
1110
|
+
are (re)built. No time comparison: a clock stepped back must not strand a rebuild request."""
|
|
1111
|
+
return it["status"] in ("detected", "building")
|
|
1112
|
+
|
|
1113
|
+
|
|
1114
|
+
def _rebuildable(it):
|
|
1115
|
+
ch = it.get("change") or {}
|
|
1116
|
+
if it.get("origin") == "upkeep":
|
|
1117
|
+
return ch.get("check") == "upstream" and str(ch.get("kind", "")).lower() == "kicad changed"
|
|
1118
|
+
return ch.get("kind") != "removed"
|
|
1119
|
+
|
|
1120
|
+
|
|
1121
|
+
def _skipped_proof(reason):
|
|
1122
|
+
return {"verdict": "skipped", "score": None, "reason": reason, "findings": [], "images": {"ours": [], "datasheet": []},
|
|
1123
|
+
"model": "", "cost_usd": 0.0}
|
|
1124
|
+
|
|
1125
|
+
|
|
1126
|
+
def _prove(q, item_id, ws, rules, client, prover, rep, note=None):
|
|
1127
|
+
"""proving -> review / failed for one item (the PROVE step; also resumes an item a dead run left in proving)."""
|
|
1128
|
+
it = q.get(item_id)
|
|
1129
|
+
b = dict(it.get("build") or {})
|
|
1130
|
+
b["proof_attempts"] = int(b.get("proof_attempts") or 0) + 1
|
|
1131
|
+
if b["proof_attempts"] > MAX_ATTEMPTS:
|
|
1132
|
+
q.update(item_id, build=b)
|
|
1133
|
+
q.transition(item_id, "failed", user=AGENT, note=f"the proof stopped the agent {MAX_ATTEMPTS} times: look at it")
|
|
1134
|
+
rep["failed"].append({"id": item_id, "reason": "proof interrupted repeatedly"})
|
|
1135
|
+
return
|
|
1136
|
+
it = q.update(item_id, build=b)
|
|
1137
|
+
if b.get("action") == "none":
|
|
1138
|
+
proof = _skipped_proof("nothing was built")
|
|
1139
|
+
else:
|
|
1140
|
+
try:
|
|
1141
|
+
proof = prover.prove(it, workspace=ws, rules=rules, client=client)
|
|
1142
|
+
except Exception as e:
|
|
1143
|
+
proof = _skipped_proof(f"proof crashed: {type(e).__name__}: {e}")
|
|
1144
|
+
q.update(item_id, proof=proof)
|
|
1145
|
+
if proof.get("verdict") == "fail":
|
|
1146
|
+
q.transition(item_id, "failed", user=AGENT, note=f"proof failed: {proof.get('reason', '')}")
|
|
1147
|
+
rep["failed"].append({"id": item_id, "reason": f"proof failed: {proof.get('reason', '')}"})
|
|
1148
|
+
else:
|
|
1149
|
+
q.transition(item_id, "review", user=AGENT, note=note or f"proof {proof.get('verdict')}: {proof.get('reason', '')}"[:300])
|
|
1150
|
+
rep["review"].append(item_id)
|
|
1151
|
+
|
|
1152
|
+
|
|
1153
|
+
def _changed_meanwhile(rep, it, e):
|
|
1154
|
+
"""A person decided on the item while the agent worked on it (reject, ...): their decision stands."""
|
|
1155
|
+
rep["notes"].append(f"{it.get('mpn') or it['id']}: left as it is, changed meanwhile ({type(e).__name__}: {e})")
|
|
1156
|
+
|
|
1157
|
+
|
|
1158
|
+
def _read_lines(ws):
|
|
1159
|
+
f = ws / "agent" / "library_bom.json"
|
|
1160
|
+
if not f.exists():
|
|
1161
|
+
return {}
|
|
1162
|
+
try:
|
|
1163
|
+
lines = json.loads(f.read_text())
|
|
1164
|
+
except ValueError:
|
|
1165
|
+
lines = None
|
|
1166
|
+
if not isinstance(lines, dict): # starting from {} would drop every part built so far from the plan
|
|
1167
|
+
raise RuntimeError(f"{f} is damaged; restore it (its lines are also in {BOM_NAME}) before the agent builds again")
|
|
1168
|
+
out = {}
|
|
1169
|
+
for k, ln in lines.items(): # keys from before manufacturer names were normalised (or another table)
|
|
1170
|
+
m, _, n = k.partition("|")
|
|
1171
|
+
out.setdefault(part_key((ln or {}).get("manufacturer") or m, n), ln)
|
|
1172
|
+
for k in [k for k in out if k.startswith("|")]: # a blank maker line once that MPN has exactly one known maker
|
|
1173
|
+
if len([x for x in out if x.endswith(k) and x != k]) == 1:
|
|
1174
|
+
del out[k]
|
|
1175
|
+
return out
|
|
1176
|
+
|
|
1177
|
+
|
|
1178
|
+
def _build(ws, rules, cfg, t, client, q, prover, builder, libs, log, rep):
|
|
1179
|
+
for it in q.list(status="proving"): # a run died between the build and its proof: prove it now
|
|
1180
|
+
try:
|
|
1181
|
+
_prove(q, it["id"], ws, rules, client, prover, rep)
|
|
1182
|
+
except (ValueError, KeyError) as e:
|
|
1183
|
+
_changed_meanwhile(rep, it, e)
|
|
1184
|
+
todo = [i for i in q.list() if _wants_build(i)]
|
|
1185
|
+
if not todo:
|
|
1186
|
+
return
|
|
1187
|
+
bom = ws / "agent" / BOM_NAME
|
|
1188
|
+
out = _librarian_out(ws, bom)
|
|
1189
|
+
plan_before = read_json(out / "plan.json", [])
|
|
1190
|
+
repo = repo_root(Path(cfg["library_repo"]) if cfg["library_repo"] else ws)
|
|
1191
|
+
lines = _read_lines(ws)
|
|
1192
|
+
build_these, refresh = [], []
|
|
1193
|
+
for it in todo:
|
|
1194
|
+
try:
|
|
1195
|
+
if it["status"] == "detected":
|
|
1196
|
+
# a new build cycle (retry / re-opened): it gets a new batch and PR, and a follow-up change is in it
|
|
1197
|
+
fresh = {k: None for k in ("batch", "pr_url", "followup") if it.get(k)}
|
|
1198
|
+
if fresh:
|
|
1199
|
+
q.update(it["id"], **fresh)
|
|
1200
|
+
it = q.transition(it["id"], "building", user=AGENT, note="agent run")
|
|
1201
|
+
bb = dict(it.get("build") or {})
|
|
1202
|
+
bb["attempts"] = int(bb.get("attempts") or 0) + 1
|
|
1203
|
+
if bb["attempts"] > MAX_ATTEMPTS:
|
|
1204
|
+
_fail(q, it, rep, f"the build stopped the agent {MAX_ATTEMPTS} times: look at its BOM line", t, bb)
|
|
1205
|
+
continue
|
|
1206
|
+
it = q.update(it["id"], build=bb)
|
|
1207
|
+
notes = []
|
|
1208
|
+
old = _find_plan_item(plan_before, it)
|
|
1209
|
+
before = []
|
|
1210
|
+
if old: # BEFORE pictures: the current library version
|
|
1211
|
+
files = _model_files(old, out)
|
|
1212
|
+
with tempfile.TemporaryDirectory() as td:
|
|
1213
|
+
files = {**_git_before(repo, out, old, td), **files} if len(files) < 2 else files
|
|
1214
|
+
before = pictures_for(ws, prover, it["id"], files, "before", notes) if files else []
|
|
1215
|
+
if not _rebuildable(it):
|
|
1216
|
+
ch = it.get("change") or {}
|
|
1217
|
+
why = ("removed from " + str((ch.get("source") or {}).get("name") or "a watched source") if ch.get("kind") == "removed"
|
|
1218
|
+
else f"{ch.get('check')} finding: {ch.get('kind')} ({ch.get('was')} -> {ch.get('now')})")
|
|
1219
|
+
q.update(it["id"], build={"action": "none", "reason": why + "; a person decides", "built": t, "user": AGENT,
|
|
1220
|
+
"part": old.get("name") if old else None, "hash": None,
|
|
1221
|
+
"bom": bom.relative_to(ws).as_posix()},
|
|
1222
|
+
pictures={"before": before, "after": []})
|
|
1223
|
+
q.transition(it["id"], "proving", user=AGENT, note="nothing to build")
|
|
1224
|
+
_prove(q, it["id"], ws, rules, client, prover, rep, note=why)
|
|
1225
|
+
continue
|
|
1226
|
+
except (ValueError, KeyError) as e:
|
|
1227
|
+
_changed_meanwhile(rep, it, e)
|
|
1228
|
+
continue
|
|
1229
|
+
ln = _line_of(it, plan_before)
|
|
1230
|
+
lines[it["key"]] = {k: ln.get(k, "") for k, _ in BOM_COLS}
|
|
1231
|
+
if old and ((it.get("build") or {}).get("requested_at") or it.get("origin") == "upkeep"):
|
|
1232
|
+
refresh.append(old["name"])
|
|
1233
|
+
build_these.append((it, before, notes))
|
|
1234
|
+
if not build_these:
|
|
1235
|
+
return
|
|
1236
|
+
write_json(ws / "agent" / "library_bom.json", lines)
|
|
1237
|
+
write_bom(ws, lines)
|
|
1238
|
+
from .agent_queue import plan_lock
|
|
1239
|
+
try:
|
|
1240
|
+
with plan_lock(ws): # a person's approve/reject waits: the librarian rewrites plan.json from its copy
|
|
1241
|
+
plan = (builder or (lambda *a, **k: librarian_builder(*a, libs=libs, **k)))(ws, bom, out, refresh, rules, log)
|
|
1242
|
+
except Exception as e:
|
|
1243
|
+
for it, _, _ in build_these:
|
|
1244
|
+
try:
|
|
1245
|
+
q.update(it["id"], build={"error": f"{type(e).__name__}: {e}", "built": t, "user": AGENT})
|
|
1246
|
+
q.transition(it["id"], "failed", user=AGENT, note=f"librarian failed: {type(e).__name__}: {e}")
|
|
1247
|
+
rep["failed"].append({"id": it["id"], "reason": f"librarian failed: {e}"})
|
|
1248
|
+
except (ValueError, KeyError) as e2:
|
|
1249
|
+
_changed_meanwhile(rep, it, e2)
|
|
1250
|
+
return
|
|
1251
|
+
from . import librarian as L
|
|
1252
|
+
rows = {r["part"]: r for r in L.review_rows(plan)} if plan else {}
|
|
1253
|
+
for it, before, notes in build_these:
|
|
1254
|
+
try:
|
|
1255
|
+
pi = _find_plan_item(plan, it)
|
|
1256
|
+
if pi is None:
|
|
1257
|
+
_fail(q, it, rep, "the librarian made no plan entry for this part (check its BOM line)", t)
|
|
1258
|
+
continue
|
|
1259
|
+
files = _model_files(pi, out)
|
|
1260
|
+
tried = (pi.get("librarian") or {}).get("tried", [])
|
|
1261
|
+
b = {"part": pi["name"], "bom": bom.relative_to(ws).as_posix(), "user": AGENT, "built": t, "status": pi.get("status"),
|
|
1262
|
+
"hold_reasons": pi.get("hold_reasons", []), "verify": pi.get("verify", []), "source": pi.get("source"),
|
|
1263
|
+
"cross_check": pi.get("cross_check"), "tried": tried, "notes": notes, "files": {"symbol": (pi.get("symbol") or {}).get("file"),
|
|
1264
|
+
"footprint": (pi.get("footprint") or {}).get("file"),
|
|
1265
|
+
"kicad_mod": (pi.get("footprint") or {}).get("kicad_mod")},
|
|
1266
|
+
"plan": (out / "plan.json").relative_to(ws).as_posix()}
|
|
1267
|
+
prev = q.get(it["id"]).get("build") or {} # who asked for this rebuild: agent_queue refuses their approval
|
|
1268
|
+
b.update({k: prev[k] for k in ("requested_by", "requested_at", "request_note") if prev.get(k)})
|
|
1269
|
+
if not (pi["symbol"].get("file") and pi["footprint"].get("file")):
|
|
1270
|
+
if any("run again to collect" in x for x in tried):
|
|
1271
|
+
b["pending"] = "datasheet batch submitted; the next run collects it"
|
|
1272
|
+
q.update(it["id"], build=b)
|
|
1273
|
+
rep["waiting"].append(it["id"])
|
|
1274
|
+
continue
|
|
1275
|
+
_fail(q, it, rep, "build incomplete: " + (rows.get(pi["name"], {}).get("why") or "no symbol or footprint"), t, b)
|
|
1276
|
+
continue
|
|
1277
|
+
if q.get(it["id"])["status"] != "building":
|
|
1278
|
+
raise ValueError(f"it is {q.get(it['id'])['status']} now")
|
|
1279
|
+
b["hash"] = _hash_files({k: v for k, v in files.items()})
|
|
1280
|
+
after = pictures_for(ws, prover, it["id"], files, "after", notes)
|
|
1281
|
+
q.update(it["id"], build=b, pictures={"before": before, "after": after})
|
|
1282
|
+
q.transition(it["id"], "proving", user=AGENT, note=f"built {pi['name']} ({pi.get('status')})")
|
|
1283
|
+
rep["built"].append(it["id"])
|
|
1284
|
+
_prove(q, it["id"], ws, rules, client, prover, rep)
|
|
1285
|
+
except (ValueError, KeyError, OSError) as e:
|
|
1286
|
+
_changed_meanwhile(rep, it, e)
|
|
1287
|
+
|
|
1288
|
+
|
|
1289
|
+
def _fail(q, it, rep, reason, t, build=None):
|
|
1290
|
+
q.update(it["id"], build=dict(build or {}, error=reason, built=t, user=AGENT))
|
|
1291
|
+
q.transition(it["id"], "failed", user=AGENT, note=reason[:500])
|
|
1292
|
+
rep["failed"].append({"id": it["id"], "reason": reason})
|
|
1293
|
+
|
|
1294
|
+
|
|
1295
|
+
# ------------------------------------------------------------------ serve
|
|
1296
|
+
def serve(workspace, interval_s=None, rules=None, stop=None, max_runs=None, run=None, install_signals=True, log=print,
|
|
1297
|
+
sleep_s=None, **run_kw):
|
|
1298
|
+
"""Run run_once every interval (default [agent] interval_minutes) until `stop` (a threading.Event) is set, SIGTERM
|
|
1299
|
+
or SIGINT arrives, or max_runs passes ran. While [agent] enabled is false it only keeps the status file fresh.
|
|
1300
|
+
Rules are re-read every pass (unless given), so switching the agent on needs no restart."""
|
|
1301
|
+
ws = Path(workspace).resolve()
|
|
1302
|
+
stop = stop or threading.Event()
|
|
1303
|
+
run = run or run_once
|
|
1304
|
+
if install_signals and threading.current_thread() is threading.main_thread():
|
|
1305
|
+
for s in (signal.SIGTERM, signal.SIGINT):
|
|
1306
|
+
signal.signal(s, lambda *_: stop.set())
|
|
1307
|
+
runs = 0
|
|
1308
|
+
write_status(ws, state="starting", serving=True)
|
|
1309
|
+
wait = interval_s if interval_s is not None else 60
|
|
1310
|
+
while not stop.is_set():
|
|
1311
|
+
try:
|
|
1312
|
+
r = RU.resolve(rules, ws) if rules is not None else RU.load(None, start=ws)
|
|
1313
|
+
cfg = settings(r)
|
|
1314
|
+
except Exception as e: # a typo saved in ecadlib.toml: say so and try again, don't stop
|
|
1315
|
+
log(f"agent: rules not loaded, no run: {type(e).__name__}: {e}")
|
|
1316
|
+
with contextlib.suppress(OSError):
|
|
1317
|
+
write_status(ws, state="error", serving=True, last_error=f"rules: {type(e).__name__}: {e}")
|
|
1318
|
+
stop.wait(sleep_s if sleep_s is not None else wait)
|
|
1319
|
+
continue
|
|
1320
|
+
wait = interval_s if interval_s is not None else cfg["interval_minutes"] * 60
|
|
1321
|
+
if cfg["enabled"]:
|
|
1322
|
+
try:
|
|
1323
|
+
rep = run(ws, r, **run_kw)
|
|
1324
|
+
log(f"agent: {json.dumps(_summary(rep), default=str)}")
|
|
1325
|
+
except Exception as e: # one bad pass never ends the service
|
|
1326
|
+
log(f"agent: run failed: {type(e).__name__}: {e}")
|
|
1327
|
+
runs += 1
|
|
1328
|
+
else:
|
|
1329
|
+
log("agent: [agent] enabled = false in ecadlib.toml; idle")
|
|
1330
|
+
write_status(ws, state="idle" if cfg["enabled"] else "disabled", serving=True, enabled=cfg["enabled"],
|
|
1331
|
+
interval_s=wait, next_run=stamp(time.time() + wait))
|
|
1332
|
+
if max_runs is not None and runs >= max_runs:
|
|
1333
|
+
break
|
|
1334
|
+
stop.wait(sleep_s if sleep_s is not None else wait)
|
|
1335
|
+
write_status(ws, state="stopped", serving=False)
|
|
1336
|
+
return runs
|
|
1337
|
+
|
|
1338
|
+
|
|
1339
|
+
# ------------------------------------------------------------------ CLI
|
|
1340
|
+
def main(argv=None):
|
|
1341
|
+
ap = argparse.ArgumentParser(prog="ecadlib agent", description="library agent: watch, build, prove, review PRs, push, upkeep")
|
|
1342
|
+
ap.add_argument("--workspace", default=os.environ.get("ECADLIB_WORKSPACE", "ecadlib_workspace"))
|
|
1343
|
+
ap.add_argument("--rules", help="rules file (default: <workspace>/ecadlib.toml)")
|
|
1344
|
+
sub = ap.add_subparsers(dest="cmd", required=True)
|
|
1345
|
+
r = sub.add_parser("run", help="one pass now (works even while [agent] enabled is false)")
|
|
1346
|
+
r.add_argument("--dry-run", action="store_true", help="show what would happen; change nothing")
|
|
1347
|
+
r.add_argument("--upkeep", action="store_true", help="run the upkeep checks now, whatever the schedule")
|
|
1348
|
+
r.add_argument("--lib", action="append", default=[], help="local KiCad library folder (repeatable)")
|
|
1349
|
+
s = sub.add_parser("serve", help="run every [agent] interval_minutes until stopped")
|
|
1350
|
+
s.add_argument("--interval", type=float, help="seconds between runs (default: the rules)")
|
|
1351
|
+
s.add_argument("--lib", action="append", default=[])
|
|
1352
|
+
st = sub.add_parser("status", help="last run, queue counts, batches")
|
|
1353
|
+
st.add_argument("--healthy", type=float, metavar="SECONDS", help="exit 1 unless the status was written within SECONDS")
|
|
1354
|
+
st.add_argument("--json", action="store_true", help="print status.json + queue counts + batches as JSON")
|
|
1355
|
+
for name in ("approve", "reject"):
|
|
1356
|
+
p = sub.add_parser(name, help=f"{name} a review-queue item")
|
|
1357
|
+
p.add_argument("id")
|
|
1358
|
+
p.add_argument("--user", help="who signs (default: $ECADLIB_USER, then git config)")
|
|
1359
|
+
p.add_argument("--role")
|
|
1360
|
+
p.add_argument("-m", "--note", default="")
|
|
1361
|
+
pu = sub.add_parser("push", help="push approved parts to Altium 365 now")
|
|
1362
|
+
pu.add_argument("--dry-run", action="store_true")
|
|
1363
|
+
a = ap.parse_args(argv)
|
|
1364
|
+
ws = Path(a.workspace).resolve()
|
|
1365
|
+
if not (a.cmd == "run" and a.dry_run): # a dry run writes nothing, not even the workspace folder
|
|
1366
|
+
ws.mkdir(parents=True, exist_ok=True)
|
|
1367
|
+
rules = RU.load(a.rules, start=ws)
|
|
1368
|
+
if a.cmd == "run":
|
|
1369
|
+
rep = run_once(ws, rules, dry_run=a.dry_run, do_upkeep=True if a.upkeep else None, libs=a.lib, log=print)
|
|
1370
|
+
print(json.dumps(rep, indent=2, default=str))
|
|
1371
|
+
return 1 if rep.get("error") or rep.get("skipped") else 0
|
|
1372
|
+
if a.cmd == "serve":
|
|
1373
|
+
serve(ws, a.interval, rules if a.rules else None, libs=a.lib)
|
|
1374
|
+
return 0
|
|
1375
|
+
if a.cmd == "status":
|
|
1376
|
+
stt = status(ws)
|
|
1377
|
+
if a.json:
|
|
1378
|
+
print(json.dumps(stt, indent=2, default=str))
|
|
1379
|
+
else:
|
|
1380
|
+
from .term_views import agent_status
|
|
1381
|
+
agent_status(stt, a.workspace)
|
|
1382
|
+
if a.healthy is not None:
|
|
1383
|
+
hb = _epoch(stt.get("heartbeat"))
|
|
1384
|
+
return 0 if hb and time.time() - hb <= a.healthy else 1
|
|
1385
|
+
return 0
|
|
1386
|
+
if a.cmd in ("approve", "reject"):
|
|
1387
|
+
from . import agent_queue as AQ
|
|
1388
|
+
from .approvals import identity
|
|
1389
|
+
if a.cmd == "reject" and not a.note.strip():
|
|
1390
|
+
print("a reject needs -m saying what is wrong", file=sys.stderr)
|
|
1391
|
+
return 2
|
|
1392
|
+
from .app import Workspace
|
|
1393
|
+
try: # decide() records through Workspace.approve_part, so it needs a Workspace, not a path
|
|
1394
|
+
r = AQ.decide(Workspace(ws), a.id, a.cmd, a.user or identity(cwd=ws), role=a.role, note=a.note, via="ecadlib agent")
|
|
1395
|
+
except (ValueError, PermissionError, KeyError) as e:
|
|
1396
|
+
print(f"error: {e.args[0] if e.args else e}", file=sys.stderr)
|
|
1397
|
+
return 2
|
|
1398
|
+
ap_ = r.get("approval")
|
|
1399
|
+
print(f"{a.id}: {r['item']['status']}" + (f"; sign-off: {'recorded' if ap_['recorded'] else ap_['reason']}" if ap_ else ""))
|
|
1400
|
+
return 0
|
|
1401
|
+
rep = push_approved(ws, rules, a.dry_run)
|
|
1402
|
+
print(json.dumps(rep, indent=2, default=str))
|
|
1403
|
+
return 1 if rep.get("skipped") or any(x.get("action") == "error" for x in rep["results"]) else 0
|
|
1404
|
+
|
|
1405
|
+
|
|
1406
|
+
if __name__ == "__main__":
|
|
1407
|
+
sys.exit(main())
|