edify-agents-cli 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- edify/__init__.py +19 -0
- edify/__main__.py +6 -0
- edify/_toml.py +286 -0
- edify/agents.py +261 -0
- edify/anim.py +499 -0
- edify/assets/commands/build.md +64 -0
- edify/assets/commands/plan.md +68 -0
- edify/assets/commands/spec.md +63 -0
- edify/assets/commands/tasks.md +65 -0
- edify/assets/commands/verify.md +67 -0
- edify/assets/formats/plan.example.md +455 -0
- edify/assets/formats/plan.format.md +128 -0
- edify/assets/formats/spec.example.md +483 -0
- edify/assets/formats/spec.format.md +116 -0
- edify/assets/formats/tasks.example.md +712 -0
- edify/assets/formats/tasks.format.md +199 -0
- edify/assets/harvest/harvest.md +91 -0
- edify/assets/harvest/sources.md +295 -0
- edify/assets/harvest/vocab.md +52 -0
- edify/assets/manifest.md +30 -0
- edify/assets/mcp/default-registry.md +66 -0
- edify/assets/skills/adversary.md +61 -0
- edify/assets/skills/debugger.md +64 -0
- edify/assets/skills/implementer-general.md +53 -0
- edify/assets/skills/implementer-go.md +63 -0
- edify/assets/skills/implementer-node.md +61 -0
- edify/assets/skills/implementer-postgres.md +63 -0
- edify/assets/skills/implementer-python.md +65 -0
- edify/assets/skills/implementer-react.md +65 -0
- edify/assets/skills/planner.md +58 -0
- edify/assets/skills/test-writer-node.md +64 -0
- edify/assets/skills/test-writer-python.md +60 -0
- edify/assets/skills/verifier.md +67 -0
- edify/assets/templates/CLAUDE.md +18 -0
- edify/assets.py +77 -0
- edify/cli.py +681 -0
- edify/commands/__init__.py +1 -0
- edify/commands/check_cmd.py +72 -0
- edify/commands/doctor_cmd.py +206 -0
- edify/commands/feedback_cmd.py +90 -0
- edify/commands/governance_cmd.py +65 -0
- edify/commands/graph_cmd.py +156 -0
- edify/commands/harvest_cmd.py +538 -0
- edify/commands/init_cmd.py +333 -0
- edify/commands/init_new_cmd.py +151 -0
- edify/commands/license_cmd.py +80 -0
- edify/commands/mcp_cmd.py +159 -0
- edify/commands/self_cmd.py +185 -0
- edify/commands/setup_cmd.py +79 -0
- edify/commands/skills_cmd.py +177 -0
- edify/commands/upgrade_cmd.py +178 -0
- edify/context.py +57 -0
- edify/conventions.py +275 -0
- edify/detect.py +313 -0
- edify/distribution.py +40 -0
- edify/errors.py +66 -0
- edify/feedback.py +207 -0
- edify/formats/__init__.py +11 -0
- edify/formats/check.py +573 -0
- edify/formats/document.py +238 -0
- edify/formats/finding.py +50 -0
- edify/formats/fix.py +114 -0
- edify/formats/mcp.py +126 -0
- edify/formats/plan.py +151 -0
- edify/formats/skill.py +139 -0
- edify/formats/spec.py +164 -0
- edify/formats/tasks.py +321 -0
- edify/governance.py +316 -0
- edify/graph/__init__.py +11 -0
- edify/graph/extract.py +351 -0
- edify/graph/ignore.py +107 -0
- edify/graph/langs/__init__.py +93 -0
- edify/graph/langs/ctags.py +145 -0
- edify/graph/langs/markdown.py +41 -0
- edify/graph/langs/patterns.py +361 -0
- edify/graph/langs/python_ast.py +154 -0
- edify/graph/langs/sqlish.py +86 -0
- edify/graph/model.py +102 -0
- edify/graph/queries.py +211 -0
- edify/graph/store.py +157 -0
- edify/harvest/__init__.py +28 -0
- edify/harvest/ledger.py +191 -0
- edify/harvest/screen.py +236 -0
- edify/harvest/sources.py +298 -0
- edify/host.py +345 -0
- edify/interview.py +354 -0
- edify/library.py +176 -0
- edify/licensing/__init__.py +30 -0
- edify/licensing/ed25519.py +154 -0
- edify/licensing/projects.py +223 -0
- edify/licensing/tier.py +172 -0
- edify/licensing/token.py +149 -0
- edify/paths.py +282 -0
- edify/selfinstall.py +244 -0
- edify/skills.py +108 -0
- edify/tsv.py +109 -0
- edify/ui.py +334 -0
- edify_agents_cli-0.2.0.dist-info/METADATA +408 -0
- edify_agents_cli-0.2.0.dist-info/RECORD +102 -0
- edify_agents_cli-0.2.0.dist-info/WHEEL +4 -0
- edify_agents_cli-0.2.0.dist-info/entry_points.txt +2 -0
- edify_agents_cli-0.2.0.dist-info/licenses/LICENSE +21 -0
edify/graph/model.py
ADDED
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
"""What a node and an edge are.
|
|
2
|
+
|
|
3
|
+
Granularity is the public surface: files, modules, exported symbols, routes,
|
|
4
|
+
tables, packages, doc sections. Local variables are not indexed — they churn on
|
|
5
|
+
every commit and answer questions a model resolves better by reading the function.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import hashlib
|
|
11
|
+
from dataclasses import dataclass, field
|
|
12
|
+
|
|
13
|
+
NODE_KINDS = ("file", "module", "symbol", "route", "table", "package", "doc-section")
|
|
14
|
+
EDGE_KINDS = ("defines", "references", "imports", "exposes", "documents", "migrates")
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def node_id(kind: str, file: str, name: str) -> str:
|
|
18
|
+
"""A stable id.
|
|
19
|
+
|
|
20
|
+
Space-separated in the manner of the code-indexing standard's symbol strings,
|
|
21
|
+
so an external indexer can be pointed at this format without translation.
|
|
22
|
+
The id is derived only from things that survive a reformat: the kind, the
|
|
23
|
+
file, and the name. Not the line — a symbol that moves down ten lines is the
|
|
24
|
+
same symbol.
|
|
25
|
+
"""
|
|
26
|
+
if kind in ("table", "package"):
|
|
27
|
+
return f"{kind} . {name}"
|
|
28
|
+
if kind in ("file", "module"):
|
|
29
|
+
return f"{kind} {file} ."
|
|
30
|
+
return f"{kind} {file} {name}"
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def fingerprint(signature: str) -> str:
|
|
34
|
+
"""Twelve hex characters of the normalised signature.
|
|
35
|
+
|
|
36
|
+
Enough to tell "this symbol changed shape" from "this symbol moved", which is
|
|
37
|
+
the only question the column is asked.
|
|
38
|
+
"""
|
|
39
|
+
normalised = " ".join(signature.split())
|
|
40
|
+
return hashlib.sha256(normalised.encode("utf-8")).hexdigest()[:12]
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
@dataclass(frozen=True)
|
|
44
|
+
class Node:
|
|
45
|
+
id: str
|
|
46
|
+
kind: str
|
|
47
|
+
file: str
|
|
48
|
+
line: int
|
|
49
|
+
name: str
|
|
50
|
+
fingerprint: str = "-"
|
|
51
|
+
|
|
52
|
+
def row(self) -> list[str]:
|
|
53
|
+
return [self.id, self.kind, self.file, str(self.line), self.name, self.fingerprint]
|
|
54
|
+
|
|
55
|
+
@staticmethod
|
|
56
|
+
def from_row(row: list[str]) -> "Node":
|
|
57
|
+
while len(row) < 6:
|
|
58
|
+
row = row + ["-"]
|
|
59
|
+
try:
|
|
60
|
+
line = int(row[3])
|
|
61
|
+
except ValueError:
|
|
62
|
+
line = 0
|
|
63
|
+
return Node(row[0], row[1], row[2], line, row[4], row[5])
|
|
64
|
+
|
|
65
|
+
@property
|
|
66
|
+
def ref(self) -> str:
|
|
67
|
+
"""The `file:line` form every reference in the system is written in (P3)."""
|
|
68
|
+
return f"{self.file}:{self.line}" if self.line else self.file
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
@dataclass(frozen=True)
|
|
72
|
+
class Edge:
|
|
73
|
+
src: str
|
|
74
|
+
relation: str
|
|
75
|
+
dst: str
|
|
76
|
+
|
|
77
|
+
def row(self) -> list[str]:
|
|
78
|
+
return [self.src, self.relation, self.dst]
|
|
79
|
+
|
|
80
|
+
@staticmethod
|
|
81
|
+
def from_row(row: list[str]) -> "Edge":
|
|
82
|
+
return Edge(row[0], row[1], row[2] if len(row) > 2 else "")
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
@dataclass
|
|
86
|
+
class Extraction:
|
|
87
|
+
"""What one extractor pass produced, before reference resolution."""
|
|
88
|
+
|
|
89
|
+
nodes: list[Node] = field(default_factory=list)
|
|
90
|
+
edges: list[Edge] = field(default_factory=list)
|
|
91
|
+
# (source node id, referenced name, relation) — resolved against the global
|
|
92
|
+
# index after the walk, and dropped when it resolves to nothing in the repo.
|
|
93
|
+
# An unresolved reference is an external one, and external things are not in
|
|
94
|
+
# this graph.
|
|
95
|
+
pending_refs: list[tuple[str, str, str]] = field(default_factory=list)
|
|
96
|
+
languages: set[str] = field(default_factory=set)
|
|
97
|
+
|
|
98
|
+
def extend(self, other: "Extraction") -> None:
|
|
99
|
+
self.nodes.extend(other.nodes)
|
|
100
|
+
self.edges.extend(other.edges)
|
|
101
|
+
self.pending_refs.extend(other.pending_refs)
|
|
102
|
+
self.languages |= other.languages
|
edify/graph/queries.py
ADDED
|
@@ -0,0 +1,211 @@
|
|
|
1
|
+
"""The queries. Everything the system asks the graph reduces to these.
|
|
2
|
+
|
|
3
|
+
`where` · `dependents` · `defines` · `overlap` · `references`. Each is a scan or a
|
|
4
|
+
lookup over sorted text: no model, no network, no state, and the same answer on
|
|
5
|
+
Tuesday.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import fnmatch
|
|
11
|
+
from dataclasses import dataclass
|
|
12
|
+
|
|
13
|
+
from .model import Node
|
|
14
|
+
from .store import GraphStore
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@dataclass(frozen=True)
|
|
18
|
+
class Hit:
|
|
19
|
+
node: Node
|
|
20
|
+
signature: str
|
|
21
|
+
|
|
22
|
+
def line(self) -> str:
|
|
23
|
+
return f"{self.node.ref}\t{self.node.kind}\t{self.node.name}"
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def where(store: GraphStore, name: str, kind: str | None = None, limit: int = 50) -> list[Node]:
|
|
27
|
+
"""Where is this defined — file, line, kind.
|
|
28
|
+
|
|
29
|
+
Exact match first. Only when nothing matches exactly does it fall back to a
|
|
30
|
+
case-insensitive and then a glob match, so an exact name never returns a
|
|
31
|
+
fuzzy answer.
|
|
32
|
+
"""
|
|
33
|
+
exact = [n for n in store.by_name.get(name, []) if kind in (None, n.kind)]
|
|
34
|
+
if exact:
|
|
35
|
+
return _ordered(exact)[:limit]
|
|
36
|
+
|
|
37
|
+
lowered = name.lower()
|
|
38
|
+
ci = [
|
|
39
|
+
n
|
|
40
|
+
for n in store.nodes
|
|
41
|
+
if n.name.lower() == lowered and kind in (None, n.kind)
|
|
42
|
+
]
|
|
43
|
+
if ci:
|
|
44
|
+
return _ordered(ci)[:limit]
|
|
45
|
+
|
|
46
|
+
if any(ch in name for ch in "*?["):
|
|
47
|
+
globbed = [
|
|
48
|
+
n
|
|
49
|
+
for n in store.nodes
|
|
50
|
+
if kind in (None, n.kind) and fnmatch.fnmatch(n.name, name)
|
|
51
|
+
]
|
|
52
|
+
return _ordered(globbed)[:limit]
|
|
53
|
+
|
|
54
|
+
substring = [
|
|
55
|
+
n
|
|
56
|
+
for n in store.nodes
|
|
57
|
+
if kind in (None, n.kind)
|
|
58
|
+
and n.kind not in ("file", "module")
|
|
59
|
+
and lowered in n.name.lower()
|
|
60
|
+
]
|
|
61
|
+
return _ordered(substring)[:limit]
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def defines(store: GraphStore, file: str) -> list[Node]:
|
|
65
|
+
"""What this file exports. Asked before writing a helper that may exist."""
|
|
66
|
+
target = file.replace("\\", "/").lstrip("./")
|
|
67
|
+
nodes = store.by_file.get(target)
|
|
68
|
+
if nodes is None:
|
|
69
|
+
matches = [f for f in store.by_file if f.endswith("/" + target) or f == target]
|
|
70
|
+
if len(matches) != 1:
|
|
71
|
+
matches = [f for f in store.by_file if fnmatch.fnmatch(f, target)]
|
|
72
|
+
nodes = [n for f in matches for n in store.by_file[f]]
|
|
73
|
+
return _ordered([n for n in nodes if n.kind not in ("file",)])
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def dependents(store: GraphStore, name: str, depth: int = 1) -> list[Node]:
|
|
77
|
+
"""Everything that breaks if this changes.
|
|
78
|
+
|
|
79
|
+
Depth 1 is the direct callers and importers. Deeper walks the transitive
|
|
80
|
+
closure, which is what makes a task's file set complete rather than
|
|
81
|
+
optimistic — and what `overlap` uses.
|
|
82
|
+
"""
|
|
83
|
+
seeds = {n.id for n in where(store, name)}
|
|
84
|
+
if not seeds:
|
|
85
|
+
return []
|
|
86
|
+
return _reverse_closure(store, seeds, depth)
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def dependent_files(store: GraphStore, files: list[str], depth: int = 1) -> set[str]:
|
|
90
|
+
"""The blast radius of a set of files, as a set of files.
|
|
91
|
+
|
|
92
|
+
The closure runs file to file rather than node to node. One file's import may
|
|
93
|
+
land on another's module node while a second lands on its file node, and at
|
|
94
|
+
the level this question is asked — can these two tasks run at once — those are
|
|
95
|
+
the same thing.
|
|
96
|
+
"""
|
|
97
|
+
normalised = {f.replace("\\", "/").lstrip("./") for f in files}
|
|
98
|
+
incoming: dict[str, set[str]] = {}
|
|
99
|
+
for edge in store.edges:
|
|
100
|
+
if edge.relation not in ("references", "imports", "exposes"):
|
|
101
|
+
continue
|
|
102
|
+
src = store.by_id.get(edge.src)
|
|
103
|
+
dst = store.by_id.get(edge.dst)
|
|
104
|
+
if src is None or dst is None or src.file == dst.file:
|
|
105
|
+
continue
|
|
106
|
+
incoming.setdefault(dst.file, set()).add(src.file)
|
|
107
|
+
|
|
108
|
+
out = set(normalised)
|
|
109
|
+
frontier = set(normalised)
|
|
110
|
+
for _ in range(max(1, depth)):
|
|
111
|
+
nxt = {f for target in frontier for f in incoming.get(target, ()) if f not in out}
|
|
112
|
+
if not nxt:
|
|
113
|
+
break
|
|
114
|
+
out |= nxt
|
|
115
|
+
frontier = nxt
|
|
116
|
+
return out
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def references(store: GraphStore, name: str) -> list[Node]:
|
|
120
|
+
"""Who calls or imports this. The other direction from `defines`."""
|
|
121
|
+
return dependents(store, name, depth=1)
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
@dataclass
|
|
125
|
+
class Overlap:
|
|
126
|
+
overlapping: set[str]
|
|
127
|
+
a_radius: set[str]
|
|
128
|
+
b_radius: set[str]
|
|
129
|
+
|
|
130
|
+
@property
|
|
131
|
+
def safe(self) -> bool:
|
|
132
|
+
return not self.overlapping
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def overlap(store: GraphStore, a: list[str], b: list[str], depth: int = 1) -> Overlap:
|
|
136
|
+
"""Do these two file sets intersect, dependents included.
|
|
137
|
+
|
|
138
|
+
This is what makes parallel execution provably safe instead of hopefully
|
|
139
|
+
safe. A task that cites a whole file inflates its radius and quietly collapses
|
|
140
|
+
a phase to serial execution, which is why `edify check` complains about one.
|
|
141
|
+
"""
|
|
142
|
+
ra = dependent_files(store, a, depth)
|
|
143
|
+
rb = dependent_files(store, b, depth)
|
|
144
|
+
return Overlap(overlapping=ra & rb, a_radius=ra, b_radius=rb)
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def resolve_ref(store: GraphStore, ref: str) -> Node | None:
|
|
148
|
+
"""`path/to/file.ts:22-38` or `path/to/file.ts` → the file node, if it exists."""
|
|
149
|
+
path = ref.split(":", 1)[0].replace("\\", "/").lstrip("./")
|
|
150
|
+
for node in store.by_file.get(path, []):
|
|
151
|
+
if node.kind == "file":
|
|
152
|
+
return node
|
|
153
|
+
return None
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def line_range_for(store: GraphStore, file: str, name: str) -> tuple[int, int] | None:
|
|
157
|
+
"""The line range of a named symbol in a file — what `check --fix` writes.
|
|
158
|
+
|
|
159
|
+
The end is the line before the next symbol in the same file, which is exact
|
|
160
|
+
enough to open the right code and never wrong in a way that hides something.
|
|
161
|
+
"""
|
|
162
|
+
target = file.replace("\\", "/").lstrip("./")
|
|
163
|
+
in_file = sorted(
|
|
164
|
+
(n for n in store.by_file.get(target, []) if n.kind in ("symbol", "route", "table")),
|
|
165
|
+
key=lambda n: n.line,
|
|
166
|
+
)
|
|
167
|
+
for i, node in enumerate(in_file):
|
|
168
|
+
if node.name == name or node.name.rsplit(".", 1)[-1] == name:
|
|
169
|
+
end = in_file[i + 1].line - 1 if i + 1 < len(in_file) else node.line + 40
|
|
170
|
+
return node.line, max(node.line, end)
|
|
171
|
+
return None
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
# ---------------------------------------------------------------------------
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def _reverse_closure(store: GraphStore, seeds: set[str], depth: int) -> list[Node]:
|
|
178
|
+
incoming: dict[str, set[str]] = {}
|
|
179
|
+
for edge in store.edges:
|
|
180
|
+
if edge.relation in ("references", "imports", "exposes"):
|
|
181
|
+
incoming.setdefault(edge.dst, set()).add(edge.src)
|
|
182
|
+
|
|
183
|
+
# An import may land on a file node or on its module node depending on the
|
|
184
|
+
# language. They are the same place, so the walk treats them as one.
|
|
185
|
+
for nodes in store.by_file.values():
|
|
186
|
+
pair = [n.id for n in nodes if n.kind in ("file", "module")]
|
|
187
|
+
for node_id in pair:
|
|
188
|
+
for sibling in pair:
|
|
189
|
+
if sibling != node_id:
|
|
190
|
+
incoming.setdefault(node_id, set()).update(incoming.get(sibling, set()))
|
|
191
|
+
|
|
192
|
+
seen: set[str] = set()
|
|
193
|
+
frontier = set(seeds)
|
|
194
|
+
for _ in range(max(1, depth)):
|
|
195
|
+
nxt: set[str] = set()
|
|
196
|
+
for node_id in frontier:
|
|
197
|
+
for src in incoming.get(node_id, ()):
|
|
198
|
+
if src not in seen and src not in seeds:
|
|
199
|
+
seen.add(src)
|
|
200
|
+
nxt.add(src)
|
|
201
|
+
if not nxt:
|
|
202
|
+
break
|
|
203
|
+
frontier = nxt
|
|
204
|
+
|
|
205
|
+
return _ordered([store.by_id[i] for i in seen if i in store.by_id])
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
def _ordered(nodes: list[Node]) -> list[Node]:
|
|
209
|
+
rank = {"symbol": 0, "route": 1, "table": 2, "module": 3, "package": 4, "doc-section": 5, "file": 6}
|
|
210
|
+
unique = {n.id: n for n in nodes}
|
|
211
|
+
return sorted(unique.values(), key=lambda n: (rank.get(n.kind, 9), n.file, n.line, n.name))
|
edify/graph/store.py
ADDED
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
"""Reading and writing `.edify/graph/`.
|
|
2
|
+
|
|
3
|
+
Three files: nodes.tsv, edges.tsv, meta. Sorted and checksummed, so a rebuild on
|
|
4
|
+
unchanged source is byte-identical and a graph diff is a structural diff of the
|
|
5
|
+
codebase.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
|
|
12
|
+
from .. import GRAPH_SCHEMA
|
|
13
|
+
from ..errors import GraphMissing
|
|
14
|
+
from ..paths import Layout
|
|
15
|
+
from ..tsv import read_kv, read_rows, sha256_file, write_kv, write_rows
|
|
16
|
+
from .model import Edge, Node
|
|
17
|
+
|
|
18
|
+
NODE_HEADER = ("id", "kind", "file", "line", "name", "fingerprint")
|
|
19
|
+
EDGE_HEADER = ("from", "relation", "to")
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class GraphStore:
|
|
23
|
+
"""The graph on disk, loaded lazily.
|
|
24
|
+
|
|
25
|
+
Nodes and edges are read on first use and kept in memory for the life of one
|
|
26
|
+
command. At the scale that matters — around a hundred thousand exported
|
|
27
|
+
symbols for a million lines — that is tens of megabytes and one pass over
|
|
28
|
+
sorted text.
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
def __init__(self, layout: Layout):
|
|
32
|
+
self.layout = layout
|
|
33
|
+
self._nodes: list[Node] | None = None
|
|
34
|
+
self._edges: list[Edge] | None = None
|
|
35
|
+
self._by_name: dict[str, list[Node]] | None = None
|
|
36
|
+
self._by_id: dict[str, Node] | None = None
|
|
37
|
+
self._by_file: dict[str, list[Node]] | None = None
|
|
38
|
+
|
|
39
|
+
# -- presence --------------------------------------------------------
|
|
40
|
+
|
|
41
|
+
def exists(self) -> bool:
|
|
42
|
+
return self.layout.nodes.exists()
|
|
43
|
+
|
|
44
|
+
def require(self) -> None:
|
|
45
|
+
if not self.exists():
|
|
46
|
+
raise GraphMissing()
|
|
47
|
+
|
|
48
|
+
# -- loading ---------------------------------------------------------
|
|
49
|
+
|
|
50
|
+
@property
|
|
51
|
+
def nodes(self) -> list[Node]:
|
|
52
|
+
if self._nodes is None:
|
|
53
|
+
self.require()
|
|
54
|
+
self._nodes = [Node.from_row(r) for r in read_rows(self.layout.nodes)]
|
|
55
|
+
return self._nodes
|
|
56
|
+
|
|
57
|
+
@property
|
|
58
|
+
def edges(self) -> list[Edge]:
|
|
59
|
+
if self._edges is None:
|
|
60
|
+
self.require()
|
|
61
|
+
self._edges = [Edge.from_row(r) for r in read_rows(self.layout.edges)]
|
|
62
|
+
return self._edges
|
|
63
|
+
|
|
64
|
+
@property
|
|
65
|
+
def by_name(self) -> dict[str, list[Node]]:
|
|
66
|
+
if self._by_name is None:
|
|
67
|
+
index: dict[str, list[Node]] = {}
|
|
68
|
+
for n in self.nodes:
|
|
69
|
+
index.setdefault(n.name, []).append(n)
|
|
70
|
+
# The last segment too, so `where accept` finds `Service.accept`.
|
|
71
|
+
tail = n.name.rsplit(".", 1)[-1]
|
|
72
|
+
if tail != n.name:
|
|
73
|
+
index.setdefault(tail, []).append(n)
|
|
74
|
+
self._by_name = index
|
|
75
|
+
return self._by_name
|
|
76
|
+
|
|
77
|
+
@property
|
|
78
|
+
def by_id(self) -> dict[str, Node]:
|
|
79
|
+
if self._by_id is None:
|
|
80
|
+
self._by_id = {n.id: n for n in self.nodes}
|
|
81
|
+
return self._by_id
|
|
82
|
+
|
|
83
|
+
@property
|
|
84
|
+
def by_file(self) -> dict[str, list[Node]]:
|
|
85
|
+
if self._by_file is None:
|
|
86
|
+
index: dict[str, list[Node]] = {}
|
|
87
|
+
for n in self.nodes:
|
|
88
|
+
index.setdefault(n.file, []).append(n)
|
|
89
|
+
self._by_file = index
|
|
90
|
+
return self._by_file
|
|
91
|
+
|
|
92
|
+
@property
|
|
93
|
+
def meta(self) -> dict[str, str]:
|
|
94
|
+
return read_kv(self.layout.meta)
|
|
95
|
+
|
|
96
|
+
# -- writing ---------------------------------------------------------
|
|
97
|
+
|
|
98
|
+
def write(self, nodes: list[Node], edges: list[Edge], meta: dict[str, str]) -> dict[str, str]:
|
|
99
|
+
"""Write all three files and return the meta actually recorded."""
|
|
100
|
+
self.layout.graph_dir.mkdir(parents=True, exist_ok=True)
|
|
101
|
+
seen_nodes: dict[str, Node] = {}
|
|
102
|
+
for n in nodes:
|
|
103
|
+
# First writer wins on a duplicate id, which makes the result
|
|
104
|
+
# independent of directory iteration order.
|
|
105
|
+
seen_nodes.setdefault(n.id, n)
|
|
106
|
+
deduped_nodes = sorted(seen_nodes.values(), key=lambda n: n.id)
|
|
107
|
+
deduped_edges = sorted({(e.src, e.relation, e.dst) for e in edges})
|
|
108
|
+
|
|
109
|
+
nodes_sha = write_rows(self.layout.nodes, (n.row() for n in deduped_nodes), NODE_HEADER)
|
|
110
|
+
edges_sha = write_rows(
|
|
111
|
+
self.layout.edges, ([s, r, d] for s, r, d in deduped_edges), EDGE_HEADER
|
|
112
|
+
)
|
|
113
|
+
full = dict(meta)
|
|
114
|
+
full.update(
|
|
115
|
+
{
|
|
116
|
+
"schema": str(GRAPH_SCHEMA),
|
|
117
|
+
"nodes": str(len(deduped_nodes)),
|
|
118
|
+
"edges": str(len(deduped_edges)),
|
|
119
|
+
"nodes_sha256": nodes_sha,
|
|
120
|
+
"edges_sha256": edges_sha,
|
|
121
|
+
}
|
|
122
|
+
)
|
|
123
|
+
write_kv(self.layout.meta, full)
|
|
124
|
+
self._nodes = deduped_nodes
|
|
125
|
+
self._edges = [Edge(s, r, d) for s, r, d in deduped_edges]
|
|
126
|
+
self._by_name = self._by_id = self._by_file = None
|
|
127
|
+
return full
|
|
128
|
+
|
|
129
|
+
# -- integrity -------------------------------------------------------
|
|
130
|
+
|
|
131
|
+
def verify_checksums(self) -> list[str]:
|
|
132
|
+
"""Complaints, one line each. Empty means the files match their meta."""
|
|
133
|
+
problems: list[str] = []
|
|
134
|
+
meta = self.meta
|
|
135
|
+
for path, key in ((self.layout.nodes, "nodes_sha256"), (self.layout.edges, "edges_sha256")):
|
|
136
|
+
recorded = meta.get(key)
|
|
137
|
+
if not recorded:
|
|
138
|
+
problems.append(f"{path.name}: no checksum recorded in meta")
|
|
139
|
+
continue
|
|
140
|
+
actual = sha256_file(path)
|
|
141
|
+
if actual != recorded:
|
|
142
|
+
problems.append(f"{path.name}: content does not match its recorded checksum")
|
|
143
|
+
if meta.get("schema") and meta["schema"] != str(GRAPH_SCHEMA):
|
|
144
|
+
problems.append(
|
|
145
|
+
f"graph schema is {meta['schema']}, this edify writes {GRAPH_SCHEMA}"
|
|
146
|
+
" — run `edify graph build`"
|
|
147
|
+
)
|
|
148
|
+
return problems
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def load(layout: Layout) -> GraphStore:
|
|
152
|
+
return GraphStore(layout)
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def graph_paths(root: Path) -> tuple[Path, Path, Path]:
|
|
156
|
+
base = root / ".edify" / "graph"
|
|
157
|
+
return base / "nodes.tsv", base / "edges.tsv", base / "meta"
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
"""The mechanical half of `/harvest`.
|
|
2
|
+
|
|
3
|
+
Curation is four steps and one door.
|
|
4
|
+
Three of the four steps are judgment and belong to a model; the parts that are not
|
|
5
|
+
judgment live here, because a lookup that gives the same answer on Tuesday is worth
|
|
6
|
+
more than a careful reading that does not.
|
|
7
|
+
|
|
8
|
+
What this package decides, and a model never does:
|
|
9
|
+
|
|
10
|
+
- **the licence** — an allowlist lookup, fail-closed. Whether we may ship somebody
|
|
11
|
+
else's work is a legal question, not a plausible-sounding one.
|
|
12
|
+
- **whether we have seen this before** — a hash comparison against `ledger.json`.
|
|
13
|
+
- **whether the text is trying to give instructions** — a pattern scan whose only
|
|
14
|
+
outcome is quarantine, which is a place things go rather than a decision made in
|
|
15
|
+
passing.
|
|
16
|
+
|
|
17
|
+
What it refuses to decide: admission. There is one door and a person is standing
|
|
18
|
+
in it (`admit` will not run unattended, by construction).
|
|
19
|
+
|
|
20
|
+
Nothing here opens a socket. Gathering is the caller's job — the CLI stays a pure
|
|
21
|
+
function over files, so the bytes are already on disk before any of this runs.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
from __future__ import annotations
|
|
25
|
+
|
|
26
|
+
from . import ledger, screen
|
|
27
|
+
|
|
28
|
+
__all__ = ["ledger", "screen"]
|
edify/harvest/ledger.py
ADDED
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
"""The harvest ledger — every candidate ever seen, and what became of it.
|
|
2
|
+
|
|
3
|
+
One file, one job: **nothing is gathered, screened, or offered twice.** A harvest
|
|
4
|
+
run that re-proposes last month's rejection wastes the only expensive step in the
|
|
5
|
+
pipeline, which is the human reading at the end of it.
|
|
6
|
+
|
|
7
|
+
A candidate is recognised by three independent keys — its URL, the sha256 of its
|
|
8
|
+
raw bytes, and the entry name it would take. Any one matching is a match, because
|
|
9
|
+
the same file moves between repositories, gets re-tagged, and arrives renamed.
|
|
10
|
+
|
|
11
|
+
JSON rather than TSV, unlike the rest of `.edify/`: a ledger row is a nested record
|
|
12
|
+
(a screen verdict holds a list of checks) and the sorted-TSV property that makes the
|
|
13
|
+
graph diffable buys nothing for a file nobody rebuilds. Written sorted and indented
|
|
14
|
+
so a diff still reads.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
import json
|
|
20
|
+
import re
|
|
21
|
+
from dataclasses import dataclass, field
|
|
22
|
+
from datetime import date
|
|
23
|
+
from pathlib import Path
|
|
24
|
+
from typing import Any, Iterator
|
|
25
|
+
|
|
26
|
+
SCHEMA = 1
|
|
27
|
+
|
|
28
|
+
# In flight, then terminal. A candidate in a terminal state is never offered again;
|
|
29
|
+
# one in flight is resumable but also never re-gathered.
|
|
30
|
+
IN_FLIGHT = ("staged", "screened", "proposed")
|
|
31
|
+
TERMINAL = ("admitted", "rejected", "quarantined")
|
|
32
|
+
STATES = IN_FLIGHT + TERMINAL
|
|
33
|
+
|
|
34
|
+
_SLUG = re.compile(r"[^a-z0-9]+")
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def slug(text: str) -> str:
|
|
38
|
+
return _SLUG.sub("-", text.strip().lower()).strip("-") or "candidate"
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def candidate_id(name: str, content_sha: str) -> str:
|
|
42
|
+
"""Stable across runs: the same bytes under the same name are the same id."""
|
|
43
|
+
return f"{slug(name)}-{content_sha[:8]}"
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def normalise_url(url: str) -> str:
|
|
47
|
+
"""Enough normalisation that a trailing slash is not a second candidate."""
|
|
48
|
+
u = url.strip().lower()
|
|
49
|
+
u = re.sub(r"^https?://", "", u)
|
|
50
|
+
u = re.sub(r"^www\.", "", u)
|
|
51
|
+
return u.rstrip("/")
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
@dataclass
|
|
55
|
+
class Candidate:
|
|
56
|
+
id: str
|
|
57
|
+
name: str
|
|
58
|
+
url: str
|
|
59
|
+
content_sha256: str
|
|
60
|
+
source: str = "-"
|
|
61
|
+
commit: str = "-"
|
|
62
|
+
license: str = "NONE-FOUND"
|
|
63
|
+
state: str = "staged"
|
|
64
|
+
first_seen: str = ""
|
|
65
|
+
updated: str = ""
|
|
66
|
+
screen: dict[str, Any] = field(default_factory=dict)
|
|
67
|
+
disposition: dict[str, Any] = field(default_factory=dict)
|
|
68
|
+
|
|
69
|
+
@property
|
|
70
|
+
def screened_pass(self) -> bool:
|
|
71
|
+
return self.screen.get("verdict") == "pass"
|
|
72
|
+
|
|
73
|
+
def to_json(self) -> dict[str, Any]:
|
|
74
|
+
return {
|
|
75
|
+
"id": self.id,
|
|
76
|
+
"name": self.name,
|
|
77
|
+
"url": self.url,
|
|
78
|
+
"content_sha256": self.content_sha256,
|
|
79
|
+
"source": self.source,
|
|
80
|
+
"commit": self.commit,
|
|
81
|
+
"license": self.license,
|
|
82
|
+
"state": self.state,
|
|
83
|
+
"first_seen": self.first_seen,
|
|
84
|
+
"updated": self.updated,
|
|
85
|
+
"screen": self.screen,
|
|
86
|
+
"disposition": self.disposition,
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
@classmethod
|
|
90
|
+
def from_json(cls, data: dict[str, Any]) -> "Candidate":
|
|
91
|
+
known = {f for f in cls.__dataclass_fields__}
|
|
92
|
+
return cls(**{k: v for k, v in data.items() if k in known})
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
class Ledger:
|
|
96
|
+
"""The whole file, held in memory. It is a few hundred records at most."""
|
|
97
|
+
|
|
98
|
+
def __init__(self, path: Path, candidates: dict[str, Candidate] | None = None):
|
|
99
|
+
self.path = path
|
|
100
|
+
self.candidates: dict[str, Candidate] = candidates or {}
|
|
101
|
+
|
|
102
|
+
# -- io --------------------------------------------------------------
|
|
103
|
+
|
|
104
|
+
@classmethod
|
|
105
|
+
def load(cls, path: Path) -> "Ledger":
|
|
106
|
+
"""A missing ledger is an empty one — the first run must not need a setup step."""
|
|
107
|
+
if not path.is_file():
|
|
108
|
+
return cls(path)
|
|
109
|
+
raw = json.loads(path.read_text(encoding="utf-8") or "{}")
|
|
110
|
+
found = raw.get("schema", SCHEMA)
|
|
111
|
+
if found > SCHEMA:
|
|
112
|
+
raise ValueError(f"{path} is schema {found}; this edify understands {SCHEMA}")
|
|
113
|
+
entries = {k: Candidate.from_json(v) for k, v in (raw.get("candidates") or {}).items()}
|
|
114
|
+
return cls(path, entries)
|
|
115
|
+
|
|
116
|
+
def save(self) -> None:
|
|
117
|
+
payload = {
|
|
118
|
+
"schema": SCHEMA,
|
|
119
|
+
"updated": date.today().isoformat(),
|
|
120
|
+
"candidates": {k: self.candidates[k].to_json() for k in sorted(self.candidates)},
|
|
121
|
+
}
|
|
122
|
+
self.path.parent.mkdir(parents=True, exist_ok=True)
|
|
123
|
+
self.path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
|
124
|
+
|
|
125
|
+
# -- the one question this file exists to answer ----------------------
|
|
126
|
+
|
|
127
|
+
def seen(self, *, url: str = "", content_sha: str = "", name: str = "") -> Candidate | None:
|
|
128
|
+
"""Have we handled this before, under any of its three identities?"""
|
|
129
|
+
wanted_url = normalise_url(url) if url else ""
|
|
130
|
+
for cand in self.candidates.values():
|
|
131
|
+
if wanted_url and normalise_url(cand.url) == wanted_url:
|
|
132
|
+
return cand
|
|
133
|
+
if content_sha and cand.content_sha256 == content_sha:
|
|
134
|
+
return cand
|
|
135
|
+
if name and cand.name == name:
|
|
136
|
+
return cand
|
|
137
|
+
return None
|
|
138
|
+
|
|
139
|
+
# -- mutation --------------------------------------------------------
|
|
140
|
+
|
|
141
|
+
def add(self, cand: Candidate) -> Candidate:
|
|
142
|
+
today = date.today().isoformat()
|
|
143
|
+
cand.first_seen = cand.first_seen or today
|
|
144
|
+
cand.updated = today
|
|
145
|
+
self.candidates[cand.id] = cand
|
|
146
|
+
return cand
|
|
147
|
+
|
|
148
|
+
def get(self, cid: str) -> Candidate | None:
|
|
149
|
+
if cid in self.candidates:
|
|
150
|
+
return self.candidates[cid]
|
|
151
|
+
# A prefix is enough as long as it is unambiguous — ids carry a hash tail
|
|
152
|
+
# nobody wants to type.
|
|
153
|
+
matches = [c for k, c in self.candidates.items() if k.startswith(cid)]
|
|
154
|
+
return matches[0] if len(matches) == 1 else None
|
|
155
|
+
|
|
156
|
+
def set_state(self, cand: Candidate, state: str) -> None:
|
|
157
|
+
if state not in STATES:
|
|
158
|
+
raise ValueError(f"unknown state: {state}")
|
|
159
|
+
cand.state = state
|
|
160
|
+
cand.updated = date.today().isoformat()
|
|
161
|
+
|
|
162
|
+
# -- reading ---------------------------------------------------------
|
|
163
|
+
|
|
164
|
+
def in_state(self, *states: str) -> list[Candidate]:
|
|
165
|
+
return [c for c in self.sorted() if c.state in states]
|
|
166
|
+
|
|
167
|
+
def sorted(self) -> list[Candidate]:
|
|
168
|
+
return [self.candidates[k] for k in sorted(self.candidates)]
|
|
169
|
+
|
|
170
|
+
def counts(self) -> dict[str, int]:
|
|
171
|
+
out = {s: 0 for s in STATES}
|
|
172
|
+
for cand in self.candidates.values():
|
|
173
|
+
out[cand.state] = out.get(cand.state, 0) + 1
|
|
174
|
+
return out
|
|
175
|
+
|
|
176
|
+
def by_source(self) -> Iterator[tuple[str, int, int, int, str]]:
|
|
177
|
+
"""`source, admitted, rejected, seen, last run` — the yield history sources.md carries.
|
|
178
|
+
|
|
179
|
+
Counts come from the human door, not the screen. A source with a high screen
|
|
180
|
+
pass rate and a low admission rate is producing plausible files, which is the
|
|
181
|
+
failure mode worth noticing.
|
|
182
|
+
"""
|
|
183
|
+
sources: dict[str, list[Candidate]] = {}
|
|
184
|
+
for cand in self.candidates.values():
|
|
185
|
+
sources.setdefault(cand.source or "-", []).append(cand)
|
|
186
|
+
for name in sorted(sources):
|
|
187
|
+
group = sources[name]
|
|
188
|
+
admitted = sum(1 for c in group if c.state == "admitted")
|
|
189
|
+
rejected = sum(1 for c in group if c.state in ("rejected", "quarantined"))
|
|
190
|
+
last = max((c.updated for c in group if c.updated), default="never")
|
|
191
|
+
yield name, admitted, rejected, len(group), last
|