reqctl 0.1.38__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- reqctl/__init__.py +0 -0
- reqctl/baseline.py +380 -0
- reqctl/cite.py +1229 -0
- reqctl/cli.py +1514 -0
- reqctl/corpus.py +1283 -0
- reqctl/fields.py +361 -0
- reqctl/graph.py +207 -0
- reqctl/history.py +175 -0
- reqctl/portal.html +2213 -0
- reqctl/portal.py +131 -0
- reqctl/reach.py +425 -0
- reqctl/resolve.py +74 -0
- reqctl/schemas/baseline.schema.yaml +42 -0
- reqctl/schemas/dictionary.schema.yaml +256 -0
- reqctl/schemas/guard.schema.yaml +131 -0
- reqctl/schemas/requirement.schema.yaml +152 -0
- reqctl/settings.py +183 -0
- reqctl/typescript.py +143 -0
- reqctl/uncited.py +257 -0
- reqctl/validate.py +1929 -0
- reqctl/write.py +2004 -0
- reqctl-0.1.38.dist-info/METADATA +11 -0
- reqctl-0.1.38.dist-info/RECORD +26 -0
- reqctl-0.1.38.dist-info/WHEEL +5 -0
- reqctl-0.1.38.dist-info/entry_points.txt +2 -0
- reqctl-0.1.38.dist-info/top_level.txt +1 -0
reqctl/settings.py
ADDED
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
import re
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
|
|
4
|
+
from .baseline import _default_ref, _git
|
|
5
|
+
from .corpus import ReqctlError, loads, mapping
|
|
6
|
+
|
|
7
|
+
SETTINGS = Path("requirements") / "settings.yml"
|
|
8
|
+
PINNED = re.compile(r"claude-[a-z]+-([0-9]{1,2})(?:-([0-9]{1,2}))?(-[0-9]{8})?")
|
|
9
|
+
DATELESS_FROM = (4, 6)
|
|
10
|
+
AGENTS = "elucidate_agents"
|
|
11
|
+
NESTED = ("nest each value under its data item, entry and field, as "
|
|
12
|
+
"review_limits: {page_bytes: {quantity: 10485760}}")
|
|
13
|
+
# @req+ REQ-87066486@AtzPEAcAbJ1v 7yxrtp
|
|
14
|
+
# @req> review_limits@m0AH5ohPl52k ulaguu
|
|
15
|
+
REVIEW_LIMITS = {
|
|
16
|
+
"fetch_attempts": {"quantity": 3},
|
|
17
|
+
"fetch_seconds": {"quantity": 60},
|
|
18
|
+
"page_bytes": {"quantity": 5242880},
|
|
19
|
+
"redirects": {"quantity": 5},
|
|
20
|
+
"summary_lines": {"quantity": 10},
|
|
21
|
+
}
|
|
22
|
+
# @req> challenge_bounds@M9O-R5iFxL52 j6wvnl
|
|
23
|
+
CHALLENGE_BOUNDS = {
|
|
24
|
+
"agent_ceiling": {"quantity": 150},
|
|
25
|
+
"floor_k": {"quantity": 10},
|
|
26
|
+
"judge_bound": {"quantity": 45},
|
|
27
|
+
"judge_group": {"quantity": 20},
|
|
28
|
+
"recall_batch": {"quantity": 40},
|
|
29
|
+
}
|
|
30
|
+
# @req> elucidate_agents@n9BNBtOe7I6g uyqkbs
|
|
31
|
+
ELUCIDATE_AGENTS = {
|
|
32
|
+
"best_in_class": {"model": "claude-sonnet-5-5", "effort": "high"},
|
|
33
|
+
"coverage": {"model": "claude-sonnet-5-5", "effort": "high"},
|
|
34
|
+
"recall": {"model": "claude-sonnet-5-5", "effort": "medium"},
|
|
35
|
+
"judge": {"model": "claude-opus-5-5", "effort": "high"},
|
|
36
|
+
}
|
|
37
|
+
# @req> portal_token_permissions@vjGyU5xrFJwz xjvwbq
|
|
38
|
+
PORTAL_TOKEN_PERMISSIONS = {
|
|
39
|
+
"actions": {"access": "read and write"},
|
|
40
|
+
"contents": {"access": "read"},
|
|
41
|
+
"pull_requests": {"access": "read"},
|
|
42
|
+
}
|
|
43
|
+
# @req> uncited_code@unpaxg37rwqQ ishqik
|
|
44
|
+
UNCITED_CODE = {
|
|
45
|
+
"check": {"mode": "off"},
|
|
46
|
+
}
|
|
47
|
+
SHIPPED = {"review_limits": REVIEW_LIMITS, "challenge_bounds": CHALLENGE_BOUNDS,
|
|
48
|
+
AGENTS: ELUCIDATE_AGENTS,
|
|
49
|
+
"portal_token_permissions": PORTAL_TOKEN_PERMISSIONS,
|
|
50
|
+
"uncited_code": UNCITED_CODE}
|
|
51
|
+
# @req- 7yxrtp
|
|
52
|
+
MODES = ("off", "all", "changed_lines", "ratchet")
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def path(root):
|
|
56
|
+
return Path(root) / SETTINGS
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def pinned(value):
|
|
60
|
+
found = PINNED.fullmatch(value)
|
|
61
|
+
return bool(found) and ((int(found[1]), int(found[2] or 0)) >= DATELESS_FROM) != bool(found[3])
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
# @req> REQ-42769459@j_YdEMFBAA1J ipg3lp
|
|
65
|
+
def _default(root):
|
|
66
|
+
try:
|
|
67
|
+
# @req> REQ-32331515@Nn3qX9jMrAU- myaobz
|
|
68
|
+
return _default_ref(root)
|
|
69
|
+
except ReqctlError:
|
|
70
|
+
return None
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _leaves(held, shipped, at):
|
|
74
|
+
if not isinstance(held, dict) or not isinstance(shipped, dict):
|
|
75
|
+
yield at, held, shipped
|
|
76
|
+
return
|
|
77
|
+
# @req> REQ-99278123@vBQXVC8aZZp3 moafaq
|
|
78
|
+
if not held and at:
|
|
79
|
+
yield at, held, shipped
|
|
80
|
+
for key, value in held.items():
|
|
81
|
+
yield from _leaves(value, shipped.get(key, {}), (*at, key))
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def faults(document):
|
|
85
|
+
if document is None:
|
|
86
|
+
return []
|
|
87
|
+
# @req> REQ-99278123@vBQXVC8aZZp3 fv2afe
|
|
88
|
+
if not isinstance(document, dict):
|
|
89
|
+
return [f"{document!r} names no value a settings file may replace -- "
|
|
90
|
+
f"{NESTED}"]
|
|
91
|
+
found = []
|
|
92
|
+
for at, value, shipped in _leaves(document, SHIPPED, ()):
|
|
93
|
+
key = ".".join(map(str, at))
|
|
94
|
+
# @req> REQ-99278123@vBQXVC8aZZp3 ck7iqy
|
|
95
|
+
if isinstance(shipped, dict):
|
|
96
|
+
found.append(f"{key} names no value a settings file may replace -- "
|
|
97
|
+
f"{NESTED}")
|
|
98
|
+
# @req> REQ-78887253@ZroFZuS_zv0q 33gi6b
|
|
99
|
+
elif type(value) is not type(shipped):
|
|
100
|
+
# @req> REQ-30898926@Vu28eWaG_1mE aq4jso
|
|
101
|
+
found.append(f"{key}: {value!r} is not of the YAML type of the "
|
|
102
|
+
f"{shipped!r} the product ships -- state a value of "
|
|
103
|
+
"that type, on the default branch")
|
|
104
|
+
# @req> REQ-96727029@rK6NcCslzP2S hrxovc
|
|
105
|
+
elif at == ("uncited_code", "check", "mode") and value not in MODES:
|
|
106
|
+
found.append(f"{key}: {value!r} is not a mode -- state off, all, "
|
|
107
|
+
"changed_lines or ratchet")
|
|
108
|
+
# @req> REQ-21585959@QkioE_Ciddeh q7jgw2
|
|
109
|
+
elif at[-1] == "quantity" and value < 1:
|
|
110
|
+
# @req> REQ-30898926@Vu28eWaG_1mE c6k5gb
|
|
111
|
+
found.append(f"{key}: {value!r} is not a whole number of at least "
|
|
112
|
+
"one -- state one, on the default branch")
|
|
113
|
+
# @req> REQ-15362279@YwMnA4sMLpfF 5pi6tp
|
|
114
|
+
# @req> REQ-82356895@juJepJj5Wt1K rtqqce
|
|
115
|
+
elif at[-1] == "model" and not pinned(value):
|
|
116
|
+
found.append(f"{key}: {value!r} is not a pinned model ID, such as "
|
|
117
|
+
"claude-opus-5-5")
|
|
118
|
+
# @req+ REQ-59792666@dkbVbhN0Y4FS sspl2m
|
|
119
|
+
# @req> REQ-96661837@HbvreaRTySWH 4iahip
|
|
120
|
+
for entry, fields in mapping(document, AGENTS).items():
|
|
121
|
+
if (isinstance(fields, dict) and {"model", "effort"} & fields.keys()
|
|
122
|
+
and not (fields.get("model") and fields.get("effort"))):
|
|
123
|
+
found.append(f"{AGENTS}.{entry} states a model without an effort, "
|
|
124
|
+
"or an effort without a model -- state both")
|
|
125
|
+
# @req- sspl2m
|
|
126
|
+
return found
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
# @req> REQ-34403718@OXYW6pAY1oxO 7tnuse
|
|
130
|
+
def read(root):
|
|
131
|
+
# @req+ REQ-32331515@Nn3qX9jMrAU- 7jdohw
|
|
132
|
+
ref = _default(root)
|
|
133
|
+
# @req> REQ-42769459@j_YdEMFBAA1J mfrzc7
|
|
134
|
+
if ref is None:
|
|
135
|
+
return None, {}
|
|
136
|
+
named = SETTINGS.as_posix()
|
|
137
|
+
where = f"{ref.removeprefix('refs/remotes/')}:{named}"
|
|
138
|
+
listed = _git(root, "ls-tree", ref, "--", named)
|
|
139
|
+
# @req> REQ-29005932@BIHyox0VxBlI v2mmft
|
|
140
|
+
if listed.returncode:
|
|
141
|
+
# @req> REQ-30898926@Vu28eWaG_1mE 2omnbm
|
|
142
|
+
raise ReqctlError(f"cannot read the tree {ref} names, so whether it "
|
|
143
|
+
f"holds {named} cannot be told: "
|
|
144
|
+
f"{listed.stderr.strip()} -- run `git fetch origin`, "
|
|
145
|
+
"then retry")
|
|
146
|
+
# @req> REQ-31526771@dzOAej5vGh2Y wa4iuz
|
|
147
|
+
if not listed.stdout.strip():
|
|
148
|
+
return where, {}
|
|
149
|
+
shown = _git(root, "show", f"{ref}:{named}")
|
|
150
|
+
# @req> REQ-29005932@BIHyox0VxBlI s4pa57
|
|
151
|
+
if shown.returncode:
|
|
152
|
+
# @req> REQ-30898926@Vu28eWaG_1mE vocmcm
|
|
153
|
+
raise ReqctlError(f"{where} cannot be read: {shown.stderr.strip()} -- "
|
|
154
|
+
"restore the file on the default branch")
|
|
155
|
+
# @req> REQ-29005932@BIHyox0VxBlI mgfbvw
|
|
156
|
+
document = loads(shown.stdout, where)
|
|
157
|
+
# @req- 7jdohw
|
|
158
|
+
found = faults(document)
|
|
159
|
+
# @req> REQ-99278123@vBQXVC8aZZp3 lcnls5
|
|
160
|
+
# @req> REQ-78887253@ZroFZuS_zv0q iywc46
|
|
161
|
+
# @req> REQ-21585959@QkioE_Ciddeh blmk44
|
|
162
|
+
# @req> REQ-15362279@YwMnA4sMLpfF fxnobd
|
|
163
|
+
# @req> REQ-59792666@dkbVbhN0Y4FS txoz2e
|
|
164
|
+
if found:
|
|
165
|
+
# @req> REQ-30898926@Vu28eWaG_1mE jg7ukv
|
|
166
|
+
raise ReqctlError(f"{where} cannot be used -- correct it on the "
|
|
167
|
+
"default branch:\n"
|
|
168
|
+
+ "\n".join(f" {fault}" for fault in found))
|
|
169
|
+
return where, {".".join(at): value
|
|
170
|
+
for at, value, _ in _leaves(document or {}, SHIPPED, ())}
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
# @req> REQ-40447106@_7d9O2HDnoPr jyttul
|
|
174
|
+
# @req> REQ-31526771@dzOAej5vGh2Y ncvzwe
|
|
175
|
+
def table(stated, item):
|
|
176
|
+
return {entry: {field: stated.get(f"{item}.{entry}.{field}", value)
|
|
177
|
+
for field, value in fields.items()}
|
|
178
|
+
for entry, fields in SHIPPED[item].items()}
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def quantities(stated, item):
|
|
182
|
+
return {entry: fields["quantity"]
|
|
183
|
+
for entry, fields in table(stated, item).items()}
|
reqctl/typescript.py
ADDED
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
import functools
|
|
2
|
+
from dataclasses import dataclass, field
|
|
3
|
+
|
|
4
|
+
import tree_sitter
|
|
5
|
+
import tree_sitter_typescript
|
|
6
|
+
|
|
7
|
+
GRAMMARS = {".ts": tree_sitter.Language(tree_sitter_typescript.language_typescript()),
|
|
8
|
+
".tsx": tree_sitter.Language(tree_sitter_typescript.language_tsx())}
|
|
9
|
+
SUFFIXES = tuple(GRAMMARS)
|
|
10
|
+
KEPT = ("string", "template_string", "regex", "jsx_text")
|
|
11
|
+
TYPED = ("type_annotation", "type_arguments", "type_parameters", "type_alias_declaration",
|
|
12
|
+
"interface_declaration", "type_query", "implements_clause", "extends_type_clause",
|
|
13
|
+
"nested_type_identifier", "type_predicate_annotation", "asserts_annotation",
|
|
14
|
+
"ambient_declaration", "import_statement")
|
|
15
|
+
NAMED = ("identifier", "shorthand_property_identifier")
|
|
16
|
+
TAGS = ("jsx_opening_element", "jsx_closing_element", "jsx_self_closing_element")
|
|
17
|
+
EMBEDDING = ("jsx_expression", "jsx_opening_element", "jsx_self_closing_element")
|
|
18
|
+
DECLARED = ("function_declaration", "generator_function_declaration", "enum_declaration")
|
|
19
|
+
BINDING = ("lexical_declaration", "variable_declaration")
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
@dataclass
|
|
23
|
+
class Reading:
|
|
24
|
+
spans: list = field(default_factory=list)
|
|
25
|
+
literals: dict = field(default_factory=dict)
|
|
26
|
+
regions: list = field(default_factory=list)
|
|
27
|
+
table: dict = field(default_factory=dict)
|
|
28
|
+
names: list = field(default_factory=list)
|
|
29
|
+
imports: set = field(default_factory=set)
|
|
30
|
+
broken: int | None = None
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _bound(node):
|
|
34
|
+
if node is None:
|
|
35
|
+
return []
|
|
36
|
+
if node.type in ("identifier", "shorthand_property_identifier_pattern"):
|
|
37
|
+
return [node.text.decode()]
|
|
38
|
+
if node.type == "pair_pattern":
|
|
39
|
+
return _bound(node.child_by_field_name("value"))
|
|
40
|
+
if node.type in ("object_assignment_pattern", "assignment_pattern"):
|
|
41
|
+
return _bound(node.child_by_field_name("left"))
|
|
42
|
+
if node.type in ("object_pattern", "array_pattern", "rest_pattern"):
|
|
43
|
+
return [name for child in node.named_children for name in _bound(child)]
|
|
44
|
+
return []
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
# @req> REQ-45330509@y34lW04HzVzy u54ghq
|
|
48
|
+
def _declarations(root):
|
|
49
|
+
table = {}
|
|
50
|
+
for outer in root.named_children:
|
|
51
|
+
inner = outer.child_by_field_name("declaration") if outer.type == "export_statement" else outer
|
|
52
|
+
if inner is None:
|
|
53
|
+
continue
|
|
54
|
+
span = (outer.start_point.row + 1, outer.end_point.row + 1)
|
|
55
|
+
if inner.type in DECLARED:
|
|
56
|
+
names = _bound(inner.child_by_field_name("name"))
|
|
57
|
+
elif inner.type in BINDING:
|
|
58
|
+
names = [name for child in inner.named_children if child.type == "variable_declarator"
|
|
59
|
+
for name in _bound(child.child_by_field_name("name"))]
|
|
60
|
+
else:
|
|
61
|
+
names = []
|
|
62
|
+
for name in names:
|
|
63
|
+
table.setdefault(name, []).append(("def", span))
|
|
64
|
+
return table
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _intrinsic(node):
|
|
68
|
+
parent = node.parent
|
|
69
|
+
return (parent is not None and parent.type in TAGS
|
|
70
|
+
and parent.child_by_field_name("name") == node and node.text[:1].islower())
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _region(node):
|
|
74
|
+
if node.type == "jsx_element":
|
|
75
|
+
opened, closed = node.child_by_field_name("open_tag"), node.child_by_field_name("close_tag")
|
|
76
|
+
return (opened.end_point if opened else node.start_point,
|
|
77
|
+
closed.start_point if closed else node.end_point, True)
|
|
78
|
+
if node.type in EMBEDDING:
|
|
79
|
+
return node.start_point, node.end_point, False
|
|
80
|
+
return None
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def _among(regions, at):
|
|
84
|
+
held = [region for region in regions if region[0] <= at < region[1]]
|
|
85
|
+
return bool(held) and max(held, key=lambda region: region[0])[2]
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _offsets(text):
|
|
89
|
+
lines = text.split("\n")
|
|
90
|
+
starts, at = [], 0
|
|
91
|
+
for line in lines:
|
|
92
|
+
starts.append(at)
|
|
93
|
+
at += len(line) + 1
|
|
94
|
+
return lambda point: starts[point.row] + len(
|
|
95
|
+
lines[point.row].encode()[:point.column].decode(errors="ignore"))
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
# @req+ REQ-60833823@pdX0Xr4bVNDt n7el7s
|
|
99
|
+
# @req+ REQ-71419978@R9LaJT-ldfB9 v46n7n
|
|
100
|
+
@functools.cache
|
|
101
|
+
def read(suffix, text):
|
|
102
|
+
tree = tree_sitter.Parser(GRAMMARS[suffix]).parse(text.encode())
|
|
103
|
+
reading, offset = Reading(), _offsets(text)
|
|
104
|
+
reading.table = _declarations(tree.root_node)
|
|
105
|
+
every, todo = [], [tree.root_node]
|
|
106
|
+
while todo:
|
|
107
|
+
node = todo.pop()
|
|
108
|
+
every.append(node)
|
|
109
|
+
todo += reversed(node.children)
|
|
110
|
+
reading.regions = [region for node in every if (region := _region(node)) is not None]
|
|
111
|
+
reading.broken = next((node.start_point.row + 1 for node in every
|
|
112
|
+
if node.is_error or node.is_missing), None)
|
|
113
|
+
held_until = typed_until = -1
|
|
114
|
+
for node in every:
|
|
115
|
+
if node.start_byte >= held_until and (node.type == "comment" or node.type in KEPT):
|
|
116
|
+
rendered = node.type != "comment" or _among(reading.regions, node.start_point)
|
|
117
|
+
reading.spans.append((offset(node.start_point), offset(node.end_point), rendered))
|
|
118
|
+
held_until = node.end_byte
|
|
119
|
+
if node.type == "template_string":
|
|
120
|
+
reading.literals.update(dict.fromkeys(
|
|
121
|
+
range(node.start_point.row + 2, node.end_point.row + 1), "template literal"))
|
|
122
|
+
if node.type == "import_statement":
|
|
123
|
+
reading.imports.update(range(node.start_point.row + 1, node.end_point.row + 2))
|
|
124
|
+
if node.type in TYPED or node.type.endswith("_type"):
|
|
125
|
+
typed_until = max(typed_until, node.end_byte)
|
|
126
|
+
if node.type in NAMED and node.start_byte >= typed_until and not _intrinsic(node):
|
|
127
|
+
reading.names.append((node.start_point.row + 1, node.text.decode()))
|
|
128
|
+
return reading
|
|
129
|
+
# @req- v46n7n
|
|
130
|
+
# @req- n7el7s
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
# @req> REQ-62110789@XlGrbQnAd80g 3nom7d
|
|
134
|
+
# @req> REQ-29782682@vNwe2DOIu-N0 ckfd4o
|
|
135
|
+
def blanked(text, marked):
|
|
136
|
+
return "\n".join(" " * len(line) if number in marked else line
|
|
137
|
+
for number, line in enumerate(text.split("\n"), start=1))
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def embedded(suffix, text, marked):
|
|
141
|
+
reading = read(suffix, blanked(text, marked))
|
|
142
|
+
return {**reading.literals, **{number: "JSX element" for number in marked
|
|
143
|
+
if _among(reading.regions, (number - 1, 0))}}
|
reqctl/uncited.py
ADDED
|
@@ -0,0 +1,257 @@
|
|
|
1
|
+
import ast
|
|
2
|
+
import difflib
|
|
3
|
+
import io
|
|
4
|
+
import re
|
|
5
|
+
import tokenize
|
|
6
|
+
from pathlib import Path, PurePosixPath
|
|
7
|
+
|
|
8
|
+
from . import cite, corpus, settings, typescript
|
|
9
|
+
from .reach import Reach
|
|
10
|
+
|
|
11
|
+
RECORD = PurePosixPath("requirements") / "uncited.yml"
|
|
12
|
+
|
|
13
|
+
# @req> REQ-27534660@OcqGUyoB-NOR zu7qeg
|
|
14
|
+
MODES = settings.MODES[1:]
|
|
15
|
+
# @req> uncited_code_exemptions@UTuKr6P-FxU0 7z5oma
|
|
16
|
+
EXEMPTIONS = {
|
|
17
|
+
"claude_md": {"files": "any file named CLAUDE.md in any folder"},
|
|
18
|
+
"claude_rules": {"files": "every file under a .claude/rules folder"},
|
|
19
|
+
"gitignore": {"files": "any file named .gitignore in any folder"},
|
|
20
|
+
"license": {"files": "any file whose name begins LICENSE in any folder"},
|
|
21
|
+
"readme": {"files": "any file named README or README.* in any folder"},
|
|
22
|
+
}
|
|
23
|
+
EXEMPT_FILES = {
|
|
24
|
+
"claude_md": re.compile(r"(?:^|/)CLAUDE\.md$"),
|
|
25
|
+
"claude_rules": re.compile(r"(?:^|/)\.claude/rules/"),
|
|
26
|
+
"gitignore": re.compile(r"(?:^|/)\.gitignore$"),
|
|
27
|
+
"license": re.compile(r"(?:^|/)LICENSE[^/]*$"),
|
|
28
|
+
"readme": re.compile(r"(?:^|/)README(?:\.[^/]*)?$"),
|
|
29
|
+
}
|
|
30
|
+
HASHED = (".yml", ".yaml", ".toml", ".sh", ".cfg", ".txt")
|
|
31
|
+
# @req> commentless_formats.json@zjmZ8jE-Au2_ yjmqbb
|
|
32
|
+
COMMENTLESS = (".json",)
|
|
33
|
+
SCRIPT = (("//", None), ("/*", "*/"))
|
|
34
|
+
MARKUP = (("<!--", "-->"),)
|
|
35
|
+
UNREAD = (tokenize.COMMENT, tokenize.NL, tokenize.NEWLINE, tokenize.INDENT,
|
|
36
|
+
tokenize.DEDENT, tokenize.ENDMARKER)
|
|
37
|
+
IMPORT = re.compile(r"^[ \t]*import\b(?!\s*\()[^;]*?(['\"])[^'\"\n]*\1[ \t]*;?",
|
|
38
|
+
re.MULTILINE | re.DOTALL)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _lines(text):
|
|
42
|
+
return io.StringIO(text).readlines()
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _fenced(text, pairs, quoted):
|
|
46
|
+
quiet, inside = set(), None
|
|
47
|
+
for number, line in enumerate(_lines(text), 1):
|
|
48
|
+
code, at = False, 0
|
|
49
|
+
while at < len(line):
|
|
50
|
+
if inside:
|
|
51
|
+
end = line.find(inside, at)
|
|
52
|
+
at, inside = (len(line), inside) if end < 0 else (end + len(inside), None)
|
|
53
|
+
continue
|
|
54
|
+
if quoted and line[at] in "'\"`":
|
|
55
|
+
close = line.find(line[at], at + 1)
|
|
56
|
+
code, at = True, len(line) if close < 0 else close + 1
|
|
57
|
+
continue
|
|
58
|
+
opened = next((pair for pair in pairs if line.startswith(pair[0], at)), None)
|
|
59
|
+
if opened and opened[1] is None:
|
|
60
|
+
break
|
|
61
|
+
if opened:
|
|
62
|
+
inside, at = opened[1], at + len(opened[0])
|
|
63
|
+
continue
|
|
64
|
+
code = code or not line[at].isspace()
|
|
65
|
+
at += 1
|
|
66
|
+
if line.strip() and not code:
|
|
67
|
+
quiet.add(number)
|
|
68
|
+
return quiet
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def _python(text):
|
|
72
|
+
held = set()
|
|
73
|
+
try:
|
|
74
|
+
for token in tokenize.generate_tokens(io.StringIO(text).readline):
|
|
75
|
+
if token.type not in UNREAD:
|
|
76
|
+
held.update(range(token.start[0], token.end[0] + 1))
|
|
77
|
+
tree = ast.parse(text)
|
|
78
|
+
except (tokenize.TokenError, IndentationError, SyntaxError, ValueError):
|
|
79
|
+
return set()
|
|
80
|
+
imports = {line for node in ast.walk(tree) if isinstance(node, (ast.Import, ast.ImportFrom))
|
|
81
|
+
for line in range(node.lineno, (node.end_lineno or node.lineno) + 1)}
|
|
82
|
+
return imports | {number for number, line in enumerate(_lines(text), 1)
|
|
83
|
+
if line.strip() and number not in held}
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _script(text):
|
|
87
|
+
imports = {line for found in IMPORT.finditer(text)
|
|
88
|
+
for line in range(text.count("\n", 0, found.start()) + 1,
|
|
89
|
+
text.count("\n", 0, found.end()) + 2)}
|
|
90
|
+
return imports | _fenced(text, SCRIPT, True)
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
# @req> REQ-58348893@GQ3L2sQGRlkW yvnqdz
|
|
94
|
+
def code_lines(path, text):
|
|
95
|
+
suffix = PurePosixPath(path).suffix
|
|
96
|
+
lines = _lines(text)
|
|
97
|
+
quiet = {number for number, line in enumerate(lines, 1) if not line.strip()}
|
|
98
|
+
if suffix == ".py":
|
|
99
|
+
quiet |= _python(text)
|
|
100
|
+
elif suffix == ".js":
|
|
101
|
+
quiet |= _script(text)
|
|
102
|
+
elif suffix in typescript.SUFFIXES:
|
|
103
|
+
quiet |= typescript.read(suffix, text).imports | cite.remarks(suffix, text)
|
|
104
|
+
elif suffix == ".sql":
|
|
105
|
+
quiet |= cite.remarks(suffix, text)
|
|
106
|
+
elif suffix in (".md", ".html"):
|
|
107
|
+
quiet |= _fenced(text, MARKUP, False)
|
|
108
|
+
close = cite._front(text)
|
|
109
|
+
quiet |= {number for number, line in enumerate(lines, 1)
|
|
110
|
+
if 1 < number < close and line.lstrip().startswith("#")}
|
|
111
|
+
quiet |= {1, close} if close else set()
|
|
112
|
+
elif suffix in HASHED:
|
|
113
|
+
quiet |= {number for number, line in enumerate(lines, 1)
|
|
114
|
+
if line.lstrip().startswith("#")}
|
|
115
|
+
return [number for number in range(1, len(lines) + 1) if number not in quiet]
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
# @req> REQ-97409889@RWCU92P-iWbp 2c3ddu
|
|
119
|
+
def exempt(path):
|
|
120
|
+
return corpus.is_test(path) or any(EXEMPT_FILES[entry].search(path)
|
|
121
|
+
for entry in EXEMPTIONS)
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def unknown_syntax(path):
|
|
125
|
+
return PurePosixPath(path).suffix not in (*cite.COMMENTS, *HASHED, *COMMENTLESS)
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def _moved(path, was, now):
|
|
129
|
+
before, after = code_lines(path, was), code_lines(path, now)
|
|
130
|
+
was_lines, now_lines = _lines(was), _lines(now)
|
|
131
|
+
matcher = difflib.SequenceMatcher(None, [was_lines[n - 1].rstrip("\r\n") for n in before],
|
|
132
|
+
[now_lines[n - 1].rstrip("\r\n") for n in after],
|
|
133
|
+
autojunk=False)
|
|
134
|
+
return {after[at] for tag, _, _, first, last in matcher.get_opcodes() if tag != "equal"
|
|
135
|
+
for at in range(first, last)}
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
# @req> REQ-27097724@WuK_Hn1OEs1s a5rd75
|
|
139
|
+
def _changed(root, base, texts):
|
|
140
|
+
fork = cite._git(root, "merge-base", "HEAD", base).strip()
|
|
141
|
+
named = cite._git(root, "-c", "core.quotePath=false", "diff", "--name-only",
|
|
142
|
+
"--no-renames", "-z", fork, "--").split("\0")
|
|
143
|
+
named += cite._git(root, "ls-files", "--others", "--exclude-standard", "-z").split("\0")
|
|
144
|
+
held = set(filter(None, cite._git(root, "ls-tree", "-r", "--name-only", "-z",
|
|
145
|
+
fork).split("\0")))
|
|
146
|
+
changed = {}
|
|
147
|
+
for path in filter(None, named):
|
|
148
|
+
if texts.get(path) is None:
|
|
149
|
+
continue
|
|
150
|
+
was = cite._git(root, "show", f"{fork}:{path}") if path in held else ""
|
|
151
|
+
changed[path] = _moved(path, was, texts[path])
|
|
152
|
+
return changed
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def _spans(numbers):
|
|
156
|
+
spans = []
|
|
157
|
+
for number in numbers:
|
|
158
|
+
if spans and spans[-1][1] == number - 1:
|
|
159
|
+
spans[-1][1] = number
|
|
160
|
+
else:
|
|
161
|
+
spans.append([number, number])
|
|
162
|
+
return spans
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def _remedy(path, first, last):
|
|
166
|
+
if PurePosixPath(path).suffix in cite.COMMENTS:
|
|
167
|
+
return f"cite it with reqctl tag {path} --from {first} --to {last} --req UID"
|
|
168
|
+
return f"have the item governing it hold the whole file with reqctl tag {path} --req UID"
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def _uncited(tree, root, citations):
|
|
172
|
+
texts = {path.replace("\\", "/"): text for path, text in cite._sources(root)}
|
|
173
|
+
reach = Reach(texts)
|
|
174
|
+
covered = {(citation["path"], line) for citation in citations
|
|
175
|
+
for line in citation["lines"]}
|
|
176
|
+
for citation in citations:
|
|
177
|
+
covered |= reach.reached(citation["path"], citation["lines"])
|
|
178
|
+
whole = {path for item in corpus.items(tree)
|
|
179
|
+
for path in corpus.mapping(item.data, corpus.WHOLE_FILES)}
|
|
180
|
+
return texts, {path: [line for line in code_lines(path, text) if (path, line) not in covered]
|
|
181
|
+
for path, text in sorted(texts.items())
|
|
182
|
+
if text is not None and path not in whole and not exempt(path)}
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def counts(tree, root, citations):
|
|
186
|
+
return {path: len(lines) for path, lines in _uncited(tree, root, citations)[1].items()
|
|
187
|
+
if lines}
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
def recorded(root):
|
|
191
|
+
path = Path(root) / RECORD
|
|
192
|
+
if not path.exists():
|
|
193
|
+
return {}
|
|
194
|
+
held = corpus.read(path)
|
|
195
|
+
if held is None:
|
|
196
|
+
return {}
|
|
197
|
+
if not isinstance(held, dict) or not all(
|
|
198
|
+
isinstance(key, str) and type(count) is int and count > 0
|
|
199
|
+
for key, count in held.items()):
|
|
200
|
+
raise corpus.ReqctlError(f"{RECORD.as_posix()} is not a mapping of file paths to "
|
|
201
|
+
"counts of at least one -- record the uncited counts "
|
|
202
|
+
"with reqctl trace --record")
|
|
203
|
+
return held
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
# @req> REQ-21635723@CvxpeHCRTgGz arkc4u
|
|
207
|
+
def record(tree, root, citations):
|
|
208
|
+
corpus.write_all([(Path(root) / RECORD, corpus.dump(counts(tree, root, citations)))])
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def _ratchet(root, uncited):
|
|
212
|
+
held, found = recorded(root), []
|
|
213
|
+
# @req> REQ-91618546@X04wZs4bpj5D 7ckhhs
|
|
214
|
+
for path, lines in uncited.items():
|
|
215
|
+
if len(lines) > (was := held.get(path, 0)):
|
|
216
|
+
found.append(f"{path}: {len(lines)} uncited line(s), above the "
|
|
217
|
+
f"{was} {RECORD.as_posix()} states -- cite them with "
|
|
218
|
+
f"reqctl tag {path} --from N --to M --req UID, or delete them")
|
|
219
|
+
# @req> REQ-56849355@Ko7rmWqVFjxy vwtwkp
|
|
220
|
+
for path, count in sorted(held.items()):
|
|
221
|
+
if count > (now := len(uncited.get(path, ()))):
|
|
222
|
+
found.append(f"{path}: {RECORD.as_posix()} states {count} uncited line(s), above "
|
|
223
|
+
f"the {now} the file holds -- record the "
|
|
224
|
+
"uncited counts with reqctl trace --record")
|
|
225
|
+
return found
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
# @req> REQ-27534660@OcqGUyoB-NOR tml2jy
|
|
229
|
+
def problems(tree, root, citations, base=None):
|
|
230
|
+
mode = settings.table(settings.read(root)[1], "uncited_code")["check"]["mode"]
|
|
231
|
+
if mode not in MODES or (mode == "changed_lines" and not base):
|
|
232
|
+
return []
|
|
233
|
+
texts, uncited = _uncited(tree, root, citations)
|
|
234
|
+
# @req> REQ-91618546@X04wZs4bpj5D u5c2ay
|
|
235
|
+
# @req> REQ-56849355@Ko7rmWqVFjxy m4cztr
|
|
236
|
+
if mode == "ratchet":
|
|
237
|
+
return _ratchet(root, uncited)
|
|
238
|
+
# @req> REQ-27097724@WuK_Hn1OEs1s jo2uxv
|
|
239
|
+
changed = _changed(root, base, texts) if mode == "changed_lines" else None
|
|
240
|
+
found = []
|
|
241
|
+
for path, uncovered in uncited.items():
|
|
242
|
+
# @req> REQ-27097724@WuK_Hn1OEs1s x7jy55
|
|
243
|
+
if changed is not None:
|
|
244
|
+
uncovered = [line for line in uncovered if line in changed.get(path, ())]
|
|
245
|
+
# @req+ REQ-33946954@dE-TCnSywwn3 u54yfr
|
|
246
|
+
# @req+ REQ-46463201@KQkxvX2fdJ1W aoss24
|
|
247
|
+
if uncovered and unknown_syntax(path):
|
|
248
|
+
found.append(f"{path}: reqctl knows no comment syntax for this file -- have the "
|
|
249
|
+
f"item governing it hold the whole file with reqctl tag {path} "
|
|
250
|
+
"--req UID, or add its comment syntax to reqctl")
|
|
251
|
+
continue
|
|
252
|
+
# @req- aoss24
|
|
253
|
+
# @req- u54yfr
|
|
254
|
+
for first, last in _spans(uncovered):
|
|
255
|
+
found.append(f"{path}:{first}-{last}: no statement citation covers this code "
|
|
256
|
+
f"-- {_remedy(path, first, last)}, or delete it")
|
|
257
|
+
return found
|