falsegreen-robot 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- falsegreen_robot/__init__.py +3 -0
- falsegreen_robot/__main__.py +6 -0
- falsegreen_robot/scanner.py +408 -0
- falsegreen_robot-0.1.0.dist-info/METADATA +116 -0
- falsegreen_robot-0.1.0.dist-info/RECORD +8 -0
- falsegreen_robot-0.1.0.dist-info/WHEEL +4 -0
- falsegreen_robot-0.1.0.dist-info/entry_points.txt +2 -0
- falsegreen_robot-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,408 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
# -*- coding: utf-8 -*-
|
|
3
|
+
"""
|
|
4
|
+
falsegreen-robot: deterministic false-positive scanner for Robot Framework tests.
|
|
5
|
+
|
|
6
|
+
Parses .robot files with the official Robot Framework parser (robot.api.get_model)
|
|
7
|
+
- no execution - and flags test cases that pass green without protecting anything:
|
|
8
|
+
a test with no verification keyword, a swallowed Run Keyword And Ignore Error, an
|
|
9
|
+
always-true Should Be True ${TRUE}, a self-compare, Sleep used as a wait, a skipped
|
|
10
|
+
test. Sibling of falsegreen (Python/pytest) and falsegreen-js (JS/TS).
|
|
11
|
+
|
|
12
|
+
Output: readable text (default) or JSON (--json).
|
|
13
|
+
Exit: 0 clean, 10 low-confidence only, 20 high-confidence present.
|
|
14
|
+
"""
|
|
15
|
+
import argparse
|
|
16
|
+
import json
|
|
17
|
+
import os
|
|
18
|
+
import sys
|
|
19
|
+
|
|
20
|
+
__version__ = "0.1.0"
|
|
21
|
+
TOOL_URI = "https://github.com/vinicq/falsegreen-robot"
|
|
22
|
+
|
|
23
|
+
# --- case catalog. code -> (title, confidence, judgment J1-J6) -------------
|
|
24
|
+
JUDGMENTS = {
|
|
25
|
+
"J1": "does the verification run?",
|
|
26
|
+
"J2": "is the oracle independent of the code?",
|
|
27
|
+
"J4": "does it check enough, and the right thing?",
|
|
28
|
+
"J5": "is it coupled / hard to maintain?",
|
|
29
|
+
}
|
|
30
|
+
CASES = {
|
|
31
|
+
"C2": ("empty test case (no keywords run)", "high", "J1"),
|
|
32
|
+
"C2b": ("runs keywords but no verification keyword (no oracle)", "low", "J1"),
|
|
33
|
+
"C3": ("Run Keyword And Ignore Error/Return Status swallows the failure and the status is never asserted", "high", "J1"),
|
|
34
|
+
"C5": ("always-true check (Should Be True ${TRUE} / Should Be Equal with equal literals)", "high", "J2"),
|
|
35
|
+
"C7": ("self-compare (Should Be Equal ${x} ${x})", "high", "J2"),
|
|
36
|
+
"C16": ("Sleep used as synchronization (result depends on timing)", "low", "J1"),
|
|
37
|
+
"C21": ("verification only runs conditionally (inside IF / Run Keyword If) — it may never execute", "low", "J1"),
|
|
38
|
+
"C32": ("skipped test (robot:skip / Skip) never runs", "low", "J1"),
|
|
39
|
+
"R1": ("Pass Execution forces the test to pass regardless of any check (forced green)", "high", "J1"),
|
|
40
|
+
"R2": ("user keyword named like a verifier (Verify/Assert/Should...) but its body contains no verification — a hollow oracle", "low", "J1"),
|
|
41
|
+
# --- diagnostic group (maintainability; default off, opt-in via --diagnostics) ---
|
|
42
|
+
"D2": ("control flow (IF/FOR/WHILE/TRY) at the test/task level — the guide advises against it", "off", "J4"),
|
|
43
|
+
# --- coupling group (structure; default off, opt-in) ----------------------
|
|
44
|
+
"M2": ("test/task has too many steps (the guide suggests max ~10)", "off", "J5"),
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
# Default thresholds for the opt-in groups (overridable later via config).
|
|
48
|
+
DIAGNOSTIC_THRESHOLDS = {"long_test_steps": 10}
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def group_of(code):
|
|
52
|
+
"""false-positive (C*/R*) / diagnostic (D*) / coupling (M*) — mirrors the siblings."""
|
|
53
|
+
if code.startswith("D"):
|
|
54
|
+
return "diagnostic"
|
|
55
|
+
if code.startswith("M"):
|
|
56
|
+
return "coupling"
|
|
57
|
+
return "false-positive"
|
|
58
|
+
|
|
59
|
+
# --- verification vocabulary (the oracle), across Robot libraries ----------
|
|
60
|
+
# Dominant convention: the word "Should". Plus library-specific forms.
|
|
61
|
+
REST_SCHEMA = {"Integer", "Number", "String", "Boolean", "Object", "Array", "Null", "Missing"}
|
|
62
|
+
BROWSER_OPS = {"==", "!=", "contains", "not contains", "validate", "matches",
|
|
63
|
+
">", "<", ">=", "<=", "*=", "^=", "$=", "then"}
|
|
64
|
+
VERIFY_PREFIXES = ("verify", "assert", "validate", "check ")
|
|
65
|
+
SWALLOW_KEYWORDS = {"run keyword and ignore error", "run keyword and return status"}
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _norm(name):
|
|
69
|
+
return (name or "").strip().lower()
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def is_verification(keyword, args):
|
|
73
|
+
"""True if this keyword call verifies an expected result (is an oracle)."""
|
|
74
|
+
if keyword is None:
|
|
75
|
+
return False
|
|
76
|
+
n = _norm(keyword)
|
|
77
|
+
if "should" in n:
|
|
78
|
+
return True # BuiltIn/Collections/String/Selenium/...
|
|
79
|
+
if keyword in REST_SCHEMA:
|
|
80
|
+
return True # RESTinstance schema assertions
|
|
81
|
+
if n.startswith(VERIFY_PREFIXES):
|
|
82
|
+
return True # custom Verify*/Assert*/Validate*/Check *
|
|
83
|
+
if n.startswith("wait until") and any(w in n for w in ("contain", "visible", "present")):
|
|
84
|
+
return True # Selenium/Appium waits that fail on timeout
|
|
85
|
+
if n.startswith("get ") and any(a in BROWSER_OPS for a in (args or ())):
|
|
86
|
+
return True # Browser assertion engine: Get ... == expected
|
|
87
|
+
return False
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def is_swallow(keyword):
|
|
91
|
+
return _norm(keyword) in SWALLOW_KEYWORDS
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _looks_constant_true(arg):
|
|
95
|
+
return _norm(arg) in ("${true}", "true", "1", "${1}")
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
# --- AST walk over the Robot model -----------------------------------------
|
|
99
|
+
def _keyword_calls(node):
|
|
100
|
+
"""Yield every KeywordCall in a block, descending into IF/FOR/TRY/WHILE."""
|
|
101
|
+
for item in getattr(node, "body", None) or []:
|
|
102
|
+
cls = type(item).__name__
|
|
103
|
+
if cls == "KeywordCall":
|
|
104
|
+
yield item
|
|
105
|
+
if hasattr(item, "body"):
|
|
106
|
+
yield from _keyword_calls(item)
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def _top_level_keyword_calls(testcase):
|
|
110
|
+
"""KeywordCalls directly in the test body (not nested in IF/FOR/TRY/WHILE)."""
|
|
111
|
+
return [i for i in (getattr(testcase, "body", None) or [])
|
|
112
|
+
if type(i).__name__ == "KeywordCall"]
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def _has_control_block(testcase):
|
|
116
|
+
return any(type(i).__name__ in ("If", "For", "While", "Try")
|
|
117
|
+
for i in (getattr(testcase, "body", None) or []))
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def _rkif_verifies(call):
|
|
121
|
+
"""A `Run Keyword If`/`Unless` whose arguments name a verification keyword -
|
|
122
|
+
that is a conditional verification (may not run)."""
|
|
123
|
+
if _norm(getattr(call, "keyword", None)) in ("run keyword if", "run keyword unless"):
|
|
124
|
+
return any("should" in _norm(a) for a in (getattr(call, "args", None) or ()))
|
|
125
|
+
return False
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def _try_blocks(node):
|
|
129
|
+
"""Yield native TRY blocks (RF 5+) anywhere in the test body."""
|
|
130
|
+
for item in getattr(node, "body", None) or []:
|
|
131
|
+
if type(item).__name__ == "Try" and _norm(getattr(item, "type", "")) == "try":
|
|
132
|
+
yield item
|
|
133
|
+
if hasattr(item, "body"):
|
|
134
|
+
yield from _try_blocks(item)
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def _except_swallows(try_node):
|
|
138
|
+
"""True if any EXCEPT branch of this TRY swallows the failure: its body has no
|
|
139
|
+
Fail, no verification keyword, and no re-raise - only logging / no-op / empty."""
|
|
140
|
+
branch = getattr(try_node, "next", None)
|
|
141
|
+
while branch is not None:
|
|
142
|
+
if _norm(getattr(branch, "type", "")) == "except":
|
|
143
|
+
harmless = True
|
|
144
|
+
for st in getattr(branch, "body", None) or []:
|
|
145
|
+
if type(st).__name__ != "KeywordCall":
|
|
146
|
+
continue
|
|
147
|
+
kw = _norm(st.keyword)
|
|
148
|
+
if kw in ("fail", "fatal error") or "should" in kw or kw.startswith(VERIFY_PREFIXES):
|
|
149
|
+
harmless = False
|
|
150
|
+
break
|
|
151
|
+
if harmless:
|
|
152
|
+
return True
|
|
153
|
+
branch = getattr(branch, "next", None)
|
|
154
|
+
return False
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def _try_body_has_keyword(try_node):
|
|
158
|
+
return any(type(st).__name__ == "KeywordCall" for st in (getattr(try_node, "body", None) or []))
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def _tags(testcase):
|
|
162
|
+
tags = []
|
|
163
|
+
for item in getattr(testcase, "body", None) or []:
|
|
164
|
+
if type(item).__name__ == "Tags":
|
|
165
|
+
tags += [str(v) for v in getattr(item, "values", []) or []]
|
|
166
|
+
return tags
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
class Finding:
|
|
170
|
+
__slots__ = ("file", "line", "test", "code", "detail")
|
|
171
|
+
|
|
172
|
+
def __init__(self, file, line, test, code, detail=""):
|
|
173
|
+
self.file = file
|
|
174
|
+
self.line = line
|
|
175
|
+
self.test = test
|
|
176
|
+
self.code = code
|
|
177
|
+
self.detail = detail
|
|
178
|
+
|
|
179
|
+
def dict(self):
|
|
180
|
+
title, conf, judg = CASES[self.code]
|
|
181
|
+
return {"file": self.file, "line": self.line, "test": self.test,
|
|
182
|
+
"code": self.code, "confidence": conf, "judgment": judg,
|
|
183
|
+
"title": title, "detail": self.detail}
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def _name_implies_verification(name):
|
|
187
|
+
"""A user-keyword name that promises to verify. 'Check' is excluded - it often
|
|
188
|
+
names a getter that returns a status for the caller to assert."""
|
|
189
|
+
n = _norm(name)
|
|
190
|
+
return "should" in n or n.startswith(("verify", "assert", "validate"))
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
def _call_level_smells(file, owner, calls, findings):
|
|
194
|
+
"""Per-call false-green checks shared by test cases, tasks, and user keywords:
|
|
195
|
+
C5 (always-true), C7 (self-compare), C16 (Sleep). Returns whether any keyword
|
|
196
|
+
call verifies something."""
|
|
197
|
+
has_verification = False
|
|
198
|
+
for c in calls:
|
|
199
|
+
kw, args = c.keyword, list(getattr(c, "args", []) or [])
|
|
200
|
+
ln = getattr(c, "lineno", 0) or 0
|
|
201
|
+
if _norm(kw) == "should be true" and args and _looks_constant_true(args[0]):
|
|
202
|
+
findings.append(Finding(file, ln, owner, "C5", "Should Be True on a constant"))
|
|
203
|
+
has_verification = True
|
|
204
|
+
continue
|
|
205
|
+
if _norm(kw) == "should be equal" and len(args) >= 2:
|
|
206
|
+
if args[0] == args[1]:
|
|
207
|
+
code = "C7" if args[0].startswith("${") else "C5"
|
|
208
|
+
findings.append(Finding(file, ln, owner, code, "both sides are identical"))
|
|
209
|
+
has_verification = True
|
|
210
|
+
continue
|
|
211
|
+
if _norm(kw) == "sleep":
|
|
212
|
+
findings.append(Finding(file, ln, owner, "C16"))
|
|
213
|
+
if is_verification(kw, args) or _rkif_verifies(c):
|
|
214
|
+
has_verification = True
|
|
215
|
+
return has_verification
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
def analyze_keyword(file, kw, findings):
|
|
219
|
+
"""Analyze a User Keyword definition (.robot Keywords section or .resource).
|
|
220
|
+
Flags call-level smells inside the body, and R2 when the keyword is named like a
|
|
221
|
+
verifier but verifies nothing (a hollow oracle used by tests)."""
|
|
222
|
+
name = getattr(kw, "name", "") or ""
|
|
223
|
+
line = getattr(kw, "lineno", 0) or 0
|
|
224
|
+
calls = list(_keyword_calls(kw))
|
|
225
|
+
has_verification = _call_level_smells(file, name, calls, findings)
|
|
226
|
+
if _name_implies_verification(name) and not has_verification:
|
|
227
|
+
findings.append(Finding(file, line, name, "R2"))
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
def analyze_testcase(file, tc, findings):
|
|
231
|
+
name = getattr(tc, "name", "") or ""
|
|
232
|
+
line = getattr(tc, "lineno", 0) or 0
|
|
233
|
+
calls = list(_keyword_calls(tc))
|
|
234
|
+
tags = [_norm(t) for t in _tags(tc)]
|
|
235
|
+
|
|
236
|
+
# C32: skipped
|
|
237
|
+
if any("robot:skip" in t for t in tags) or any(_norm(c.keyword) in ("skip",) for c in calls):
|
|
238
|
+
findings.append(Finding(file, line, name, "C32"))
|
|
239
|
+
return
|
|
240
|
+
|
|
241
|
+
# R1: Pass Execution at the top level forces the test green regardless of checks
|
|
242
|
+
if any(_norm(c.keyword) == "pass execution" for c in _top_level_keyword_calls(tc)):
|
|
243
|
+
findings.append(Finding(file, line, name, "R1"))
|
|
244
|
+
return
|
|
245
|
+
|
|
246
|
+
# C2: empty (no keyword calls at all)
|
|
247
|
+
if not calls:
|
|
248
|
+
findings.append(Finding(file, line, name, "C2"))
|
|
249
|
+
return
|
|
250
|
+
|
|
251
|
+
has_verification = _call_level_smells(file, name, calls, findings)
|
|
252
|
+
|
|
253
|
+
# diagnostic/coupling group (off by default; emitted always, filtered in scan)
|
|
254
|
+
if _has_control_block(tc):
|
|
255
|
+
findings.append(Finding(file, line, name, "D2"))
|
|
256
|
+
if len(calls) > DIAGNOSTIC_THRESHOLDS["long_test_steps"]:
|
|
257
|
+
findings.append(Finding(file, line, name, "M2", "%d steps" % len(calls)))
|
|
258
|
+
|
|
259
|
+
# C3: native TRY/EXCEPT whose EXCEPT swallows the failure
|
|
260
|
+
for tb in _try_blocks(tc):
|
|
261
|
+
if _try_body_has_keyword(tb) and _except_swallows(tb):
|
|
262
|
+
findings.append(Finding(file, getattr(tb, "lineno", line), name, "C3",
|
|
263
|
+
"TRY failure is swallowed by EXCEPT"))
|
|
264
|
+
return
|
|
265
|
+
|
|
266
|
+
# C3: swallow without an asserted status
|
|
267
|
+
if not has_verification and any(is_swallow(c.keyword) for c in calls):
|
|
268
|
+
findings.append(Finding(file, line, name, "C3"))
|
|
269
|
+
return
|
|
270
|
+
|
|
271
|
+
# C2b: keywords ran but nothing verified
|
|
272
|
+
if not has_verification:
|
|
273
|
+
findings.append(Finding(file, line, name, "C2b"))
|
|
274
|
+
return
|
|
275
|
+
|
|
276
|
+
# C21: a verification exists, but none runs unconditionally — the only
|
|
277
|
+
# verification lives inside an IF/FOR block or a Run Keyword If. The guide is
|
|
278
|
+
# explicit: no if/else/for at the test-case level.
|
|
279
|
+
top = _top_level_keyword_calls(tc)
|
|
280
|
+
has_unconditional = any(
|
|
281
|
+
is_verification(c.keyword, list(getattr(c, "args", []) or []))
|
|
282
|
+
for c in top
|
|
283
|
+
if _norm(c.keyword) not in ("run keyword if", "run keyword unless")
|
|
284
|
+
)
|
|
285
|
+
if not has_unconditional and (_has_control_block(tc) or any(_rkif_verifies(c) for c in calls)):
|
|
286
|
+
findings.append(Finding(file, line, name, "C21"))
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
def analyze_file(path):
|
|
290
|
+
findings = []
|
|
291
|
+
try:
|
|
292
|
+
from robot.api import get_model
|
|
293
|
+
from robot.parsing import ModelVisitor
|
|
294
|
+
except Exception as exc: # pragma: no cover
|
|
295
|
+
sys.stderr.write("falsegreen-robot: robotframework is required (%s)\n" % exc)
|
|
296
|
+
return findings
|
|
297
|
+
try:
|
|
298
|
+
model = get_model(path)
|
|
299
|
+
except Exception:
|
|
300
|
+
return findings
|
|
301
|
+
self_findings = findings
|
|
302
|
+
|
|
303
|
+
class _V(ModelVisitor):
|
|
304
|
+
def visit_TestCase(self, node):
|
|
305
|
+
analyze_testcase(path, node, self_findings)
|
|
306
|
+
|
|
307
|
+
def visit_Task(self, node): # RPA suites use *** Tasks ***, not *** Test Cases ***
|
|
308
|
+
analyze_testcase(path, node, self_findings)
|
|
309
|
+
|
|
310
|
+
def visit_Keyword(self, node): # user keyword defs in .robot Keywords + .resource
|
|
311
|
+
analyze_keyword(path, node, self_findings)
|
|
312
|
+
|
|
313
|
+
_V().visit(model)
|
|
314
|
+
return findings
|
|
315
|
+
|
|
316
|
+
|
|
317
|
+
# --- discovery + CLI -------------------------------------------------------
|
|
318
|
+
IGNORED_DIRS = {".git", ".tox", "venv", ".venv", "node_modules", "results", "output"}
|
|
319
|
+
|
|
320
|
+
|
|
321
|
+
def is_robot_file(path):
|
|
322
|
+
return path.endswith((".robot", ".resource"))
|
|
323
|
+
|
|
324
|
+
|
|
325
|
+
def discover(paths):
|
|
326
|
+
files = []
|
|
327
|
+
for root in paths:
|
|
328
|
+
if os.path.isfile(root):
|
|
329
|
+
if is_robot_file(root):
|
|
330
|
+
files.append(root)
|
|
331
|
+
continue
|
|
332
|
+
for dirpath, dirnames, filenames in os.walk(root):
|
|
333
|
+
dirnames[:] = [d for d in dirnames if d not in IGNORED_DIRS]
|
|
334
|
+
for f in filenames:
|
|
335
|
+
if is_robot_file(f):
|
|
336
|
+
files.append(os.path.join(dirpath, f))
|
|
337
|
+
return sorted(set(files))
|
|
338
|
+
|
|
339
|
+
|
|
340
|
+
def _eff_conf(code):
|
|
341
|
+
"""Effective confidence: an off-by-default (D*/M*) code shows as 'low' when enabled."""
|
|
342
|
+
c = CASES[code][1]
|
|
343
|
+
return "low" if c == "off" else c
|
|
344
|
+
|
|
345
|
+
|
|
346
|
+
def scan(paths, disable=None, diagnostics=False):
|
|
347
|
+
disable = disable or set()
|
|
348
|
+
out = []
|
|
349
|
+
for f in discover(paths):
|
|
350
|
+
for finding in analyze_file(f):
|
|
351
|
+
conf = CASES[finding.code][1]
|
|
352
|
+
if finding.code in disable:
|
|
353
|
+
continue
|
|
354
|
+
if conf == "off" and not diagnostics:
|
|
355
|
+
continue
|
|
356
|
+
out.append(finding)
|
|
357
|
+
return out
|
|
358
|
+
|
|
359
|
+
|
|
360
|
+
def _render_text(findings):
|
|
361
|
+
if not findings:
|
|
362
|
+
return "falsegreen-robot: no false-positive patterns found."
|
|
363
|
+
lines, high, low = [], 0, 0
|
|
364
|
+
by_file = {}
|
|
365
|
+
for f in findings:
|
|
366
|
+
by_file.setdefault(f.file, []).append(f)
|
|
367
|
+
for file, fs in by_file.items():
|
|
368
|
+
lines.append("\n" + file)
|
|
369
|
+
for f in sorted(fs, key=lambda x: x.line):
|
|
370
|
+
title = CASES[f.code][0]
|
|
371
|
+
conf = _eff_conf(f.code)
|
|
372
|
+
tag = "HIGH" if conf == "high" else "low "
|
|
373
|
+
high += conf == "high"
|
|
374
|
+
low += conf == "low"
|
|
375
|
+
lines.append(" %s %-4s L%-4d %s %s" % (tag, f.code, f.line, f.test, title))
|
|
376
|
+
if f.detail:
|
|
377
|
+
lines.append(" " + f.detail)
|
|
378
|
+
lines.append("\n%d high, %d low. %s" % (high, low, TOOL_URI))
|
|
379
|
+
return "\n".join(lines)
|
|
380
|
+
|
|
381
|
+
|
|
382
|
+
def main(argv=None):
|
|
383
|
+
p = argparse.ArgumentParser(prog="falsegreen-robot",
|
|
384
|
+
description="Find false-positive Robot Framework tests (static).")
|
|
385
|
+
p.add_argument("paths", nargs="*", default=["."], help="files or directories (default: cwd)")
|
|
386
|
+
p.add_argument("--json", action="store_true", help="JSON output")
|
|
387
|
+
p.add_argument("--disable", default="", help="comma-separated codes to turn off")
|
|
388
|
+
p.add_argument("--diagnostics", action="store_true",
|
|
389
|
+
help="also report the opt-in maintainability group (D*/M*)")
|
|
390
|
+
p.add_argument("--version", action="version", version=__version__)
|
|
391
|
+
args = p.parse_args(argv)
|
|
392
|
+
disable = {c.strip() for c in args.disable.split(",") if c.strip()}
|
|
393
|
+
findings = scan(args.paths or ["."], disable=disable, diagnostics=args.diagnostics)
|
|
394
|
+
if args.json:
|
|
395
|
+
print(json.dumps({"tool": "falsegreen-robot", "version": __version__,
|
|
396
|
+
"judgments": JUDGMENTS,
|
|
397
|
+
"findings": [f.dict() for f in findings]}, indent=2))
|
|
398
|
+
else:
|
|
399
|
+
print(_render_text(findings))
|
|
400
|
+
if any(_eff_conf(f.code) == "high" for f in findings):
|
|
401
|
+
return 20
|
|
402
|
+
if findings:
|
|
403
|
+
return 10
|
|
404
|
+
return 0
|
|
405
|
+
|
|
406
|
+
|
|
407
|
+
if __name__ == "__main__":
|
|
408
|
+
sys.exit(main())
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: falsegreen-robot
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Find Robot Framework tests that pass green without protecting anything: tests with no verification keyword, swallowed failures, always-true checks. Deterministic static scan, sibling of falsegreen (Python) and falsegreen-js (JS/TS).
|
|
5
|
+
Project-URL: Homepage, https://github.com/vinicq/falsegreen-robot
|
|
6
|
+
Project-URL: Issues, https://github.com/vinicq/falsegreen-robot/issues
|
|
7
|
+
Author: Vinicius Queiroz
|
|
8
|
+
License-Expression: MIT
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Keywords: code-quality,false-positive,robot-framework,robotframework,static-analysis,test-smells,testing
|
|
11
|
+
Classifier: Development Status :: 3 - Alpha
|
|
12
|
+
Classifier: Framework :: Robot Framework
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Topic :: Software Development :: Quality Assurance
|
|
16
|
+
Classifier: Topic :: Software Development :: Testing
|
|
17
|
+
Requires-Python: >=3.8
|
|
18
|
+
Requires-Dist: robotframework>=4.0
|
|
19
|
+
Provides-Extra: dev
|
|
20
|
+
Requires-Dist: pytest>=7; extra == 'dev'
|
|
21
|
+
Requires-Dist: ruff>=0.5; extra == 'dev'
|
|
22
|
+
Description-Content-Type: text/markdown
|
|
23
|
+
|
|
24
|
+
# falsegreen-robot
|
|
25
|
+
|
|
26
|
+
**One problem, one tool: the false positive.** falsegreen-robot finds Robot Framework
|
|
27
|
+
tests that pass green without protecting anything - tests that let broken behavior
|
|
28
|
+
through because no keyword verifies anything, the failure is swallowed, the check is
|
|
29
|
+
always true, or the test is skipped.
|
|
30
|
+
|
|
31
|
+
Deterministic static scan over the official Robot Framework parser
|
|
32
|
+
(`robot.api.get_model`) - no execution. Sibling of
|
|
33
|
+
[falsegreen](https://github.com/vinicq/falsegreen) (Python/pytest) and
|
|
34
|
+
[falsegreen-js](https://github.com/vinicq/falsegreen-js) (JS/TS). The semantic,
|
|
35
|
+
intent-based pass lives in [falsegreen-skill](https://github.com/vinicq/falsegreen-skill).
|
|
36
|
+
|
|
37
|
+
## Why
|
|
38
|
+
|
|
39
|
+
A green Robot suite is not proof of correctness. A test case can run keywords and never
|
|
40
|
+
call a verification keyword; a `Run Keyword And Ignore Error` can absorb the failure; a
|
|
41
|
+
`Should Be True ${TRUE}` can never fail. This tool flags the patterns a parser can
|
|
42
|
+
prove, before they reach review.
|
|
43
|
+
|
|
44
|
+
## Install
|
|
45
|
+
|
|
46
|
+
```bash
|
|
47
|
+
pip install falsegreen-robot
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
## Usage
|
|
51
|
+
|
|
52
|
+
```bash
|
|
53
|
+
falsegreen-robot # scan cwd
|
|
54
|
+
falsegreen-robot tests/ # scan a path
|
|
55
|
+
falsegreen-robot --json # machine-readable output
|
|
56
|
+
falsegreen-robot --disable C16 # turn off specific codes
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
Exit code: `0` clean, `10` low-confidence only, `20` high-confidence present. Wire exit
|
|
60
|
+
`20` into CI to block the merge.
|
|
61
|
+
|
|
62
|
+
## What it detects
|
|
63
|
+
|
|
64
|
+
The oracle in Robot is the **verification keyword**. The scanner recognizes them across
|
|
65
|
+
libraries (the `Should` convention plus library-specific forms: SeleniumLibrary
|
|
66
|
+
`Element Should Be Visible`, Browser's assertion engine `Get Text sel == expected`,
|
|
67
|
+
RESTinstance schema keywords, DatabaseLibrary `Row Count Should Be Equal`, custom
|
|
68
|
+
`Verify*`/`Assert*` keywords). A test with none of them verifies nothing.
|
|
69
|
+
|
|
70
|
+
| Code | Confidence | What it flags |
|
|
71
|
+
|---|---|---|
|
|
72
|
+
| C2 | high | empty test case (no keywords run) |
|
|
73
|
+
| C2b | low | runs keywords but no verification keyword (no oracle) |
|
|
74
|
+
| C3 | high | `Run Keyword And Ignore Error`/`Return Status` swallows the failure, status never asserted |
|
|
75
|
+
| C5 | high | always-true (`Should Be True ${TRUE}`, `Should Be Equal` with equal literals) |
|
|
76
|
+
| C7 | high | self-compare (`Should Be Equal ${x} ${x}`) |
|
|
77
|
+
| C16 | low | `Sleep` used as synchronization (timing dependence) |
|
|
78
|
+
| C21 | low | verification only runs conditionally (inside `IF` / `Run Keyword If`) — it may never execute |
|
|
79
|
+
| C32 | low | skipped test (`robot:skip` / `Skip`) |
|
|
80
|
+
| R1 | high | `Pass Execution` forces the test green regardless of any check |
|
|
81
|
+
| R2 | low | user keyword named like a verifier (`Verify`/`Assert`/`Should`...) but its body verifies nothing — a hollow oracle |
|
|
82
|
+
|
|
83
|
+
Scans `*** Test Cases ***`, `*** Tasks ***` (RPA), and `*** Keywords ***` definitions in
|
|
84
|
+
both `.robot` and `.resource` files. R2 catches the root cause of a missed C2b: a test
|
|
85
|
+
calls `Verify Login` and looks protected, but that keyword never asserts anything.
|
|
86
|
+
|
|
87
|
+
### Opt-in: maintainability group (default off)
|
|
88
|
+
|
|
89
|
+
Not false-green - the test still verifies - so off by default. Enable with `--diagnostics`.
|
|
90
|
+
Three groups, mirroring `falsegreen` and `falsegreen-js`: `false-positive` (C*/R*, on),
|
|
91
|
+
`diagnostic` (D*, opt-in), `coupling` (M*, opt-in).
|
|
92
|
+
|
|
93
|
+
| Code | Group | What it flags |
|
|
94
|
+
|---|---|---|
|
|
95
|
+
| D2 | diagnostic | control flow (`IF`/`FOR`/`WHILE`/`TRY`) at the test/task level (the guide advises against it) |
|
|
96
|
+
| M2 | coupling | test/task with too many steps (guide suggests max ~10) |
|
|
97
|
+
|
|
98
|
+
```bash
|
|
99
|
+
falsegreen-robot --diagnostics # include D*/M* as warnings
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
Codes share ids with the sibling scanners where the concept matches (C2/C2b/C3/C5/C7/C16/C21/C32).
|
|
103
|
+
A Browser `Get` keyword with no assertion operator is a plain getter, so a test whose only
|
|
104
|
+
step is `Get Text h1` surfaces as no-verification (C2b).
|
|
105
|
+
|
|
106
|
+
## Scope and honesty
|
|
107
|
+
|
|
108
|
+
Static scan: it owns what the keyword structure proves. It does not run the suite, so it
|
|
109
|
+
cannot see runtime-only smells (Test Run War, order dependence across suites). Whether the
|
|
110
|
+
expected value contradicts the intended behavior is semantic and belongs to
|
|
111
|
+
`falsegreen-skill`. Precision over recall: `C2b` is low-confidence because a custom keyword
|
|
112
|
+
may assert internally without `Should` in its name.
|
|
113
|
+
|
|
114
|
+
## License
|
|
115
|
+
|
|
116
|
+
MIT, Vinicius Queiroz.
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
falsegreen_robot/__init__.py,sha256=SjrmpczQ08XrcIMSAgUVMqG8uHdq6rLqjaSilsC_1sA,134
|
|
2
|
+
falsegreen_robot/__main__.py,sha256=n_pUpMzyNzzyqoBaKbHVgLI1lbDPImYhIENNulBOZPY,87
|
|
3
|
+
falsegreen_robot/scanner.py,sha256=WHEbaKaT6t0buEm-MkMNkyZqpJCSMqka0RmBu2FjAYM,16506
|
|
4
|
+
falsegreen_robot-0.1.0.dist-info/METADATA,sha256=K--pGwzvhbMu2RDLUdo0MEJeKsTNy1JRRUKp9SNw9gc,5383
|
|
5
|
+
falsegreen_robot-0.1.0.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87
|
|
6
|
+
falsegreen_robot-0.1.0.dist-info/entry_points.txt,sha256=rH_6UKHH7VkCvPtX4oWUP9ZcniQafqrBPCpKtpUZJyk,67
|
|
7
|
+
falsegreen_robot-0.1.0.dist-info/licenses/LICENSE,sha256=u347iJB1k4mASuUpZnxYdINA-jKLOkzyaVHOOzQf70U,1073
|
|
8
|
+
falsegreen_robot-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Vinicius Queiroz
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|