robotframework-falsegreen 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- robotframework_falsegreen-0.2.0/.all-contributorsrc +41 -0
- robotframework_falsegreen-0.2.0/.github/workflows/ci.yml +28 -0
- robotframework_falsegreen-0.2.0/.github/workflows/codex-review-gate.yml +42 -0
- robotframework_falsegreen-0.2.0/.github/workflows/credit-contributor.yml +135 -0
- robotframework_falsegreen-0.2.0/.github/workflows/release.yml +47 -0
- robotframework_falsegreen-0.2.0/.gitignore +17 -0
- robotframework_falsegreen-0.2.0/ARCHITECTURE.md +85 -0
- robotframework_falsegreen-0.2.0/CHANGELOG.md +68 -0
- robotframework_falsegreen-0.2.0/CODE_OF_CONDUCT.md +29 -0
- robotframework_falsegreen-0.2.0/CONTRIBUTING.md +49 -0
- robotframework_falsegreen-0.2.0/CREDITS.md +58 -0
- robotframework_falsegreen-0.2.0/LICENSE +21 -0
- robotframework_falsegreen-0.2.0/PKG-INFO +177 -0
- robotframework_falsegreen-0.2.0/README.md +153 -0
- robotframework_falsegreen-0.2.0/RELEASE.md +50 -0
- robotframework_falsegreen-0.2.0/SECURITY.md +48 -0
- robotframework_falsegreen-0.2.0/docs/guide.md +242 -0
- robotframework_falsegreen-0.2.0/pyproject.toml +44 -0
- robotframework_falsegreen-0.2.0/src/falsegreen_robot/__init__.py +3 -0
- robotframework_falsegreen-0.2.0/src/falsegreen_robot/__main__.py +6 -0
- robotframework_falsegreen-0.2.0/src/falsegreen_robot/scanner.py +704 -0
- robotframework_falsegreen-0.2.0/tests/test_scanner.py +582 -0
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
{
|
|
2
|
+
"projectName": "robotframework-falsegreen",
|
|
3
|
+
"projectOwner": "vinicq",
|
|
4
|
+
"repoType": "github",
|
|
5
|
+
"repoHost": "https://github.com",
|
|
6
|
+
"files": [
|
|
7
|
+
"README.md"
|
|
8
|
+
],
|
|
9
|
+
"imageSize": 100,
|
|
10
|
+
"commit": false,
|
|
11
|
+
"commitConvention": "angular",
|
|
12
|
+
"contributorsSortAlphabetically": false,
|
|
13
|
+
"contributorsPerLine": 7,
|
|
14
|
+
"skipCi": true,
|
|
15
|
+
"contributors": [
|
|
16
|
+
{
|
|
17
|
+
"login": "vinicq",
|
|
18
|
+
"name": "Vinicius Queiroz",
|
|
19
|
+
"avatar_url": "https://avatars.githubusercontent.com/u/78210890?v=4",
|
|
20
|
+
"profile": "https://vinicq.github.io/md-bridge/",
|
|
21
|
+
"contributions": [
|
|
22
|
+
"code",
|
|
23
|
+
"doc",
|
|
24
|
+
"ideas",
|
|
25
|
+
"maintenance",
|
|
26
|
+
"infra",
|
|
27
|
+
"test",
|
|
28
|
+
"research"
|
|
29
|
+
]
|
|
30
|
+
},
|
|
31
|
+
{
|
|
32
|
+
"login": "homesellerq-coder",
|
|
33
|
+
"name": "Home Seller",
|
|
34
|
+
"avatar_url": "https://avatars.githubusercontent.com/u/294912019?v=4",
|
|
35
|
+
"profile": "https://github.com/homesellerq-coder",
|
|
36
|
+
"contributions": [
|
|
37
|
+
"code"
|
|
38
|
+
]
|
|
39
|
+
}
|
|
40
|
+
]
|
|
41
|
+
}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
name: ci
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
pull_request:
|
|
7
|
+
|
|
8
|
+
jobs:
|
|
9
|
+
test:
|
|
10
|
+
runs-on: ubuntu-latest
|
|
11
|
+
strategy:
|
|
12
|
+
matrix:
|
|
13
|
+
python-version: ["3.9", "3.11", "3.13"]
|
|
14
|
+
steps:
|
|
15
|
+
- uses: actions/checkout@v4
|
|
16
|
+
- uses: actions/setup-python@v5
|
|
17
|
+
with:
|
|
18
|
+
python-version: ${{ matrix.python-version }}
|
|
19
|
+
- name: Install
|
|
20
|
+
run: |
|
|
21
|
+
python -m pip install --upgrade pip
|
|
22
|
+
pip install -e ".[dev]"
|
|
23
|
+
- name: Lint
|
|
24
|
+
run: ruff check src tests
|
|
25
|
+
- name: Test
|
|
26
|
+
run: pytest -q
|
|
27
|
+
- name: Self-scan (must not error)
|
|
28
|
+
run: python -m falsegreen_robot src tests
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
name: codex-review-gate
|
|
2
|
+
|
|
3
|
+
# Blocks a PR while the Codex reviewer has unresolved review threads on the
|
|
4
|
+
# CURRENT diff. It does NOT require Codex to have reviewed (that belongs in
|
|
5
|
+
# branch protection): with zero open threads the job is green, so a PR Codex has
|
|
6
|
+
# not touched still passes.
|
|
7
|
+
#
|
|
8
|
+
# - Re-runs on review, comment, and thread resolve/unresolve events, so the gate
|
|
9
|
+
# clears as soon as a thread is resolved and re-fires if one is reopened.
|
|
10
|
+
# - Outdated threads (their line changed in a newer push) are ignored: each push
|
|
11
|
+
# re-baselines the review, and Codex posts fresh threads if an issue remains.
|
|
12
|
+
# - If a PR has more than 100 threads the gate fails closed and asks for a manual
|
|
13
|
+
# review rather than risk passing on an unread page.
|
|
14
|
+
#
|
|
15
|
+
# A 👍 reaction is not a resolution signal; resolve the thread to clear the gate.
|
|
16
|
+
|
|
17
|
+
on:
|
|
18
|
+
pull_request:
|
|
19
|
+
pull_request_review:
|
|
20
|
+
pull_request_review_comment:
|
|
21
|
+
pull_request_review_thread:
|
|
22
|
+
|
|
23
|
+
permissions:
|
|
24
|
+
pull-requests: read
|
|
25
|
+
|
|
26
|
+
jobs:
|
|
27
|
+
gate:
|
|
28
|
+
runs-on: ubuntu-latest
|
|
29
|
+
steps:
|
|
30
|
+
- name: Fail on unresolved Codex review threads (current diff)
|
|
31
|
+
env:
|
|
32
|
+
GH_TOKEN: ${{ github.token }}
|
|
33
|
+
OWNER: ${{ github.repository_owner }}
|
|
34
|
+
REPO: ${{ github.event.repository.name }}
|
|
35
|
+
PR: ${{ github.event.pull_request.number }}
|
|
36
|
+
run: |
|
|
37
|
+
n=$(gh api graphql -f query='query($o:String!,$r:String!,$p:Int!){repository(owner:$o,name:$r){pullRequest(number:$p){reviewThreads(first:100){nodes{isResolved isOutdated comments(first:1){nodes{author{login}}}}}}}}' -F o="$OWNER" -F r="$REPO" -F p="$PR" --jq '[.data.repository.pullRequest.reviewThreads.nodes[] | select(.isResolved == false and .isOutdated == false) | select(.comments.nodes[0].author.login | startswith("chatgpt-codex"))] | length')
|
|
38
|
+
echo "Open Codex review threads on the current diff: $n"
|
|
39
|
+
if [ "$n" -ne 0 ]; then
|
|
40
|
+
echo "::error::$n unresolved Codex review thread(s) on the current diff. Resolve them before merging."
|
|
41
|
+
exit 1
|
|
42
|
+
fi
|
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
name: Credit contributor on merge
|
|
2
|
+
|
|
3
|
+
# Runs after every merged pull request from an external contributor. Infers
|
|
4
|
+
# which contribution categories the change touched (code, doc, test, infra),
|
|
5
|
+
# updates `.all-contributorsrc` via all-contributors-cli, regenerates the
|
|
6
|
+
# README block, and opens a `chore(docs): credit @<author>` PR.
|
|
7
|
+
#
|
|
8
|
+
# Maintainer PRs (vinicq) and bot PRs are skipped at the job-level `if`. PRs
|
|
9
|
+
# that close without merging never reach this workflow (the `merged == true`
|
|
10
|
+
# guard). Mirrors the credit-contributor workflow used in md-bridge.
|
|
11
|
+
#
|
|
12
|
+
# The credit PR opens through `gh pr create`, so the repo setting
|
|
13
|
+
# Settings -> Actions -> General -> Workflow permissions ->
|
|
14
|
+
# "Allow GitHub Actions to create and approve pull requests" must be ON.
|
|
15
|
+
|
|
16
|
+
on:
|
|
17
|
+
pull_request_target:
|
|
18
|
+
types: [closed]
|
|
19
|
+
|
|
20
|
+
permissions:
|
|
21
|
+
contents: read
|
|
22
|
+
|
|
23
|
+
concurrency:
|
|
24
|
+
group: credit-contributor
|
|
25
|
+
cancel-in-progress: false
|
|
26
|
+
|
|
27
|
+
jobs:
|
|
28
|
+
credit:
|
|
29
|
+
name: Credit @${{ github.event.pull_request.user.login }}
|
|
30
|
+
if: >-
|
|
31
|
+
github.event.pull_request.merged == true &&
|
|
32
|
+
github.event.pull_request.user.login != 'vinicq' &&
|
|
33
|
+
github.event.pull_request.user.type != 'Bot'
|
|
34
|
+
runs-on: ubuntu-latest
|
|
35
|
+
permissions:
|
|
36
|
+
contents: write
|
|
37
|
+
pull-requests: write
|
|
38
|
+
steps:
|
|
39
|
+
- name: Check out the default branch
|
|
40
|
+
uses: actions/checkout@v4
|
|
41
|
+
with:
|
|
42
|
+
ref: main
|
|
43
|
+
fetch-depth: 0
|
|
44
|
+
token: ${{ secrets.GITHUB_TOKEN }}
|
|
45
|
+
|
|
46
|
+
- name: Set up Node
|
|
47
|
+
uses: actions/setup-node@v4
|
|
48
|
+
with:
|
|
49
|
+
node-version: "20"
|
|
50
|
+
|
|
51
|
+
- name: Determine contribution types from the merged diff
|
|
52
|
+
id: types
|
|
53
|
+
env:
|
|
54
|
+
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
55
|
+
PR_NUMBER: ${{ github.event.pull_request.number }}
|
|
56
|
+
run: |
|
|
57
|
+
set -euo pipefail
|
|
58
|
+
files=$(gh pr diff "$PR_NUMBER" -R "$GITHUB_REPOSITORY" --name-only)
|
|
59
|
+
echo "Changed files:"; echo "$files"; echo "---"
|
|
60
|
+
|
|
61
|
+
types=()
|
|
62
|
+
src_re='\.(py|ts|tsx|js|jsx|mjs|cjs|mts|cts|robot|resource)$'
|
|
63
|
+
if echo "$files" | grep -qE "$src_re"; then
|
|
64
|
+
non_test=$(echo "$files" | grep -E "$src_re" \
|
|
65
|
+
| grep -vE '(^|/)tests?/|(^|/)test_|_test\.|\.(test|spec)\.' || true)
|
|
66
|
+
[ -n "$non_test" ] && types+=("code")
|
|
67
|
+
fi
|
|
68
|
+
if echo "$files" | grep -qE '(^|/)tests?/|(^|/)test_|_test\.py$|\.(test|spec)\.(ts|tsx|js|jsx|mjs)$'; then
|
|
69
|
+
types+=("test")
|
|
70
|
+
fi
|
|
71
|
+
if echo "$files" | grep -qE '(\.md$|(^|/)docs/|(^|/)(README|CONTRIBUTING|CHANGELOG|SECURITY|CODE_OF_CONDUCT|CREDITS))'; then
|
|
72
|
+
types+=("doc")
|
|
73
|
+
fi
|
|
74
|
+
if echo "$files" | grep -qE '(^|/)\.github/|Dockerfile|(^|/)pyproject\.toml$|(^|/)package\.json$|(^|/)tsconfig|\.pre-commit-config'; then
|
|
75
|
+
types+=("infra")
|
|
76
|
+
fi
|
|
77
|
+
[ ${#types[@]} -eq 0 ] && types+=("code")
|
|
78
|
+
joined=$(printf "%s\n" "${types[@]}" | sort -u | paste -sd, -)
|
|
79
|
+
echo "Inferred categories: $joined"
|
|
80
|
+
echo "types=$joined" >> "$GITHUB_OUTPUT"
|
|
81
|
+
|
|
82
|
+
- name: Add contributor and regenerate the README block
|
|
83
|
+
env:
|
|
84
|
+
AUTHOR: ${{ github.event.pull_request.user.login }}
|
|
85
|
+
TYPES: ${{ steps.types.outputs.types }}
|
|
86
|
+
run: |
|
|
87
|
+
set -euo pipefail
|
|
88
|
+
# all-contributors-cli `add` is declarative, not additive: it narrows
|
|
89
|
+
# the entry to exactly the types passed. Read existing types, union
|
|
90
|
+
# with the inferred ones, and pass the full set.
|
|
91
|
+
existing=$(node -e "const c=JSON.parse(require('fs').readFileSync('.all-contributorsrc','utf8')).contributors||[];const u=process.env.AUTHOR.toLowerCase();const m=c.find(x=>x.login.toLowerCase()===u);process.stdout.write(m?(m.contributions||[]).join(','):'')")
|
|
92
|
+
combined=$(printf '%s,%s' "$existing" "$TYPES" | tr ',' '\n' | sed '/^$/d' | sort -u | paste -sd, -)
|
|
93
|
+
echo "Existing: ${existing:-<none>}"; echo "Combined: $combined"
|
|
94
|
+
npx -y all-contributors-cli@latest add "$AUTHOR" "$combined"
|
|
95
|
+
npx -y all-contributors-cli@latest generate
|
|
96
|
+
|
|
97
|
+
- name: Check whether anything changed
|
|
98
|
+
id: changes
|
|
99
|
+
run: |
|
|
100
|
+
if git diff --quiet -- .all-contributorsrc README.md; then
|
|
101
|
+
echo "changed=false" >> "$GITHUB_OUTPUT"
|
|
102
|
+
else
|
|
103
|
+
echo "changed=true" >> "$GITHUB_OUTPUT"
|
|
104
|
+
fi
|
|
105
|
+
|
|
106
|
+
- name: Open the credit PR
|
|
107
|
+
if: steps.changes.outputs.changed == 'true'
|
|
108
|
+
env:
|
|
109
|
+
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
110
|
+
AUTHOR: ${{ github.event.pull_request.user.login }}
|
|
111
|
+
TYPES: ${{ steps.types.outputs.types }}
|
|
112
|
+
PR_NUMBER: ${{ github.event.pull_request.number }}
|
|
113
|
+
run: |
|
|
114
|
+
set -euo pipefail
|
|
115
|
+
git config user.name "github-actions[bot]"
|
|
116
|
+
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
|
|
117
|
+
branch="chore/credit-${AUTHOR}-${PR_NUMBER}"
|
|
118
|
+
git checkout -b "$branch"
|
|
119
|
+
git add .all-contributorsrc README.md
|
|
120
|
+
git commit -m "chore(docs): credit @${AUTHOR} for ${TYPES} (PR #${PR_NUMBER})"
|
|
121
|
+
git push -u origin "$branch" --force-with-lease
|
|
122
|
+
body_file=$(mktemp)
|
|
123
|
+
cat > "$body_file" <<EOF
|
|
124
|
+
Automated post-merge credit update for #${PR_NUMBER}.
|
|
125
|
+
|
|
126
|
+
Categories inferred from the merged diff: \`${TYPES}\`. The categories come from the all-contributors specification: <https://allcontributors.org/docs/en/emoji-key>. Edit \`.all-contributorsrc\` here if a category is off.
|
|
127
|
+
EOF
|
|
128
|
+
if existing=$(gh pr list -R "$GITHUB_REPOSITORY" --head "$branch" --json number --jq '.[0].number' 2>/dev/null) && [ -n "$existing" ]; then
|
|
129
|
+
gh pr comment "$existing" -R "$GITHUB_REPOSITORY" --body "Re-ran auto-credit. Categories: \`${TYPES}\`."
|
|
130
|
+
else
|
|
131
|
+
gh pr create -R "$GITHUB_REPOSITORY" --base main \
|
|
132
|
+
--title "chore(docs): credit @${AUTHOR}" \
|
|
133
|
+
--body-file "$body_file" --assignee "vinicq"
|
|
134
|
+
fi
|
|
135
|
+
rm -f "$body_file"
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
name: release
|
|
2
|
+
|
|
3
|
+
# Builds the package and publishes it to PyPI via Trusted Publishing (OIDC).
|
|
4
|
+
# No API token: the publish job exchanges a short-lived OIDC identity with PyPI.
|
|
5
|
+
# Fires on a published GitHub release, or manually for an existing tag.
|
|
6
|
+
#
|
|
7
|
+
# One-time setup (see RELEASE notes): on PyPI add a Trusted Publisher for project
|
|
8
|
+
# `robotframework-falsegreen` (owner vinicq, repo robotframework-falsegreen, workflow release.yml,
|
|
9
|
+
# environment pypi); create a GitHub environment named `pypi`.
|
|
10
|
+
|
|
11
|
+
on:
|
|
12
|
+
release:
|
|
13
|
+
types: [published]
|
|
14
|
+
workflow_dispatch:
|
|
15
|
+
|
|
16
|
+
permissions:
|
|
17
|
+
contents: read
|
|
18
|
+
|
|
19
|
+
jobs:
|
|
20
|
+
build:
|
|
21
|
+
runs-on: ubuntu-latest
|
|
22
|
+
steps:
|
|
23
|
+
- uses: actions/checkout@v4
|
|
24
|
+
- uses: actions/setup-python@v5
|
|
25
|
+
with:
|
|
26
|
+
python-version: "3.13"
|
|
27
|
+
- name: Build sdist and wheel
|
|
28
|
+
run: |
|
|
29
|
+
python -m pip install --upgrade build
|
|
30
|
+
python -m build
|
|
31
|
+
- uses: actions/upload-artifact@v4
|
|
32
|
+
with:
|
|
33
|
+
name: dist
|
|
34
|
+
path: dist/
|
|
35
|
+
|
|
36
|
+
publish:
|
|
37
|
+
needs: build
|
|
38
|
+
runs-on: ubuntu-latest
|
|
39
|
+
environment: pypi
|
|
40
|
+
permissions:
|
|
41
|
+
id-token: write # OIDC trusted publishing
|
|
42
|
+
steps:
|
|
43
|
+
- uses: actions/download-artifact@v4
|
|
44
|
+
with:
|
|
45
|
+
name: dist
|
|
46
|
+
path: dist/
|
|
47
|
+
- uses: pypa/gh-action-pypi-publish@release/v1
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
# Architecture
|
|
2
|
+
|
|
3
|
+
robotframework-falsegreen is a deterministic static scanner for Robot Framework tests. It reads
|
|
4
|
+
`.robot` and `.resource` files, parses them with the official Robot Framework parser, and
|
|
5
|
+
flags the patterns that let a test pass green without protecting anything. It never imports
|
|
6
|
+
keyword libraries, starts a browser, or runs the suite.
|
|
7
|
+
|
|
8
|
+
## The pipeline
|
|
9
|
+
|
|
10
|
+
```
|
|
11
|
+
paths ──▶ discover ──▶ get_model ──▶ ModelVisitor ──▶ findings ──▶ report / exit code
|
|
12
|
+
```
|
|
13
|
+
|
|
14
|
+
1. **Discover.** Walk the paths. A file counts when its extension is `.robot` or
|
|
15
|
+
`.resource`; build and vendor directories are skipped.
|
|
16
|
+
2. **Parse.** `robot.api.get_model` builds the same parse tree Robot itself uses. Using the
|
|
17
|
+
official parser, not a regex, means the scanner reads the file the way the runner does:
|
|
18
|
+
sections, settings, control structures, RPA tasks. No keyword is imported and nothing
|
|
19
|
+
executes.
|
|
20
|
+
3. **Visit.** A `robot.parsing.ModelVisitor` subclass walks the tree. `visit_TestCase`,
|
|
21
|
+
`visit_Task` (RPA), and `visit_Keyword` apply the case catalog. The shared call-level
|
|
22
|
+
checks (always-true, self-compare, `Sleep`, weak truthiness) run over every keyword call;
|
|
23
|
+
the structural checks (empty, no-oracle, swallowed failure, conditional-only
|
|
24
|
+
verification, forced green) run per test or task.
|
|
25
|
+
4. **Report.** Readable text or JSON (`--json`). The exit code is the CI contract.
|
|
26
|
+
|
|
27
|
+
## Why Robot needs its own scanner
|
|
28
|
+
|
|
29
|
+
Robot Framework is a keyword DSL, not Python source, so the `ast` module that powers
|
|
30
|
+
[falsegreen](https://github.com/vinicq/falsegreen) cannot read it. The correct parse comes
|
|
31
|
+
from `robot.api.get_model`, which is why this is a separate repository that depends on
|
|
32
|
+
`robotframework`, rather than a module inside the zero-dependency Python scanner.
|
|
33
|
+
|
|
34
|
+
## The oracle problem
|
|
35
|
+
|
|
36
|
+
In pytest the oracle is `assert`; in Jest it is `expect`. In Robot there is no single
|
|
37
|
+
keyword. Verification is spread across libraries. The scanner recognizes the oracle by
|
|
38
|
+
convention and by library form, in `is_verification`:
|
|
39
|
+
|
|
40
|
+
- the **`Should`** convention (`Should Be Equal`, `Element Should Be Visible`) - BuiltIn,
|
|
41
|
+
Collections, String, SeleniumLibrary, and most others;
|
|
42
|
+
- the **Browser** assertion engine, where the operator carries the assertion
|
|
43
|
+
(`Get Text sel == expected`);
|
|
44
|
+
- **RESTinstance** schema keywords;
|
|
45
|
+
- custom **`Verify*`/`Assert*`/`Validate*`** keywords, and `Wait Until ...` keywords that
|
|
46
|
+
fail on timeout.
|
|
47
|
+
|
|
48
|
+
A test with none of these verifies nothing (C2b). The convention is documented; a custom
|
|
49
|
+
keyword that asserts without `Should` in its name is the reason C2b is low-confidence, and
|
|
50
|
+
the reason R2 exists: a keyword named like a verifier whose body asserts nothing is a hollow
|
|
51
|
+
oracle.
|
|
52
|
+
|
|
53
|
+
## Output contract
|
|
54
|
+
|
|
55
|
+
| Exit | Meaning |
|
|
56
|
+
|------|---------|
|
|
57
|
+
| `0` | clean |
|
|
58
|
+
| `10` | low-confidence findings only |
|
|
59
|
+
| `20` | at least one high-confidence finding |
|
|
60
|
+
|
|
61
|
+
Each finding carries code, confidence (`high`/`low`), file, line, and judgment (J1-J6).
|
|
62
|
+
`--disable C16` turns off specific codes. Groups split by prefix: `false-positive` (C*/R*,
|
|
63
|
+
on), `diagnostic` (D*, opt-in via `--diagnostics`), `coupling` (M*, opt-in).
|
|
64
|
+
|
|
65
|
+
## The boundary: static, semantic, runtime
|
|
66
|
+
|
|
67
|
+
The scanner owns what the keyword structure proves. Outside that line:
|
|
68
|
+
|
|
69
|
+
- **Semantic** (the expected value contradicts the intended behavior) belongs to
|
|
70
|
+
[falsegreen-skill](https://github.com/vinicq/falsegreen-skill), the LLM pass.
|
|
71
|
+
- **Runtime** (a Test Run War, order dependence across suites) needs execution, which the
|
|
72
|
+
scanner does not do.
|
|
73
|
+
- **Style and naming** belong to [Robocop](https://github.com/MarketSquare/robotframework-robocop),
|
|
74
|
+
which is complementary, not a competitor.
|
|
75
|
+
|
|
76
|
+
Precision over recall: `C2b`/`R2` are low-confidence because a custom keyword can verify
|
|
77
|
+
without `Should` in its name. A softened heuristic that misses a case beats one that flags
|
|
78
|
+
correct code.
|
|
79
|
+
|
|
80
|
+
## Siblings
|
|
81
|
+
|
|
82
|
+
[falsegreen](https://github.com/vinicq/falsegreen) (Python, `ast`) and
|
|
83
|
+
[falsegreen-js](https://github.com/vinicq/falsegreen-js) (JS/TS, TypeScript compiler API).
|
|
84
|
+
Codes share an id across the family where the smell is the same concept (C2, C2b, C3, C5,
|
|
85
|
+
C7, C16, C21, C32).
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to this project are documented here. The format is based on
|
|
4
|
+
[Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to
|
|
5
|
+
[Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
6
|
+
|
|
7
|
+
## [Unreleased]
|
|
8
|
+
|
|
9
|
+
## [0.2.0] - 2026-06-23
|
|
10
|
+
|
|
11
|
+
### Changed
|
|
12
|
+
|
|
13
|
+
- **Renamed to `robotframework-falsegreen`** to fit the Robot Framework ecosystem. The PyPI
|
|
14
|
+
package and the GitHub repo are now `robotframework-falsegreen` (the `robotframework-`
|
|
15
|
+
prefix convention), and the package carries the `Framework :: Robot Framework :: Tool`
|
|
16
|
+
classifier. The CLI command is `rffalsegreen` (short, mirroring `rfbrowser`); the import
|
|
17
|
+
package stays `falsegreen_robot`. Pre-release, so there is no PyPI migration; the old
|
|
18
|
+
GitHub URL redirects.
|
|
19
|
+
|
|
20
|
+
### Added
|
|
21
|
+
|
|
22
|
+
- `--config-audit` mode (project layer): reads the Robot run config (`robot.toml`,
|
|
23
|
+
`pyproject.toml` `[tool.robot]`, `*.args` argument files) and reports PL9 - a
|
|
24
|
+
`--skiponfailure`/`--noncritical` option that turns a failing test into a non-fatal pass
|
|
25
|
+
(legacy, removed in RF 4+). Findings carry level `project` and a fix hint. TOML sources need
|
|
26
|
+
a TOML reader (`tomllib` on 3.11+, else `tomli`); the `*.args` scan works on every version.
|
|
27
|
+
README notes that Robot has no standard mutation tester, so the F7 layer is manual review.
|
|
28
|
+
- New codes: R3 (`*** Test Cases ***` inside a `.resource` file), R4 (`No Operation` as the
|
|
29
|
+
only step), R5 (`[Template]` with no data rows). C2 now also flags an empty user keyword
|
|
30
|
+
(only settings, no steps). C23 (low): a hard-coded IP-address URL in test data, restricted
|
|
31
|
+
to IP literals so hostname URLs common in E2E are not flagged.
|
|
32
|
+
- Documented test-pyramid coverage: unit, integration (API and database), and E2E. The
|
|
33
|
+
scanner is level-agnostic; some codes are interpreted per level to keep precision.
|
|
34
|
+
- Status report output: every finding now carries its pyramid level (unit / integration /
|
|
35
|
+
e2e, detected from the suite's imported libraries: SeleniumLibrary/Browser/AppiumLibrary
|
|
36
|
+
are e2e, RequestsLibrary/RESTinstance/DatabaseLibrary are integration) and a one-line fix
|
|
37
|
+
hint. The text summary adds a per-level breakdown and the top fixes by frequency; JSON
|
|
38
|
+
gains `level` and `fix` fields.
|
|
39
|
+
- `--output` flag: write to a file, or pass a directory (e.g. `.falsegreen/`) to get
|
|
40
|
+
`report.<ext>` for the chosen format. Parent directories are created as needed.
|
|
41
|
+
|
|
42
|
+
## [0.1.0] - 2026-06-22
|
|
43
|
+
|
|
44
|
+
### Added
|
|
45
|
+
|
|
46
|
+
- `.resource` files and `*** Keywords ***` definitions are now scanned (in both `.robot`
|
|
47
|
+
and `.resource`). New code R2: a user keyword named like a verifier (Verify/Assert/Should)
|
|
48
|
+
whose body contains no verification — a hollow oracle, the root cause of missed C2b.
|
|
49
|
+
Call-level smells (C5/C7/C16) are also detected inside user keywords.
|
|
50
|
+
- Three groups (`false-positive` / `diagnostic` / `coupling`), like the sibling scanners.
|
|
51
|
+
Opt-in maintainability group (default off, `--diagnostics`): D2 (control flow at the
|
|
52
|
+
test/task level), M2 (too many steps).
|
|
53
|
+
- RPA support: scans `*** Tasks ***` in addition to `*** Test Cases ***`.
|
|
54
|
+
- Initial release. Deterministic static scanner for false-positive Robot Framework
|
|
55
|
+
tests, built on the official `robot.api.get_model` parser (no execution).
|
|
56
|
+
- Detection codes: C2 (empty test), C2b (no verification keyword), C3 (swallowed
|
|
57
|
+
`Run Keyword And Ignore Error`/`Return Status`), C5 (always-true), C7 (self-compare),
|
|
58
|
+
C6 (weak Should Be True on a bare variable), C16 (`Sleep` as synchronization), C21 (conditional-only verification), C32 (skipped), R1 (Pass Execution forced green). C3 also covers native TRY/EXCEPT swallow. Codes share ids with the sibling
|
|
59
|
+
scanners where the concept matches.
|
|
60
|
+
- Verification-keyword recognition across libraries: the `Should` convention plus
|
|
61
|
+
SeleniumLibrary/AppiumLibrary, Browser's assertion engine (`Get ... == expected`),
|
|
62
|
+
RESTinstance schema keywords, DatabaseLibrary, RequestsLibrary, and custom
|
|
63
|
+
`Verify*`/`Assert*`/`Validate*`/`Check *` keywords.
|
|
64
|
+
- CLI: paths, `--json`, `--disable`, `--version`. Exit codes 0/10/20.
|
|
65
|
+
|
|
66
|
+
[Unreleased]: https://github.com/vinicq/robotframework-falsegreen/compare/v0.2.0...HEAD
|
|
67
|
+
[0.2.0]: https://github.com/vinicq/robotframework-falsegreen/compare/v0.1.0...v0.2.0
|
|
68
|
+
[0.1.0]: https://github.com/vinicq/robotframework-falsegreen/releases/tag/v0.1.0
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
# Code of Conduct
|
|
2
|
+
|
|
3
|
+
## Our pledge
|
|
4
|
+
|
|
5
|
+
We pledge to make participation in this project a harassment-free experience for
|
|
6
|
+
everyone, regardless of age, body size, disability, ethnicity, gender identity and
|
|
7
|
+
expression, level of experience, nationality, personal appearance, race, religion, or
|
|
8
|
+
sexual identity and orientation.
|
|
9
|
+
|
|
10
|
+
## Our standards
|
|
11
|
+
|
|
12
|
+
Examples of behavior that contributes to a positive environment:
|
|
13
|
+
|
|
14
|
+
- Using welcoming and inclusive language.
|
|
15
|
+
- Respecting differing viewpoints and experiences.
|
|
16
|
+
- Accepting constructive criticism gracefully.
|
|
17
|
+
- Focusing on what is best for the project and the community.
|
|
18
|
+
|
|
19
|
+
Unacceptable behavior includes harassment, insulting or derogatory comments, public or
|
|
20
|
+
private harassment, and publishing others' private information without permission.
|
|
21
|
+
|
|
22
|
+
## Enforcement
|
|
23
|
+
|
|
24
|
+
Instances of abusive or unacceptable behavior may be reported to the maintainer at
|
|
25
|
+
`vinicq@gmail.com`. All complaints will be reviewed and investigated and will result in a
|
|
26
|
+
response appropriate to the circumstances.
|
|
27
|
+
|
|
28
|
+
This Code of Conduct is adapted from the [Contributor Covenant](https://www.contributor-covenant.org),
|
|
29
|
+
version 2.1.
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
# Contributing to robotframework-falsegreen
|
|
2
|
+
|
|
3
|
+
Thanks for helping. robotframework-falsegreen has one job: flag Robot Framework tests that pass
|
|
4
|
+
green without protecting anything. Keep contributions inside that scope.
|
|
5
|
+
|
|
6
|
+
## Scope
|
|
7
|
+
|
|
8
|
+
In scope: a test (or task) that can stay green while the behavior is wrong - no
|
|
9
|
+
verification keyword, a swallowed `Run Keyword And Ignore Error` or `TRY/EXCEPT`, an
|
|
10
|
+
always-true `Should Be True ${TRUE}`, `Pass Execution`, a verification reachable only
|
|
11
|
+
through an `IF`. Out of scope: style, naming, length, and convention, unless they make a
|
|
12
|
+
passing test unable to fail. Those belong to [Robocop](https://github.com/MarketSquare/robotframework-robocop),
|
|
13
|
+
which is complementary, not a competitor. When in doubt, ask: *is there a way for the code
|
|
14
|
+
to be wrong and this test to stay green?* If no, it is not a robotframework-falsegreen code.
|
|
15
|
+
|
|
16
|
+
## Setup
|
|
17
|
+
|
|
18
|
+
```bash
|
|
19
|
+
pip install -e ".[dev]"
|
|
20
|
+
pytest -q
|
|
21
|
+
ruff check src tests
|
|
22
|
+
python -m falsegreen_robot src tests # self-scan: the tool must not error
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
## Adding a detection code
|
|
26
|
+
|
|
27
|
+
1. Add the entry to `CASES` in `src/falsegreen_robot/scanner.py` (id, title, confidence,
|
|
28
|
+
judgment J1-J6). Reuse a `C*` id when the smell matches the Python/JS concept; use `R*`
|
|
29
|
+
for Robot-specific patterns; `D*`/`M*` for the opt-in diagnostic/coupling groups.
|
|
30
|
+
2. Implement the check over the Robot model (`robot.api.get_model` + `ModelVisitor`). It
|
|
31
|
+
must be provable from the parse tree - no execution.
|
|
32
|
+
3. Add a test in `tests/test_scanner.py`: a `.robot` snippet that must flag and a clean
|
|
33
|
+
look-alike that must not.
|
|
34
|
+
4. Document it in the README catalog, `docs/guide.md`, and `CHANGELOG.md`.
|
|
35
|
+
|
|
36
|
+
Precision over recall. `C2b`/`R2` are `low` because a custom keyword can verify without
|
|
37
|
+
`Should` in its name. A softened heuristic that misses a case is preferred to one that
|
|
38
|
+
flags correct code.
|
|
39
|
+
|
|
40
|
+
## Recognizing verification keywords
|
|
41
|
+
|
|
42
|
+
The oracle in Robot is a verification keyword. The scanner recognizes them across libraries
|
|
43
|
+
(the `Should` convention plus the Browser assertion engine and RESTinstance schema). If you
|
|
44
|
+
add support for a new library, extend `is_verification` and add a test.
|
|
45
|
+
|
|
46
|
+
## Commit and PR
|
|
47
|
+
|
|
48
|
+
Small, focused commits. Run pytest + ruff + self-scan before opening the PR. Reference the
|
|
49
|
+
issue. Match the surrounding code; keep `robotframework` as the only runtime dependency.
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
# References
|
|
2
|
+
|
|
3
|
+
robotframework-falsegreen detects false-green test smells in Robot Framework. Its catalog draws on
|
|
4
|
+
the academic test-smell literature, on the Robot Framework documentation that defines what
|
|
5
|
+
a verification keyword is, and on the existing Robot linter. This file credits those
|
|
6
|
+
sources and maps each to the codes it informs.
|
|
7
|
+
|
|
8
|
+
## Founding and conceptual
|
|
9
|
+
|
|
10
|
+
- **van Deursen, Moonen, van den Bergh, Kok (2001).** "Refactoring Test Code." XP 2001.
|
|
11
|
+
The original catalog of 11 test smells. Source of the general vocabulary that the
|
|
12
|
+
sibling scanners share.
|
|
13
|
+
- **Delplanque, Ducasse, Polito, Black, Etien (2019).** "Rotten Green Tests." ICSE 2019.
|
|
14
|
+
Green tests whose assertions never execute. The conceptual origin of the "false-green"
|
|
15
|
+
framing and of the codes where a check is present but cannot run: C21 (verification
|
|
16
|
+
guarded by `IF`) and R2 (a verifier keyword whose body asserts nothing). Cross-language
|
|
17
|
+
extension: EMSE 2021.
|
|
18
|
+
|
|
19
|
+
## Robot Framework specific
|
|
20
|
+
|
|
21
|
+
- **Robot Framework User Guide** (robotframework.org). Defines the test/task/keyword
|
|
22
|
+
structure the scanner walks, the `*** Tasks ***` section for RPA, `.resource` files, and
|
|
23
|
+
control structures (`IF`, `TRY/EXCEPT`, `FOR`, `WHILE`). Source for what D2 treats as
|
|
24
|
+
test-level control flow.
|
|
25
|
+
- **BuiltIn library (libdoc).** The `Should` naming convention for verification keywords
|
|
26
|
+
(`Should Be Equal`, `Should Be True`, `Should Contain`), plus `Pass Execution`,
|
|
27
|
+
`Run Keyword And Ignore Error`, `Run Keyword And Return Status`, and `Skip`. This is the
|
|
28
|
+
authoritative basis for the oracle vocabulary in `is_verification` and for C3, C5, C7,
|
|
29
|
+
R1, and C32.
|
|
30
|
+
- **Library assertion forms beyond `Should`.** SeleniumLibrary (`Element Should Be
|
|
31
|
+
Visible`, `Wait Until ... Visible`), the Browser library assertion engine
|
|
32
|
+
(`Get Text sel == expected`), RESTinstance schema keywords, and DatabaseLibrary
|
|
33
|
+
(`Row Count Should Be Equal`). These define the cross-library oracle set so C2b does not
|
|
34
|
+
fire on a test that verifies through a non-`Should` keyword.
|
|
35
|
+
|
|
36
|
+
## Tooling baseline
|
|
37
|
+
|
|
38
|
+
- **Robocop** (MarketSquare/robotframework-robocop). The Robot Framework linter for style,
|
|
39
|
+
naming, and convention. Complementary, not a competitor: Robocop checks how the code
|
|
40
|
+
reads; robotframework-falsegreen checks whether a passing test can still fail. The boundary
|
|
41
|
+
between the two is the scope line in CONTRIBUTING.
|
|
42
|
+
|
|
43
|
+
## Code-to-source map
|
|
44
|
+
|
|
45
|
+
| Code | Primary source(s) |
|
|
46
|
+
|---|---|
|
|
47
|
+
| C2, C2b | van Deursen 2001 (Empty/Unknown Test); BuiltIn oracle vocabulary |
|
|
48
|
+
| C3 | BuiltIn `Run Keyword And Ignore Error` / `Return Status`; native `TRY/EXCEPT` |
|
|
49
|
+
| C5, C7 | Redundant/always-true assertion (van Deursen 2001; BuiltIn) |
|
|
50
|
+
| C6 | weak truthiness check on a bare variable (BuiltIn `Should Be True`) |
|
|
51
|
+
| C16 | Sleepy Test (van Deursen 2001) applied to BuiltIn `Sleep` |
|
|
52
|
+
| C21, R2 | Rotten Green Tests (Delplanque 2019): assertion present but unreachable or hollow |
|
|
53
|
+
| C32 | BuiltIn `Skip` / `robot:skip` tag |
|
|
54
|
+
| R1 | BuiltIn `Pass Execution` (forced green) |
|
|
55
|
+
| D2 | test-level control flow (Robot Framework User Guide) |
|
|
56
|
+
| M2 | long test / too many steps (style guidance) |
|
|
57
|
+
|
|
58
|
+
Detailed per-source notes and the running research live in a separate private study.
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Vinicius Queiroz
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|