buildlog 0.6.0__tar.gz → 0.6.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {buildlog-0.6.0 → buildlog-0.6.1}/PKG-INFO +1 -1
- {buildlog-0.6.0 → buildlog-0.6.1}/pyproject.toml +1 -1
- {buildlog-0.6.0 → buildlog-0.6.1}/src/buildlog/cli.py +32 -20
- buildlog-0.6.1/src/buildlog/data/seeds/security_karen.yaml +162 -0
- buildlog-0.6.1/src/buildlog/data/seeds/test_terrorist.yaml +280 -0
- {buildlog-0.6.0 → buildlog-0.6.1}/src/buildlog/seeds.py +50 -0
- buildlog-0.6.1/template/buildlog/assets/.gitkeep +0 -0
- {buildlog-0.6.0 → buildlog-0.6.1}/.gitignore +0 -0
- {buildlog-0.6.0 → buildlog-0.6.1}/LICENSE +0 -0
- {buildlog-0.6.0 → buildlog-0.6.1}/README.md +0 -0
- {buildlog-0.6.0 → buildlog-0.6.1}/copier.yml +0 -0
- {buildlog-0.6.0 → buildlog-0.6.1}/post_gen.py +0 -0
- {buildlog-0.6.0 → buildlog-0.6.1}/src/buildlog/__init__.py +0 -0
- {buildlog-0.6.0 → buildlog-0.6.1}/src/buildlog/confidence.py +0 -0
- {buildlog-0.6.0 → buildlog-0.6.1}/src/buildlog/core/__init__.py +0 -0
- {buildlog-0.6.0 → buildlog-0.6.1}/src/buildlog/core/operations.py +0 -0
- /buildlog-0.6.0/template/buildlog/.gitkeep → /buildlog-0.6.1/src/buildlog/data/__init__.py +0 -0
- {buildlog-0.6.0 → buildlog-0.6.1}/src/buildlog/distill.py +0 -0
- {buildlog-0.6.0 → buildlog-0.6.1}/src/buildlog/embeddings.py +0 -0
- {buildlog-0.6.0 → buildlog-0.6.1}/src/buildlog/mcp/__init__.py +0 -0
- {buildlog-0.6.0 → buildlog-0.6.1}/src/buildlog/mcp/server.py +0 -0
- {buildlog-0.6.0 → buildlog-0.6.1}/src/buildlog/mcp/tools.py +0 -0
- {buildlog-0.6.0 → buildlog-0.6.1}/src/buildlog/render/__init__.py +0 -0
- {buildlog-0.6.0 → buildlog-0.6.1}/src/buildlog/render/base.py +0 -0
- {buildlog-0.6.0 → buildlog-0.6.1}/src/buildlog/render/claude_md.py +0 -0
- {buildlog-0.6.0 → buildlog-0.6.1}/src/buildlog/render/settings_json.py +0 -0
- {buildlog-0.6.0 → buildlog-0.6.1}/src/buildlog/render/skill.py +0 -0
- {buildlog-0.6.0 → buildlog-0.6.1}/src/buildlog/render/tracking.py +0 -0
- {buildlog-0.6.0 → buildlog-0.6.1}/src/buildlog/seed_engine/__init__.py +0 -0
- {buildlog-0.6.0 → buildlog-0.6.1}/src/buildlog/seed_engine/categorizers.py +0 -0
- {buildlog-0.6.0 → buildlog-0.6.1}/src/buildlog/seed_engine/extractors.py +0 -0
- {buildlog-0.6.0 → buildlog-0.6.1}/src/buildlog/seed_engine/generators.py +0 -0
- {buildlog-0.6.0 → buildlog-0.6.1}/src/buildlog/seed_engine/models.py +0 -0
- {buildlog-0.6.0 → buildlog-0.6.1}/src/buildlog/seed_engine/pipeline.py +0 -0
- {buildlog-0.6.0 → buildlog-0.6.1}/src/buildlog/seed_engine/sources.py +0 -0
- {buildlog-0.6.0 → buildlog-0.6.1}/src/buildlog/skills.py +0 -0
- {buildlog-0.6.0 → buildlog-0.6.1}/src/buildlog/stats.py +0 -0
- {buildlog-0.6.0/template/buildlog/assets → buildlog-0.6.1/template/buildlog}/.gitkeep +0 -0
- {buildlog-0.6.0 → buildlog-0.6.1}/template/buildlog/2026-01-01-example.md +0 -0
- {buildlog-0.6.0 → buildlog-0.6.1}/template/buildlog/BUILDLOG_SYSTEM.md +0 -0
- {buildlog-0.6.0 → buildlog-0.6.1}/template/buildlog/_TEMPLATE.md +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: buildlog
|
|
3
|
-
Version: 0.6.
|
|
3
|
+
Version: 0.6.1
|
|
4
4
|
Summary: Engineering notebook for AI-assisted development
|
|
5
5
|
Project-URL: Homepage, https://github.com/Peleke/buildlog-template
|
|
6
6
|
Project-URL: Repository, https://github.com/Peleke/buildlog-template
|
|
@@ -921,15 +921,18 @@ def gauntlet_list(output_json: bool):
|
|
|
921
921
|
"""
|
|
922
922
|
import json as json_module
|
|
923
923
|
|
|
924
|
-
from buildlog.seeds import load_all_seeds
|
|
924
|
+
from buildlog.seeds import get_default_seeds_dir, load_all_seeds
|
|
925
925
|
|
|
926
|
-
# Find seeds directory
|
|
927
|
-
|
|
928
|
-
seeds_dir = buildlog_dir / ".buildlog" / "seeds"
|
|
926
|
+
# Find seeds directory (local overrides > buildlog template > package bundled)
|
|
927
|
+
seeds_dir = get_default_seeds_dir()
|
|
929
928
|
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
929
|
+
if seeds_dir is None:
|
|
930
|
+
if output_json:
|
|
931
|
+
click.echo('{"personas": {}, "total_rules": 0, "error": "No seeds found"}')
|
|
932
|
+
else:
|
|
933
|
+
click.echo("No seed files found.")
|
|
934
|
+
click.echo("Seeds are bundled with buildlog - check your installation.")
|
|
935
|
+
return
|
|
933
936
|
|
|
934
937
|
seeds = load_all_seeds(seeds_dir)
|
|
935
938
|
|
|
@@ -997,18 +1000,22 @@ def gauntlet_rules(persona: str, fmt: str, output: str | None):
|
|
|
997
1000
|
"""
|
|
998
1001
|
import json as json_module
|
|
999
1002
|
|
|
1000
|
-
from buildlog.seeds import load_all_seeds
|
|
1003
|
+
from buildlog.seeds import get_default_seeds_dir, load_all_seeds
|
|
1004
|
+
|
|
1005
|
+
# Find seeds directory (local overrides > buildlog template > package bundled)
|
|
1006
|
+
seeds_dir = get_default_seeds_dir()
|
|
1001
1007
|
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
|
|
1008
|
+
if seeds_dir is None:
|
|
1009
|
+
click.echo("No seed files found.", err=True)
|
|
1010
|
+
click.echo(
|
|
1011
|
+
"Seeds are bundled with buildlog - check your installation.", err=True
|
|
1012
|
+
)
|
|
1013
|
+
raise SystemExit(1)
|
|
1006
1014
|
|
|
1007
1015
|
seeds = load_all_seeds(seeds_dir)
|
|
1008
1016
|
|
|
1009
1017
|
if not seeds:
|
|
1010
|
-
click.echo("No seed files found.", err=True)
|
|
1011
|
-
click.echo("Initialize with: buildlog init", err=True)
|
|
1018
|
+
click.echo("No seed files found in directory.", err=True)
|
|
1012
1019
|
raise SystemExit(1)
|
|
1013
1020
|
|
|
1014
1021
|
# Filter personas
|
|
@@ -1117,17 +1124,22 @@ def gauntlet_prompt(target: str, persona: tuple[str, ...], output: str | None):
|
|
|
1117
1124
|
buildlog gauntlet prompt src/api.py -p security_karen
|
|
1118
1125
|
buildlog gauntlet prompt . -o review_prompt.md
|
|
1119
1126
|
"""
|
|
1120
|
-
from buildlog.seeds import load_all_seeds
|
|
1127
|
+
from buildlog.seeds import get_default_seeds_dir, load_all_seeds
|
|
1121
1128
|
|
|
1122
|
-
# Find seeds directory
|
|
1123
|
-
seeds_dir =
|
|
1124
|
-
|
|
1125
|
-
|
|
1129
|
+
# Find seeds directory (local overrides > buildlog template > package bundled)
|
|
1130
|
+
seeds_dir = get_default_seeds_dir()
|
|
1131
|
+
|
|
1132
|
+
if seeds_dir is None:
|
|
1133
|
+
click.echo("No seed files found.", err=True)
|
|
1134
|
+
click.echo(
|
|
1135
|
+
"Seeds are bundled with buildlog - check your installation.", err=True
|
|
1136
|
+
)
|
|
1137
|
+
raise SystemExit(1)
|
|
1126
1138
|
|
|
1127
1139
|
seeds = load_all_seeds(seeds_dir)
|
|
1128
1140
|
|
|
1129
1141
|
if not seeds:
|
|
1130
|
-
click.echo("No seed files found.", err=True)
|
|
1142
|
+
click.echo("No seed files found in directory.", err=True)
|
|
1131
1143
|
raise SystemExit(1)
|
|
1132
1144
|
|
|
1133
1145
|
# Filter personas
|
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
# Security Karen - Curated Security Rules
|
|
2
|
+
# Source: OWASP Top 10 (2021), CWE, security best practices
|
|
3
|
+
# These rules are defensible - each links to authoritative sources
|
|
4
|
+
|
|
5
|
+
persona: security_karen
|
|
6
|
+
version: 1
|
|
7
|
+
|
|
8
|
+
rules:
|
|
9
|
+
# A01:2021 - Broken Access Control
|
|
10
|
+
- rule: "Verify authorization on every privileged operation"
|
|
11
|
+
category: security
|
|
12
|
+
context: "Any endpoint or function that modifies data or accesses sensitive resources"
|
|
13
|
+
antipattern: "Checking authentication but not authorization; assuming authenticated = authorized"
|
|
14
|
+
rationale: "Broken access control is OWASP #1. Users can act outside intended permissions."
|
|
15
|
+
tags: [access-control, authorization, owasp-a01]
|
|
16
|
+
references:
|
|
17
|
+
- url: "https://owasp.org/Top10/A01_2021-Broken_Access_Control/"
|
|
18
|
+
title: "OWASP A01:2021 Broken Access Control"
|
|
19
|
+
- url: "https://cwe.mitre.org/data/definitions/862.html"
|
|
20
|
+
title: "CWE-862: Missing Authorization"
|
|
21
|
+
|
|
22
|
+
- rule: "Deny by default for access control decisions"
|
|
23
|
+
category: security
|
|
24
|
+
context: "Access control logic, permission checks, authorization middleware"
|
|
25
|
+
antipattern: "Allowing access unless explicitly denied; missing else clause in auth checks"
|
|
26
|
+
rationale: "Fail-open access control leads to unauthorized access. Fail-closed is safer."
|
|
27
|
+
tags: [access-control, authorization, defense-in-depth]
|
|
28
|
+
references:
|
|
29
|
+
- url: "https://owasp.org/Top10/A01_2021-Broken_Access_Control/"
|
|
30
|
+
title: "OWASP A01:2021 Broken Access Control"
|
|
31
|
+
|
|
32
|
+
# A02:2021 - Cryptographic Failures
|
|
33
|
+
- rule: "Never store passwords in plaintext or reversible encryption"
|
|
34
|
+
category: security
|
|
35
|
+
context: "User registration, password storage, credential management"
|
|
36
|
+
antipattern: "Storing raw passwords; using MD5/SHA1 without salt; using encryption instead of hashing"
|
|
37
|
+
rationale: "Password leaks expose users. Use bcrypt/argon2 with proper work factors."
|
|
38
|
+
tags: [cryptography, passwords, owasp-a02]
|
|
39
|
+
references:
|
|
40
|
+
- url: "https://owasp.org/Top10/A02_2021-Cryptographic_Failures/"
|
|
41
|
+
title: "OWASP A02:2021 Cryptographic Failures"
|
|
42
|
+
- url: "https://cheatsheetseries.owasp.org/cheatsheets/Password_Storage_Cheat_Sheet.html"
|
|
43
|
+
title: "OWASP Password Storage Cheat Sheet"
|
|
44
|
+
|
|
45
|
+
- rule: "Use TLS 1.2+ for all data in transit"
|
|
46
|
+
category: security
|
|
47
|
+
context: "API calls, database connections, service-to-service communication"
|
|
48
|
+
antipattern: "HTTP endpoints; TLS 1.0/1.1; self-signed certs in production without pinning"
|
|
49
|
+
rationale: "Data in transit can be intercepted. TLS provides confidentiality and integrity."
|
|
50
|
+
tags: [cryptography, tls, owasp-a02]
|
|
51
|
+
references:
|
|
52
|
+
- url: "https://owasp.org/Top10/A02_2021-Cryptographic_Failures/"
|
|
53
|
+
title: "OWASP A02:2021 Cryptographic Failures"
|
|
54
|
+
|
|
55
|
+
# A03:2021 - Injection
|
|
56
|
+
- rule: "Parameterize all database queries"
|
|
57
|
+
category: security
|
|
58
|
+
context: "Any code constructing SQL, NoSQL, LDAP, or OS commands from input"
|
|
59
|
+
antipattern: "String concatenation with user input; f-strings in queries; raw SQL with variables"
|
|
60
|
+
rationale: "SQL injection allows data theft, modification, or deletion. Parameterization prevents it."
|
|
61
|
+
tags: [injection, sql, owasp-a03]
|
|
62
|
+
references:
|
|
63
|
+
- url: "https://owasp.org/Top10/A03_2021-Injection/"
|
|
64
|
+
title: "OWASP A03:2021 Injection"
|
|
65
|
+
- url: "https://cwe.mitre.org/data/definitions/89.html"
|
|
66
|
+
title: "CWE-89: SQL Injection"
|
|
67
|
+
|
|
68
|
+
- rule: "Escape or sanitize all output to prevent XSS"
|
|
69
|
+
category: security
|
|
70
|
+
context: "Rendering user-provided content in HTML, JavaScript, or other executable contexts"
|
|
71
|
+
antipattern: "innerHTML with user data; dangerouslySetInnerHTML; template literals without escaping"
|
|
72
|
+
rationale: "XSS allows attackers to execute code in users' browsers. Always escape output."
|
|
73
|
+
tags: [injection, xss, owasp-a03]
|
|
74
|
+
references:
|
|
75
|
+
- url: "https://owasp.org/Top10/A03_2021-Injection/"
|
|
76
|
+
title: "OWASP A03:2021 Injection"
|
|
77
|
+
- url: "https://cheatsheetseries.owasp.org/cheatsheets/Cross_Site_Scripting_Prevention_Cheat_Sheet.html"
|
|
78
|
+
title: "OWASP XSS Prevention Cheat Sheet"
|
|
79
|
+
|
|
80
|
+
# A04:2021 - Insecure Design
|
|
81
|
+
- rule: "Implement rate limiting on authentication endpoints"
|
|
82
|
+
category: security
|
|
83
|
+
context: "Login, registration, password reset, API authentication"
|
|
84
|
+
antipattern: "Unlimited login attempts; no lockout; no CAPTCHA on repeated failures"
|
|
85
|
+
rationale: "Credential stuffing and brute force attacks are common. Rate limiting mitigates them."
|
|
86
|
+
tags: [authentication, rate-limiting, owasp-a04]
|
|
87
|
+
references:
|
|
88
|
+
- url: "https://owasp.org/Top10/A04_2021-Insecure_Design/"
|
|
89
|
+
title: "OWASP A04:2021 Insecure Design"
|
|
90
|
+
- url: "https://cheatsheetseries.owasp.org/cheatsheets/Authentication_Cheat_Sheet.html"
|
|
91
|
+
title: "OWASP Authentication Cheat Sheet"
|
|
92
|
+
|
|
93
|
+
# A05:2021 - Security Misconfiguration
|
|
94
|
+
- rule: "Never commit secrets to version control"
|
|
95
|
+
category: security
|
|
96
|
+
context: "Any code or configuration file that might contain API keys, passwords, tokens"
|
|
97
|
+
antipattern: "Hardcoded credentials; .env files in git; secrets in docker-compose.yml"
|
|
98
|
+
rationale: "Git history is forever. Leaked secrets lead to account compromise."
|
|
99
|
+
tags: [secrets, configuration, owasp-a05]
|
|
100
|
+
references:
|
|
101
|
+
- url: "https://owasp.org/Top10/A05_2021-Security_Misconfiguration/"
|
|
102
|
+
title: "OWASP A05:2021 Security Misconfiguration"
|
|
103
|
+
|
|
104
|
+
- rule: "Disable debug mode and verbose errors in production"
|
|
105
|
+
category: security
|
|
106
|
+
context: "Production deployments, error handling, logging configuration"
|
|
107
|
+
antipattern: "DEBUG=True in production; stack traces in API responses; verbose SQL errors"
|
|
108
|
+
rationale: "Debug info reveals implementation details useful for attackers."
|
|
109
|
+
tags: [configuration, errors, owasp-a05]
|
|
110
|
+
references:
|
|
111
|
+
- url: "https://owasp.org/Top10/A05_2021-Security_Misconfiguration/"
|
|
112
|
+
title: "OWASP A05:2021 Security Misconfiguration"
|
|
113
|
+
|
|
114
|
+
# A07:2021 - Identification and Authentication Failures
|
|
115
|
+
- rule: "Use secure session management with httpOnly and secure flags"
|
|
116
|
+
category: security
|
|
117
|
+
context: "Session cookies, JWT storage, authentication tokens"
|
|
118
|
+
antipattern: "Tokens in localStorage; cookies without httpOnly; missing secure flag"
|
|
119
|
+
rationale: "Insecure session storage enables session hijacking via XSS."
|
|
120
|
+
tags: [authentication, sessions, owasp-a07]
|
|
121
|
+
references:
|
|
122
|
+
- url: "https://owasp.org/Top10/A07_2021-Identification_and_Authentication_Failures/"
|
|
123
|
+
title: "OWASP A07:2021 Identification and Authentication Failures"
|
|
124
|
+
- url: "https://cheatsheetseries.owasp.org/cheatsheets/Session_Management_Cheat_Sheet.html"
|
|
125
|
+
title: "OWASP Session Management Cheat Sheet"
|
|
126
|
+
|
|
127
|
+
# A08:2021 - Software and Data Integrity Failures
|
|
128
|
+
- rule: "Verify integrity of external dependencies"
|
|
129
|
+
category: security
|
|
130
|
+
context: "Package installation, CI/CD pipelines, dependency updates"
|
|
131
|
+
antipattern: "No lockfile; no hash verification; installing from arbitrary URLs"
|
|
132
|
+
rationale: "Supply chain attacks inject malicious code via dependencies."
|
|
133
|
+
tags: [dependencies, supply-chain, owasp-a08]
|
|
134
|
+
references:
|
|
135
|
+
- url: "https://owasp.org/Top10/A08_2021-Software_and_Data_Integrity_Failures/"
|
|
136
|
+
title: "OWASP A08:2021 Software and Data Integrity Failures"
|
|
137
|
+
|
|
138
|
+
# A09:2021 - Security Logging and Monitoring Failures
|
|
139
|
+
- rule: "Log security-relevant events with sufficient context"
|
|
140
|
+
category: security
|
|
141
|
+
context: "Authentication, authorization failures, input validation failures"
|
|
142
|
+
antipattern: "No logging; logging without timestamps; no user/IP context; PII in logs"
|
|
143
|
+
rationale: "Without logs, breaches go undetected. Logs enable incident response."
|
|
144
|
+
tags: [logging, monitoring, owasp-a09]
|
|
145
|
+
references:
|
|
146
|
+
- url: "https://owasp.org/Top10/A09_2021-Security_Logging_and_Monitoring_Failures/"
|
|
147
|
+
title: "OWASP A09:2021 Security Logging and Monitoring Failures"
|
|
148
|
+
- url: "https://cheatsheetseries.owasp.org/cheatsheets/Logging_Cheat_Sheet.html"
|
|
149
|
+
title: "OWASP Logging Cheat Sheet"
|
|
150
|
+
|
|
151
|
+
# A10:2021 - Server-Side Request Forgery (SSRF)
|
|
152
|
+
- rule: "Validate and allowlist URLs for server-side requests"
|
|
153
|
+
category: security
|
|
154
|
+
context: "Fetching external URLs, webhooks, image proxies, URL shorteners"
|
|
155
|
+
antipattern: "Fetching arbitrary user-provided URLs; no protocol/host validation"
|
|
156
|
+
rationale: "SSRF allows attackers to access internal services or cloud metadata."
|
|
157
|
+
tags: [ssrf, input-validation, owasp-a10]
|
|
158
|
+
references:
|
|
159
|
+
- url: "https://owasp.org/Top10/A10_2021-Server-Side_Request_Forgery_%28SSRF%29/"
|
|
160
|
+
title: "OWASP A10:2021 SSRF"
|
|
161
|
+
- url: "https://cheatsheetseries.owasp.org/cheatsheets/Server_Side_Request_Forgery_Prevention_Cheat_Sheet.html"
|
|
162
|
+
title: "OWASP SSRF Prevention Cheat Sheet"
|
|
@@ -0,0 +1,280 @@
|
|
|
1
|
+
persona: test_terrorist
|
|
2
|
+
version: 1
|
|
3
|
+
rules:
|
|
4
|
+
- rule: Tests must not depend on execution order
|
|
5
|
+
category: isolation
|
|
6
|
+
context: Test suites with multiple tests, shared fixtures, database state
|
|
7
|
+
antipattern: Test A creates data that Test B asserts on; tests fail when run individually
|
|
8
|
+
rationale: Order-dependent tests are flaky and hide real failures. Each test must be hermetic.
|
|
9
|
+
tags:
|
|
10
|
+
- isolation
|
|
11
|
+
- order-independent
|
|
12
|
+
- hermetic
|
|
13
|
+
- test_terrorist
|
|
14
|
+
references:
|
|
15
|
+
- url: https://testing.googleblog.com/2010/12/test-sizes.html
|
|
16
|
+
title: Google Testing Blog - Test Sizes
|
|
17
|
+
- rule: Tests must clean up after themselves
|
|
18
|
+
category: isolation
|
|
19
|
+
context: Tests using databases, files, external services, global state
|
|
20
|
+
antipattern: Tests leaving data in shared resources; no teardown; assuming clean state
|
|
21
|
+
rationale: Test pollution causes cascading failures and makes debugging impossible.
|
|
22
|
+
tags:
|
|
23
|
+
- isolation
|
|
24
|
+
- test_terrorist
|
|
25
|
+
- cleanup
|
|
26
|
+
- teardown
|
|
27
|
+
references:
|
|
28
|
+
- url: https://testing.googleblog.com/2010/12/test-sizes.html
|
|
29
|
+
title: Google Testing Blog - Test Sizes
|
|
30
|
+
- rule: Tests should run in under 10 seconds for fast feedback
|
|
31
|
+
category: anti-patterns
|
|
32
|
+
context: Unit test suites, developer workflows, pre-commit hooks
|
|
33
|
+
antipattern: Minute-long test suites; 'just run CI'; tests that require coffee breaks
|
|
34
|
+
rationale: Slow tests don't get run. Fast feedback enables TDD and catches bugs early.
|
|
35
|
+
tags:
|
|
36
|
+
- feedback
|
|
37
|
+
- anti-patterns
|
|
38
|
+
- slow-tests
|
|
39
|
+
- test_terrorist
|
|
40
|
+
references:
|
|
41
|
+
- url: https://testing.googleblog.com/2010/12/test-sizes.html
|
|
42
|
+
title: Google Testing Blog - Test Sizes
|
|
43
|
+
- rule: Every public API must have at least one happy path test
|
|
44
|
+
category: coverage
|
|
45
|
+
context: New endpoints, public functions, exported modules
|
|
46
|
+
antipattern: Shipping code with no tests; 'I'll add tests later'; PRs without test changes
|
|
47
|
+
rationale: Untested code is legacy code the moment it's merged. Tests are executable documentation.
|
|
48
|
+
tags:
|
|
49
|
+
- happy-path
|
|
50
|
+
- public-api
|
|
51
|
+
- test_terrorist
|
|
52
|
+
- coverage
|
|
53
|
+
references:
|
|
54
|
+
- url: https://testing.googleblog.com/2015/04/just-say-no-to-more-end-to-end-tests.html
|
|
55
|
+
title: Google Testing Blog - Just Say No to More E2E Tests
|
|
56
|
+
- rule: New bug fixes must include a regression test
|
|
57
|
+
category: coverage
|
|
58
|
+
context: Any bug fix PR or commit
|
|
59
|
+
antipattern: Fixing bugs without adding tests that would have caught them
|
|
60
|
+
rationale: A bug that escapes once will escape again. Regression tests prevent recurrence.
|
|
61
|
+
tags:
|
|
62
|
+
- regression
|
|
63
|
+
- bug-fix
|
|
64
|
+
- test_terrorist
|
|
65
|
+
- coverage
|
|
66
|
+
references:
|
|
67
|
+
- url: https://testing.googleblog.com/2015/04/just-say-no-to-more-end-to-end-tests.html
|
|
68
|
+
title: Google Testing Blog - Just Say No to More E2E Tests
|
|
69
|
+
- rule: Critical paths require edge case and error path coverage
|
|
70
|
+
category: coverage
|
|
71
|
+
context: Payment flows, authentication, data mutations, external integrations
|
|
72
|
+
antipattern: Only happy path tests for critical code; no error handling tests
|
|
73
|
+
rationale: Edge cases in critical paths cause production incidents. Murphy's law applies.
|
|
74
|
+
tags:
|
|
75
|
+
- test_terrorist
|
|
76
|
+
- critical-path
|
|
77
|
+
- edge-cases
|
|
78
|
+
- coverage
|
|
79
|
+
references:
|
|
80
|
+
- url: https://martinfowler.com/articles/practical-test-pyramid.html
|
|
81
|
+
title: Martin Fowler - Testing Pyramid
|
|
82
|
+
- rule: Assert on behavior, not implementation details
|
|
83
|
+
category: assertions
|
|
84
|
+
context: Unit tests, refactoring scenarios
|
|
85
|
+
antipattern: Asserting on private method calls; testing internal state; mock call counts
|
|
86
|
+
rationale: Implementation-coupled tests break on refactoring. Test the contract, not the code.
|
|
87
|
+
tags:
|
|
88
|
+
- assertions
|
|
89
|
+
- behavior
|
|
90
|
+
- test_terrorist
|
|
91
|
+
- contract
|
|
92
|
+
references:
|
|
93
|
+
- url: https://martinfowler.com/articles/practical-test-pyramid.html
|
|
94
|
+
title: Martin Fowler - Testing Pyramid
|
|
95
|
+
- rule: Mock external dependencies at test boundaries
|
|
96
|
+
category: isolation
|
|
97
|
+
context: Tests calling APIs, databases, file systems, network services
|
|
98
|
+
antipattern: Real network calls in unit tests; tests that require running services
|
|
99
|
+
rationale: External dependencies make tests slow, flaky, and expensive. Mock at boundaries.
|
|
100
|
+
tags:
|
|
101
|
+
- isolation
|
|
102
|
+
- test_terrorist
|
|
103
|
+
- mocking
|
|
104
|
+
- boundaries
|
|
105
|
+
references:
|
|
106
|
+
- url: https://martinfowler.com/bliki/TestDouble.html
|
|
107
|
+
title: Martin Fowler - Test Double
|
|
108
|
+
- rule: Use property-based testing for functions with clear invariants
|
|
109
|
+
category: property-testing
|
|
110
|
+
context: Serializers, parsers, encoders/decoders, sorting, mathematical operations
|
|
111
|
+
antipattern: Only example-based tests for encode/decode pairs; hand-picked edge cases
|
|
112
|
+
rationale: Property tests generate thousands of examples, finding edge cases humans miss.
|
|
113
|
+
tags:
|
|
114
|
+
- invariants
|
|
115
|
+
- hypothesis
|
|
116
|
+
- test_terrorist
|
|
117
|
+
- property
|
|
118
|
+
references:
|
|
119
|
+
- url: https://hypothesis.readthedocs.io/en/latest/
|
|
120
|
+
title: Hypothesis Documentation
|
|
121
|
+
- rule: Define roundtrip properties for serialization code
|
|
122
|
+
category: property-testing
|
|
123
|
+
context: JSON, protobuf, custom serializers, data transformation pipelines
|
|
124
|
+
antipattern: Testing serialize and deserialize separately with fixed examples
|
|
125
|
+
rationale: decode(encode(x)) == x is a universal property. Hypothesis finds corner cases.
|
|
126
|
+
tags:
|
|
127
|
+
- roundtrip
|
|
128
|
+
- test_terrorist
|
|
129
|
+
- property
|
|
130
|
+
- serialization
|
|
131
|
+
references:
|
|
132
|
+
- url: https://hypothesis.readthedocs.io/en/latest/
|
|
133
|
+
title: Hypothesis Documentation
|
|
134
|
+
- rule: Apply metamorphic relations when test oracles are unavailable
|
|
135
|
+
category: metamorphic-testing
|
|
136
|
+
context: ML models, search engines, optimization algorithms, complex computations
|
|
137
|
+
antipattern: No testing because 'we don't know the right answer'; only manual inspection
|
|
138
|
+
rationale: Metamorphic testing validates input-output relationships without ground truth.
|
|
139
|
+
tags:
|
|
140
|
+
- metamorphic
|
|
141
|
+
- test_terrorist
|
|
142
|
+
- ml
|
|
143
|
+
- oracle-free
|
|
144
|
+
references:
|
|
145
|
+
- url: https://www.sciencedirect.com/science/article/pii/S0950584918300016
|
|
146
|
+
title: Metamorphic Testing - Chen et al. Survey
|
|
147
|
+
- rule: Define permutation invariance for order-independent operations
|
|
148
|
+
category: metamorphic-testing
|
|
149
|
+
context: Aggregations, set operations, commutative functions
|
|
150
|
+
antipattern: Testing with single fixed input order; assuming order doesn't matter
|
|
151
|
+
rationale: sum([1,2,3]) == sum([3,1,2]) is a metamorphic relation that catches bugs.
|
|
152
|
+
tags:
|
|
153
|
+
- metamorphic
|
|
154
|
+
- test_terrorist
|
|
155
|
+
- permutation
|
|
156
|
+
- invariance
|
|
157
|
+
references:
|
|
158
|
+
- url: https://www.sciencedirect.com/science/article/pii/S0950584918300016
|
|
159
|
+
title: Metamorphic Testing - Chen et al. Survey
|
|
160
|
+
- rule: Validate data distributions at pipeline boundaries
|
|
161
|
+
category: statistical-testing
|
|
162
|
+
context: ETL pipelines, ML feature stores, data ingestion, API responses
|
|
163
|
+
antipattern: Assuming input data matches expected distribution; no schema validation
|
|
164
|
+
rationale: Distribution drift breaks models silently. Validate expectations at boundaries.
|
|
165
|
+
tags:
|
|
166
|
+
- statistical
|
|
167
|
+
- drift
|
|
168
|
+
- test_terrorist
|
|
169
|
+
- distribution
|
|
170
|
+
references:
|
|
171
|
+
- url: https://docs.greatexpectations.io/docs/
|
|
172
|
+
title: Great Expectations Documentation
|
|
173
|
+
- rule: Define and enforce data contracts with schemas
|
|
174
|
+
category: statistical-testing
|
|
175
|
+
context: Data pipelines, API integrations, database migrations
|
|
176
|
+
antipattern: Implicit schemas; duck typing for data; hoping fields exist
|
|
177
|
+
rationale: Schema validation catches contract violations before they cause failures.
|
|
178
|
+
tags:
|
|
179
|
+
- statistical
|
|
180
|
+
- contracts
|
|
181
|
+
- test_terrorist
|
|
182
|
+
- schema
|
|
183
|
+
references:
|
|
184
|
+
- url: https://pandera.readthedocs.io/en/stable/
|
|
185
|
+
title: Pandera Documentation
|
|
186
|
+
- rule: LLM outputs require structured validation beyond string matching
|
|
187
|
+
category: llm-testing
|
|
188
|
+
context: Any code using LLM-generated content in production paths
|
|
189
|
+
antipattern: No validation; trusting raw LLM output; regex-only validation
|
|
190
|
+
rationale: '[GAP] Standard test frameworks don''t cover LLM eval. See Guardrails/DeepEval for emerging
|
|
191
|
+
patterns. This is a known gap requiring specialized tooling.'
|
|
192
|
+
tags:
|
|
193
|
+
- test_terrorist
|
|
194
|
+
- emerging
|
|
195
|
+
- llm-testing
|
|
196
|
+
- validation
|
|
197
|
+
- gap
|
|
198
|
+
references:
|
|
199
|
+
- url: https://www.guardrailsai.com/docs/concepts/guard
|
|
200
|
+
title: Guardrails AI Documentation
|
|
201
|
+
- rule: LLM-based features need evaluation datasets and metrics
|
|
202
|
+
category: llm-testing
|
|
203
|
+
context: RAG systems, chatbots, content generation, code assistants
|
|
204
|
+
antipattern: Vibes-based testing; manual spot checks; no regression tracking
|
|
205
|
+
rationale: '[GAP] LLM behavior is non-deterministic. Evaluation datasets with metrics enable regression
|
|
206
|
+
detection. Tooling is immature.'
|
|
207
|
+
tags:
|
|
208
|
+
- emerging
|
|
209
|
+
- llm-testing
|
|
210
|
+
- evaluation
|
|
211
|
+
- gap
|
|
212
|
+
- test_terrorist
|
|
213
|
+
references:
|
|
214
|
+
- url: https://docs.confident-ai.com/docs/getting-started
|
|
215
|
+
title: DeepEval Documentation
|
|
216
|
+
- rule: Every test must have at least one meaningful assertion
|
|
217
|
+
category: assertions
|
|
218
|
+
context: All test functions
|
|
219
|
+
antipattern: Tests that only call code without asserting; assert True; empty test bodies
|
|
220
|
+
rationale: A test without assertions is not a test. It's a false sense of security.
|
|
221
|
+
tags:
|
|
222
|
+
- assertions
|
|
223
|
+
- no-pass-through
|
|
224
|
+
- meaningful
|
|
225
|
+
- test_terrorist
|
|
226
|
+
references:
|
|
227
|
+
- url: http://xunitpatterns.com/Test%20Smells.html
|
|
228
|
+
title: xUnit Test Patterns - Test Smells
|
|
229
|
+
- rule: Follow Arrange-Act-Assert (AAA) pattern
|
|
230
|
+
category: structure
|
|
231
|
+
context: All test functions
|
|
232
|
+
antipattern: Interleaved setup and assertions; multiple acts per test; unclear test phases
|
|
233
|
+
rationale: AAA makes tests readable and debuggable. One logical assertion per test.
|
|
234
|
+
tags:
|
|
235
|
+
- structure
|
|
236
|
+
- aaa
|
|
237
|
+
- test_terrorist
|
|
238
|
+
- readability
|
|
239
|
+
references:
|
|
240
|
+
- url: http://xunitpatterns.com/Test%20Smells.html
|
|
241
|
+
title: xUnit Test Patterns - Test Smells
|
|
242
|
+
- rule: Avoid testing implementation details that change frequently
|
|
243
|
+
category: anti-patterns
|
|
244
|
+
context: Refactoring scenarios, internal APIs, private methods
|
|
245
|
+
antipattern: Tests break on every refactor; testing private method behavior
|
|
246
|
+
rationale: Fragile tests slow development. Test stable interfaces, not implementation.
|
|
247
|
+
tags:
|
|
248
|
+
- fragile
|
|
249
|
+
- implementation-details
|
|
250
|
+
- anti-patterns
|
|
251
|
+
- test_terrorist
|
|
252
|
+
references:
|
|
253
|
+
- url: http://xunitpatterns.com/Test%20Smells.html
|
|
254
|
+
title: xUnit Test Patterns - Test Smells
|
|
255
|
+
- rule: Flaky tests must be fixed or quarantined immediately
|
|
256
|
+
category: anti-patterns
|
|
257
|
+
context: CI pipelines, test suites with intermittent failures
|
|
258
|
+
antipattern: Rerunning CI until green; ignoring flaky tests; 'it works on my machine'
|
|
259
|
+
rationale: Flaky tests erode trust in the test suite. A flaky test is worse than no test.
|
|
260
|
+
tags:
|
|
261
|
+
- anti-patterns
|
|
262
|
+
- flaky
|
|
263
|
+
- ci
|
|
264
|
+
- test_terrorist
|
|
265
|
+
references:
|
|
266
|
+
- url: https://testing.googleblog.com/2016/05/flaky-tests-at-google-and-how-we.html
|
|
267
|
+
title: Google Testing Blog - Test Flakiness
|
|
268
|
+
- rule: Test names should describe the scenario and expected outcome
|
|
269
|
+
category: structure
|
|
270
|
+
context: Test function naming
|
|
271
|
+
antipattern: test_1, test_function, test_it_works; names that don't explain the test
|
|
272
|
+
rationale: Test names are documentation. A failing test name should tell you what broke.
|
|
273
|
+
tags:
|
|
274
|
+
- structure
|
|
275
|
+
- documentation
|
|
276
|
+
- test_terrorist
|
|
277
|
+
- naming
|
|
278
|
+
references:
|
|
279
|
+
- url: https://docs.pytest.org/en/stable/explanation/goodpractices.html
|
|
280
|
+
title: pytest - Good Integration Practices
|
|
@@ -31,10 +31,13 @@ __all__ = [
|
|
|
31
31
|
"load_seed_file",
|
|
32
32
|
"load_all_seeds",
|
|
33
33
|
"seeds_to_skills",
|
|
34
|
+
"get_package_seeds_dir",
|
|
35
|
+
"get_default_seeds_dir",
|
|
34
36
|
]
|
|
35
37
|
|
|
36
38
|
import logging
|
|
37
39
|
from dataclasses import dataclass, field
|
|
40
|
+
from importlib import resources
|
|
38
41
|
from pathlib import Path
|
|
39
42
|
from typing import Any
|
|
40
43
|
|
|
@@ -45,6 +48,53 @@ from buildlog.skills import Skill, _generate_skill_id
|
|
|
45
48
|
logger = logging.getLogger(__name__)
|
|
46
49
|
|
|
47
50
|
|
|
51
|
+
def get_package_seeds_dir() -> Path | None:
|
|
52
|
+
"""Get the path to bundled seed files in the package.
|
|
53
|
+
|
|
54
|
+
Returns:
|
|
55
|
+
Path to the package's data/seeds directory, or None if not found.
|
|
56
|
+
"""
|
|
57
|
+
try:
|
|
58
|
+
# Python 3.9+ way to get package resources
|
|
59
|
+
with resources.as_file(resources.files("buildlog").joinpath("data/seeds")) as p:
|
|
60
|
+
if p.exists():
|
|
61
|
+
return p
|
|
62
|
+
except (TypeError, FileNotFoundError):
|
|
63
|
+
pass
|
|
64
|
+
|
|
65
|
+
# Fallback: try relative to this file
|
|
66
|
+
fallback = Path(__file__).parent / "data" / "seeds"
|
|
67
|
+
if fallback.exists():
|
|
68
|
+
return fallback
|
|
69
|
+
|
|
70
|
+
return None
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def get_default_seeds_dir() -> Path | None:
|
|
74
|
+
"""Get the default seeds directory, checking multiple locations.
|
|
75
|
+
|
|
76
|
+
Priority:
|
|
77
|
+
1. Local .buildlog/seeds/ (project-specific overrides)
|
|
78
|
+
2. Local buildlog/.buildlog/seeds/ (buildlog template structure)
|
|
79
|
+
3. Package bundled seeds (installed with pip)
|
|
80
|
+
|
|
81
|
+
Returns:
|
|
82
|
+
Path to the seeds directory with most precedence, or None if none found.
|
|
83
|
+
"""
|
|
84
|
+
# Check local project seeds first (allows overrides)
|
|
85
|
+
local_seeds = Path(".buildlog") / "seeds"
|
|
86
|
+
if local_seeds.exists() and any(local_seeds.glob("*.yaml")):
|
|
87
|
+
return local_seeds
|
|
88
|
+
|
|
89
|
+
# Check buildlog template structure
|
|
90
|
+
buildlog_seeds = Path("buildlog") / ".buildlog" / "seeds"
|
|
91
|
+
if buildlog_seeds.exists() and any(buildlog_seeds.glob("*.yaml")):
|
|
92
|
+
return buildlog_seeds
|
|
93
|
+
|
|
94
|
+
# Fall back to package seeds
|
|
95
|
+
return get_package_seeds_dir()
|
|
96
|
+
|
|
97
|
+
|
|
48
98
|
@dataclass
|
|
49
99
|
class SeedReference:
|
|
50
100
|
"""A reference/citation for a seed rule."""
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|