goodturn 26.5.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
goodturn/__init__.py ADDED
@@ -0,0 +1,3 @@
1
+ """GoodTurn — harvest and contribute knowledge from AI coding sessions."""
2
+
3
+ __version__ = "26.5.0"
goodturn/__main__.py ADDED
@@ -0,0 +1,6 @@
1
+ """Support `python -m goodturn`."""
2
+
3
+ from goodturn.cli import main
4
+
5
+ if __name__ == "__main__":
6
+ main()
@@ -0,0 +1,34 @@
1
+ """Regex-based duration string parser.
2
+
3
+ Replaces ``boltons.timeutils.parse_timedelta`` with a stdlib-only
4
+ implementation that handles the patterns we actually use: ``7d``,
5
+ ``2w``, ``1d12h``, ``3h``, ``30m``.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import re
11
+ from datetime import timedelta
12
+
13
+ _DURATION_RE = re.compile(
14
+ r"(?:(\d+)w)?(?:(\d+)d)?(?:(\d+)h)?(?:(\d+)m)?$"
15
+ )
16
+
17
+
18
+ def parse_duration(s: str) -> timedelta:
19
+ """Parse a human duration string into a :class:`~datetime.timedelta`.
20
+
21
+ Supported units: weeks (``w``), days (``d``), hours (``h``),
22
+ minutes (``m``). Units may be combined: ``1d12h``.
23
+
24
+ Raises :class:`ValueError` on unparseable input.
25
+ """
26
+ s = s.strip()
27
+ m = _DURATION_RE.fullmatch(s)
28
+ if m is None or not any(m.groups()):
29
+ raise ValueError(f"cannot parse duration: {s!r}")
30
+ weeks = int(m.group(1) or 0)
31
+ days = int(m.group(2) or 0)
32
+ hours = int(m.group(3) or 0)
33
+ minutes = int(m.group(4) or 0)
34
+ return timedelta(weeks=weeks, days=days, hours=hours, minutes=minutes)
goodturn/_yaml.py ADDED
@@ -0,0 +1,111 @@
1
+ """Minimal YAML serialization and parsing for frontmatter-style metadata.
2
+
3
+ Handles only the constrained schema we produce: flat scalars, flow
4
+ sequences ``[a, b]``, and inline dicts ``{k: v}``. No full YAML parser
5
+ needed.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+
11
+ def _yaml_scalar(value: object) -> str:
12
+ """Format a scalar for YAML output. Strings are double-quoted."""
13
+ if isinstance(value, bool):
14
+ return "true" if value else "false"
15
+ if isinstance(value, (int, float)):
16
+ return str(value)
17
+ # String: double-quote, escaping internal quotes and backslashes.
18
+ s = str(value).replace("\\", "\\\\").replace('"', '\\"')
19
+ return f'"{s}"'
20
+
21
+
22
+ def _yaml_list(items: list[str]) -> str:
23
+ """Format a list as YAML flow sequence ``[a, b, c]``."""
24
+ inner = ", ".join(_yaml_scalar(i) for i in items)
25
+ return f"[{inner}]"
26
+
27
+
28
+ def _yaml_dict(d: dict[str, str]) -> str:
29
+ """Format a dict as inline YAML ``{"k": "v", ...}``."""
30
+ parts = ", ".join(
31
+ f"{_yaml_scalar(k)}: {_yaml_scalar(v)}" for k, v in d.items()
32
+ )
33
+ return f"{{{parts}}}"
34
+
35
+
36
+ # ---------------------------------------------------------------------------
37
+ # Parsing
38
+ # ---------------------------------------------------------------------------
39
+
40
+
41
+ def _parse_frontmatter(text: str) -> tuple[dict[str, str], str]:
42
+ """Parse a ``---``-delimited YAML frontmatter block from *text*.
43
+
44
+ Returns ``(metadata_dict, body)`` where metadata values are raw strings.
45
+ Only handles the constrained schema we produce: flat scalars and
46
+ bracket-delimited string lists.
47
+ """
48
+ if not text.startswith("---"):
49
+ return {}, text
50
+
51
+ # Find closing '---' (must be on its own line after the opening).
52
+ end = text.find("\n---", 3)
53
+ if end == -1:
54
+ return {}, text
55
+
56
+ fm_block = text[4:end] # skip opening '---\n'
57
+ body = text[end + 4 :] # skip '\n---'
58
+ if body.startswith("\n"):
59
+ body = body[1:]
60
+
61
+ meta: dict[str, str] = {}
62
+ for line in fm_block.strip().split("\n"):
63
+ colon = line.find(":")
64
+ if colon == -1:
65
+ continue
66
+ key = line[:colon].strip()
67
+ val = line[colon + 1 :].strip()
68
+ meta[key] = val
69
+ return meta, body
70
+
71
+
72
+ def _unquote(s: str) -> str:
73
+ """Remove surrounding double quotes and unescape."""
74
+ if len(s) >= 2 and s[0] == '"' and s[-1] == '"':
75
+ return s[1:-1].replace('\\"', '"').replace("\\\\", "\\")
76
+ return s
77
+
78
+
79
+ def _parse_yaml_list(s: str) -> list[str]:
80
+ """Parse a YAML flow sequence ``[a, b, c]`` into a list of strings."""
81
+ s = s.strip()
82
+ if not s.startswith("[") or not s.endswith("]"):
83
+ return [_unquote(s)] if s else []
84
+ inner = s[1:-1].strip()
85
+ if not inner:
86
+ return []
87
+ return [_unquote(item.strip()) for item in inner.split(",")]
88
+
89
+
90
+ def _parse_yaml_dict(s: str) -> dict[str, str]:
91
+ """Parse an inline YAML dict ``{key: "val", key2: "val2"}``.
92
+
93
+ Returns an empty dict for missing or malformed input.
94
+ """
95
+ s = s.strip()
96
+ if not s.startswith("{") or not s.endswith("}"):
97
+ return {}
98
+ inner = s[1:-1].strip()
99
+ if not inner:
100
+ return {}
101
+ result: dict[str, str] = {}
102
+ for part in inner.split(","):
103
+ part = part.strip()
104
+ colon = part.find(":")
105
+ if colon == -1:
106
+ continue
107
+ key = _unquote(part[:colon].strip())
108
+ val = _unquote(part[colon + 1 :].strip())
109
+ if key:
110
+ result[key] = val
111
+ return result
goodturn/advisory.py ADDED
@@ -0,0 +1,452 @@
1
+ """Advisory detection — scans agent sessions for security-relevant patterns."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import logging
6
+ import re
7
+ from dataclasses import dataclass, field
8
+
9
+ from .harvest_types import HarvestedPost
10
+ from .parser import Session
11
+
12
+ log = logging.getLogger(__name__)
13
+
14
+ # ---------------------------------------------------------------------------
15
+ # Known-popular packages (typosquatting baseline)
16
+ # ---------------------------------------------------------------------------
17
+
18
+ POPULAR_PIP = frozenset(
19
+ {
20
+ "requests",
21
+ "flask",
22
+ "django",
23
+ "numpy",
24
+ "pandas",
25
+ "scipy",
26
+ "tensorflow",
27
+ "torch",
28
+ "pytorch",
29
+ "fastapi",
30
+ "sqlalchemy",
31
+ "celery",
32
+ "redis",
33
+ "boto3",
34
+ "pillow",
35
+ "matplotlib",
36
+ "pytest",
37
+ "black",
38
+ "mypy",
39
+ "ruff",
40
+ "uvicorn",
41
+ "gunicorn",
42
+ "pydantic",
43
+ "httpx",
44
+ "aiohttp",
45
+ "cryptography",
46
+ "paramiko",
47
+ "beautifulsoup4",
48
+ "scrapy",
49
+ "selenium",
50
+ "ansible",
51
+ "fabric",
52
+ }
53
+ )
54
+
55
+ POPULAR_NPM = frozenset(
56
+ {
57
+ "express",
58
+ "react",
59
+ "vue",
60
+ "angular",
61
+ "next",
62
+ "nuxt",
63
+ "typescript",
64
+ "webpack",
65
+ "vite",
66
+ "eslint",
67
+ "prettier",
68
+ "axios",
69
+ "lodash",
70
+ "moment",
71
+ "dayjs",
72
+ "chalk",
73
+ "commander",
74
+ "jest",
75
+ "mocha",
76
+ "cypress",
77
+ "playwright",
78
+ "tailwindcss",
79
+ "prisma",
80
+ "sequelize",
81
+ "mongoose",
82
+ "socket.io",
83
+ "nodemon",
84
+ }
85
+ )
86
+
87
+ _POPULAR_BY_MANAGER: dict[str, frozenset[str]] = {
88
+ "pip": POPULAR_PIP,
89
+ "npm": POPULAR_NPM,
90
+ }
91
+
92
+ # ---------------------------------------------------------------------------
93
+ # Insecure code patterns
94
+ # ---------------------------------------------------------------------------
95
+
96
+ # Each entry: (compiled regex, description, severity)
97
+ _INSECURE_CODE_PATTERNS: list[tuple[re.Pattern[str], str, str]] = [
98
+ (re.compile(r"\beval\s*\("), "eval() call", "medium"),
99
+ (re.compile(r"\bexec\s*\("), "exec() call", "medium"),
100
+ (
101
+ re.compile(r"\bpickle\.loads?\s*\("),
102
+ "pickle.load(s) call — arbitrary code execution risk",
103
+ "high",
104
+ ),
105
+ (
106
+ re.compile(r"\bsubprocess\.call\s*\([^)]*shell\s*=\s*True"),
107
+ "subprocess.call with shell=True",
108
+ "high",
109
+ ),
110
+ ]
111
+
112
+ # http:// in non-localhost contexts
113
+ _INSECURE_HTTP = re.compile(
114
+ r"http://(?!localhost\b|127\.0\.0\.1\b|\[::1\]\b|0\.0\.0\.0\b)"
115
+ )
116
+
117
+ # Hardcoded credentials: KEY='...', TOKEN="...", PASSWORD = "..." etc.
118
+ _HARDCODED_CRED = re.compile(
119
+ r"""(?i)(?:api[_-]?key|secret|password|token|auth)\s*=\s*['"][A-Za-z0-9+/=_\-]{8,}['"]"""
120
+ )
121
+
122
+ # ---------------------------------------------------------------------------
123
+ # Install command extraction
124
+ # ---------------------------------------------------------------------------
125
+
126
+ # Matches: pip install pkg1 pkg2, pip3 install ..., python -m pip install ...
127
+ _PIP_INSTALL = re.compile(
128
+ r"(?:python[23]?\s+-m\s+)?pip[23]?\s+install\s+(.+)",
129
+ )
130
+ # Matches: npm install pkg, npm i pkg, yarn add pkg, pnpm add pkg
131
+ _NPM_INSTALL = re.compile(
132
+ r"(?:npm\s+(?:install|i)|yarn\s+add|pnpm\s+add)\s+(.+)",
133
+ )
134
+
135
+ # Flags that aren't package names
136
+ _INSTALL_FLAG = re.compile(r"^-")
137
+
138
+
139
+ def _extract_install_commands(session: Session) -> list[tuple[str, str, str]]:
140
+ """Extract (package_manager, package_name, full_command) from bash tool calls."""
141
+ results: list[tuple[str, str, str]] = []
142
+ for turn in session.turns:
143
+ for tc in turn.tool_calls:
144
+ # Tool calls may store the command in different shapes depending
145
+ # on the parser. Handle both dict-with-command and raw string.
146
+ command = _extract_bash_command(tc)
147
+ if not command:
148
+ continue
149
+ for line in command.splitlines():
150
+ line = line.strip()
151
+ _match_install_line(line, results)
152
+ return results
153
+
154
+
155
+ def _extract_bash_command(tc: dict) -> str | None:
156
+ """Pull the shell command string from a tool-call dict, if it's a bash call."""
157
+ name = tc.get("name", tc.get("tool", ""))
158
+ if name not in ("bash", "Bash", "execute_bash", "proxy_bash"):
159
+ return None
160
+ inp = tc.get("input", tc.get("arguments", {}))
161
+ if isinstance(inp, str):
162
+ return inp
163
+ if isinstance(inp, dict):
164
+ return inp.get("command", inp.get("cmd", ""))
165
+ return None
166
+
167
+
168
+ def _match_install_line(
169
+ line: str,
170
+ out: list[tuple[str, str, str]],
171
+ ) -> None:
172
+ """Match a single shell line against pip/npm install patterns."""
173
+ for pattern, manager in [(_PIP_INSTALL, "pip"), (_NPM_INSTALL, "npm")]:
174
+ m = pattern.search(line)
175
+ if not m:
176
+ continue
177
+ args_str = m.group(1)
178
+ for token in args_str.split():
179
+ if _INSTALL_FLAG.match(token):
180
+ continue
181
+ # Strip version specifiers: pkg==1.0, pkg>=2, pkg@latest
182
+ pkg = re.split(r"[=@><~!]", token, maxsplit=1)[0]
183
+ if pkg:
184
+ out.append((manager, pkg, line))
185
+
186
+
187
+ # ---------------------------------------------------------------------------
188
+ # Levenshtein distance
189
+ # ---------------------------------------------------------------------------
190
+
191
+
192
+ def _levenshtein_distance(s1: str, s2: str) -> int:
193
+ """Standard dynamic-programming edit distance."""
194
+ if len(s1) < len(s2):
195
+ return _levenshtein_distance(s2, s1)
196
+ if not s2:
197
+ return len(s1)
198
+ prev = list(range(len(s2) + 1))
199
+ for i, c1 in enumerate(s1):
200
+ curr = [i + 1]
201
+ for j, c2 in enumerate(s2):
202
+ # Substitution / match costs
203
+ cost = 0 if c1 == c2 else 1
204
+ curr.append(
205
+ min(
206
+ curr[j] + 1, # insert
207
+ prev[j + 1] + 1, # delete
208
+ prev[j] + cost, # substitute
209
+ )
210
+ )
211
+ prev = curr
212
+ return prev[-1]
213
+
214
+
215
+ # ---------------------------------------------------------------------------
216
+ # Suspicious package heuristic
217
+ # ---------------------------------------------------------------------------
218
+
219
+
220
+ def _looks_suspicious(name: str) -> bool:
221
+ """Heuristic: very short or random-looking package names are suspicious."""
222
+ if len(name) <= 2:
223
+ return True
224
+ # High consonant-to-vowel ratio → random-looking
225
+ vowels = sum(1 for c in name.lower() if c in "aeiou")
226
+ if len(name) >= 5 and vowels == 0:
227
+ return True
228
+ return False
229
+
230
+
231
+ # ---------------------------------------------------------------------------
232
+ # Detection result (internal, converted to HarvestedPost)
233
+ # ---------------------------------------------------------------------------
234
+
235
+
236
+ @dataclass
237
+ class _Finding:
238
+ title: str
239
+ body: str
240
+ severity: str # 'low' | 'medium' | 'high'
241
+ rule: str
242
+ tags: list[str] = field(default_factory=list)
243
+ affected_packages: list[str] = field(default_factory=list)
244
+ language: str = ""
245
+ confidence: float = 0.5
246
+
247
+
248
+ # ---------------------------------------------------------------------------
249
+ # AdvisoryDetector
250
+ # ---------------------------------------------------------------------------
251
+
252
+
253
+ class AdvisoryDetector:
254
+ """Scan sessions for security-relevant patterns and produce advisory posts."""
255
+
256
+ def __init__(
257
+ self,
258
+ model: str = "anthropic/claude-sonnet-4-5",
259
+ api_key: str | None = None,
260
+ ):
261
+ self.model = model
262
+ self.api_key = api_key
263
+
264
+ def detect(self, session: Session) -> list[HarvestedPost]:
265
+ """Scan session for advisory-worthy security patterns."""
266
+ findings: list[_Finding] = []
267
+ self._detect_typosquatting(session, findings)
268
+ self._detect_suspicious_packages(session, findings)
269
+ self._detect_insecure_code(session, findings)
270
+ self._detect_insecure_http(session, findings)
271
+ self._detect_hardcoded_credentials(session, findings)
272
+
273
+ return [self._to_post(f, session) for f in findings]
274
+
275
+ # -- pattern detectors ---------------------------------------------------
276
+
277
+ def _detect_typosquatting(
278
+ self,
279
+ session: Session,
280
+ out: list[_Finding],
281
+ ) -> None:
282
+ installs = _extract_install_commands(session)
283
+ for manager, pkg, full_cmd in installs:
284
+ popular = _POPULAR_BY_MANAGER.get(manager, frozenset())
285
+ if pkg in popular:
286
+ continue # exact match, not a typosquat
287
+ for known in popular:
288
+ dist = _levenshtein_distance(pkg.lower(), known)
289
+ if 1 <= dist <= 2:
290
+ out.append(
291
+ _Finding(
292
+ title=f"Possible typosquat: `{pkg}` (close to `{known}`)",
293
+ body=(
294
+ f"The command `{full_cmd}` installs `{pkg}`, "
295
+ f"which is edit-distance {dist} from the popular "
296
+ f"package `{known}`. This could be a typosquatting "
297
+ f"attack.\n\nVerify the intended package name."
298
+ ),
299
+ severity="high",
300
+ rule="typosquatting",
301
+ tags=[manager, pkg, known],
302
+ affected_packages=[pkg],
303
+ confidence=0.7 if dist == 1 else 0.5,
304
+ )
305
+ )
306
+ break # one match per installed package is enough
307
+
308
+ def _detect_suspicious_packages(
309
+ self,
310
+ session: Session,
311
+ out: list[_Finding],
312
+ ) -> None:
313
+ installs = _extract_install_commands(session)
314
+ for manager, pkg, full_cmd in installs:
315
+ popular = _POPULAR_BY_MANAGER.get(manager, frozenset())
316
+ if pkg in popular:
317
+ continue
318
+ if _looks_suspicious(pkg):
319
+ out.append(
320
+ _Finding(
321
+ title=f"Suspicious package name: `{pkg}`",
322
+ body=(
323
+ f"The command `{full_cmd}` installs `{pkg}`, "
324
+ f"which has a suspicious-looking name "
325
+ f"(very short or random characters). "
326
+ f"Verify this is a legitimate package."
327
+ ),
328
+ severity="medium",
329
+ rule="suspicious_package",
330
+ tags=[manager, pkg],
331
+ affected_packages=[pkg],
332
+ confidence=0.4,
333
+ )
334
+ )
335
+
336
+ def _detect_insecure_code(
337
+ self,
338
+ session: Session,
339
+ out: list[_Finding],
340
+ ) -> None:
341
+ for turn in session.turns:
342
+ if not turn.content:
343
+ continue
344
+ for pattern, desc, severity in _INSECURE_CODE_PATTERNS:
345
+ if pattern.search(turn.content):
346
+ out.append(
347
+ _Finding(
348
+ title=f"Insecure pattern: {desc}",
349
+ body=(
350
+ f"Detected `{desc}` in session content. "
351
+ f"This pattern can lead to code execution "
352
+ f"vulnerabilities if used with untrusted input."
353
+ ),
354
+ severity=severity,
355
+ rule="insecure_pattern",
356
+ tags=["security", desc.split("(")[0].strip()],
357
+ confidence=0.6,
358
+ )
359
+ )
360
+ break # one finding per turn per pattern category
361
+
362
+ def _detect_insecure_http(
363
+ self,
364
+ session: Session,
365
+ out: list[_Finding],
366
+ ) -> None:
367
+ seen_urls: set[str] = set()
368
+ for turn in session.turns:
369
+ if not turn.content:
370
+ continue
371
+ for m in _INSECURE_HTTP.finditer(turn.content):
372
+ # Grab enough context for the URL
373
+ start = m.start()
374
+ end = turn.content.find(" ", start)
375
+ if end == -1:
376
+ end = min(start + 120, len(turn.content))
377
+ url = turn.content[start:end].rstrip(".,;:)'\"")
378
+ if url in seen_urls:
379
+ continue
380
+ seen_urls.add(url)
381
+ out.append(
382
+ _Finding(
383
+ title=f"Insecure HTTP URL: `{url[:80]}`",
384
+ body=(
385
+ f"Found non-HTTPS URL `{url}` in session content. "
386
+ f"Use `https://` to prevent man-in-the-middle attacks."
387
+ ),
388
+ severity="low",
389
+ rule="insecure_http",
390
+ tags=["security", "http"],
391
+ confidence=0.5,
392
+ )
393
+ )
394
+
395
+ def _detect_hardcoded_credentials(
396
+ self,
397
+ session: Session,
398
+ out: list[_Finding],
399
+ ) -> None:
400
+ seen: set[str] = set()
401
+ for turn in session.turns:
402
+ if not turn.content:
403
+ continue
404
+ for m in _HARDCODED_CRED.finditer(turn.content):
405
+ matched = m.group(0)
406
+ if matched in seen:
407
+ continue
408
+ seen.add(matched)
409
+ # Redact the value in the advisory
410
+ redacted = re.sub(
411
+ r"""(['"])[A-Za-z0-9+/=_\-]{8,}(['"])""",
412
+ r"\1<REDACTED>\2",
413
+ matched,
414
+ )
415
+ out.append(
416
+ _Finding(
417
+ title="Hardcoded credential detected",
418
+ body=(
419
+ f"Found what looks like a hardcoded credential: "
420
+ f"`{redacted}`. Hardcoded secrets should be replaced "
421
+ f"with environment variables or a secrets manager."
422
+ ),
423
+ severity="high",
424
+ rule="hardcoded_credential",
425
+ tags=["security", "credentials"],
426
+ confidence=0.6,
427
+ )
428
+ )
429
+
430
+ # -- conversion ----------------------------------------------------------
431
+
432
+ @staticmethod
433
+ def _to_post(finding: _Finding, session: Session) -> HarvestedPost:
434
+ return HarvestedPost(
435
+ title=finding.title,
436
+ body_md=finding.body,
437
+ post_kind="advisory",
438
+ summary=finding.title,
439
+ tags=finding.tags,
440
+ language=finding.language,
441
+ kind_metadata={
442
+ "severity": finding.severity,
443
+ "affected_packages": finding.affected_packages,
444
+ "detection_rule": finding.rule,
445
+ },
446
+ confidence=finding.confidence,
447
+ source="harvested",
448
+ source_session_ids=[session.session_id],
449
+ extractor="advisory_detector",
450
+ extraction_confidence=finding.confidence,
451
+ session_models=[session.model] if session.model else [],
452
+ )
goodturn/cli.py ADDED
@@ -0,0 +1,32 @@
1
+ """Root CLI entry point for the goodturn command."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import sys
6
+ import face
7
+
8
+
9
+ cmd = face.Command(name="goodturn", func=None)
10
+
11
+ # Register subcommands — each module provides a face.Command
12
+ from .cmd_harvest import harvest_cmd
13
+ from .cmd_eval import eval_cmd
14
+ from .cmd_publish import publish_cmd
15
+ from .cmd_review import review_cmd
16
+ from .cmd_status import status_cmd
17
+ from .cmd_outbox import outbox_cmd
18
+
19
+ cmd.add(harvest_cmd)
20
+ cmd.add(eval_cmd)
21
+ cmd.add(publish_cmd)
22
+ cmd.add(review_cmd)
23
+ cmd.add(status_cmd)
24
+ cmd.add(outbox_cmd)
25
+
26
+
27
+ def main(argv=None):
28
+ """Entry point for `goodturn` CLI."""
29
+ try:
30
+ sys.exit(cmd.run(argv or sys.argv) or 0)
31
+ except KeyboardInterrupt:
32
+ sys.exit(130)