open-code-review-toolkit 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ocr_toolkit/__init__.py +1 -0
- ocr_toolkit/_version.py +24 -0
- ocr_toolkit/cli.py +56 -0
- ocr_toolkit/common/__init__.py +1 -0
- ocr_toolkit/common/language.py +60 -0
- ocr_toolkit/common/markdown.py +214 -0
- ocr_toolkit/common/redaction.py +324 -0
- ocr_toolkit/config_writer.py +106 -0
- ocr_toolkit/configure.py +194 -0
- ocr_toolkit/context/__init__.py +1 -0
- ocr_toolkit/context/__main__.py +8 -0
- ocr_toolkit/context/ansible.py +561 -0
- ocr_toolkit/context/categorize.py +162 -0
- ocr_toolkit/context/instructions.py +269 -0
- ocr_toolkit/context/manifests.py +459 -0
- ocr_toolkit/context/planner.py +227 -0
- ocr_toolkit/context/render.py +955 -0
- ocr_toolkit/context/repo.py +672 -0
- ocr_toolkit/context/settings.py +97 -0
- ocr_toolkit/mcp_config.py +257 -0
- ocr_toolkit/posting/__init__.py +1 -0
- ocr_toolkit/posting/__main__.py +8 -0
- ocr_toolkit/posting/comments.py +87 -0
- ocr_toolkit/posting/formatting.py +755 -0
- ocr_toolkit/posting/gitlab.py +853 -0
- ocr_toolkit/posting/markers.py +284 -0
- ocr_toolkit/posting/payloads.py +141 -0
- ocr_toolkit/posting/result.py +116 -0
- ocr_toolkit/posting/settings.py +181 -0
- ocr_toolkit/posting/snapshot.py +468 -0
- ocr_toolkit/posting/workflow.py +873 -0
- ocr_toolkit/preflight.py +395 -0
- ocr_toolkit/py.typed +1 -0
- open_code_review_toolkit-0.1.0.dist-info/METADATA +283 -0
- open_code_review_toolkit-0.1.0.dist-info/RECORD +38 -0
- open_code_review_toolkit-0.1.0.dist-info/WHEEL +4 -0
- open_code_review_toolkit-0.1.0.dist-info/entry_points.txt +2 -0
- open_code_review_toolkit-0.1.0.dist-info/licenses/LICENSE +202 -0
ocr_toolkit/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Open Code Review CI helper package."""
|
ocr_toolkit/_version.py
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
# file generated by vcs-versioning
|
|
2
|
+
# don't change, don't track in version control
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
__all__ = [
|
|
6
|
+
"__version__",
|
|
7
|
+
"__version_tuple__",
|
|
8
|
+
"version",
|
|
9
|
+
"version_tuple",
|
|
10
|
+
"__commit_id__",
|
|
11
|
+
"commit_id",
|
|
12
|
+
]
|
|
13
|
+
|
|
14
|
+
version: str
|
|
15
|
+
__version__: str
|
|
16
|
+
__version_tuple__: tuple[int | str, ...]
|
|
17
|
+
version_tuple: tuple[int | str, ...]
|
|
18
|
+
commit_id: str | None
|
|
19
|
+
__commit_id__: str | None
|
|
20
|
+
|
|
21
|
+
__version__ = version = '0.1.0'
|
|
22
|
+
__version_tuple__ = version_tuple = (0, 1, 0)
|
|
23
|
+
|
|
24
|
+
__commit_id__ = commit_id = None
|
ocr_toolkit/cli.py
ADDED
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
"""Unified command-line interface for Open Code Review CI helpers."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
from collections.abc import Sequence
|
|
7
|
+
|
|
8
|
+
from ocr_toolkit import configure, mcp_config, preflight
|
|
9
|
+
from ocr_toolkit.context import render as context_render
|
|
10
|
+
from ocr_toolkit.posting.workflow import main as posting_main
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
14
|
+
"""Build the stable top-level parser and subcommand surface."""
|
|
15
|
+
|
|
16
|
+
parser = argparse.ArgumentParser(
|
|
17
|
+
prog="ocr-ci",
|
|
18
|
+
description="Safe CI integration helpers for Open Code Review.",
|
|
19
|
+
)
|
|
20
|
+
subparsers = parser.add_subparsers(dest="command", required=True)
|
|
21
|
+
subparsers.add_parser("preflight", help="Validate OCR, GitLab, and LLM access.")
|
|
22
|
+
subparsers.add_parser("configure", help="Write the OCR runtime configuration.")
|
|
23
|
+
subparsers.add_parser("mcp-config", help="Write OCR MCP server configuration.")
|
|
24
|
+
|
|
25
|
+
context_parser = subparsers.add_parser(
|
|
26
|
+
"context", help="Generate bounded repository review context."
|
|
27
|
+
)
|
|
28
|
+
context_parser.add_argument(
|
|
29
|
+
"--output",
|
|
30
|
+
default=".review-context/dependencies.md",
|
|
31
|
+
help="Output markdown path.",
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
post_parser = subparsers.add_parser("post", help="Publish an OCR result artifact to GitLab.")
|
|
35
|
+
post_parser.add_argument(
|
|
36
|
+
"--result", default="/tmp/ocr-result.json", help="OCR JSON result path."
|
|
37
|
+
)
|
|
38
|
+
post_parser.add_argument("--stderr", default="/tmp/ocr-stderr.log", help="OCR stderr log path.")
|
|
39
|
+
return parser
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def main(argv: Sequence[str] | None = None) -> int:
|
|
43
|
+
"""Dispatch one public subcommand and return its process exit code."""
|
|
44
|
+
|
|
45
|
+
args = build_parser().parse_args(argv)
|
|
46
|
+
if args.command == "preflight":
|
|
47
|
+
return preflight.main()
|
|
48
|
+
if args.command == "configure":
|
|
49
|
+
return configure.main()
|
|
50
|
+
if args.command == "mcp-config":
|
|
51
|
+
return mcp_config.configure_mcp_servers()
|
|
52
|
+
if args.command == "context":
|
|
53
|
+
return context_render.main(["--output", args.output])
|
|
54
|
+
if args.command == "post":
|
|
55
|
+
return posting_main([args.result, args.stderr])
|
|
56
|
+
raise AssertionError(f"unhandled command: {args.command}")
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Open Code Review CI helper package."""
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
"""Resolve the single public language setting used by OCR review flows."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
import re
|
|
7
|
+
import sys
|
|
8
|
+
|
|
9
|
+
DEFAULT_REVIEW_LANGUAGE = "English"
|
|
10
|
+
REVIEW_LANGUAGE_ENV = "OCR_REVIEW_LANGUAGE"
|
|
11
|
+
|
|
12
|
+
KNOWN_LANGUAGES = frozenset(
|
|
13
|
+
{
|
|
14
|
+
"arabic",
|
|
15
|
+
"chinese",
|
|
16
|
+
"czech",
|
|
17
|
+
"danish",
|
|
18
|
+
"dutch",
|
|
19
|
+
"english",
|
|
20
|
+
"english (uk)",
|
|
21
|
+
"english (us)",
|
|
22
|
+
"finnish",
|
|
23
|
+
"french",
|
|
24
|
+
"german",
|
|
25
|
+
"hebrew",
|
|
26
|
+
"hindi",
|
|
27
|
+
"indonesian",
|
|
28
|
+
"italian",
|
|
29
|
+
"japanese",
|
|
30
|
+
"korean",
|
|
31
|
+
"norwegian",
|
|
32
|
+
"polish",
|
|
33
|
+
"portuguese",
|
|
34
|
+
"russian",
|
|
35
|
+
"spanish",
|
|
36
|
+
"swedish",
|
|
37
|
+
"thai",
|
|
38
|
+
"turkish",
|
|
39
|
+
"ukrainian",
|
|
40
|
+
"vietnamese",
|
|
41
|
+
}
|
|
42
|
+
)
|
|
43
|
+
BCP47_RE = re.compile(r"^[A-Za-z]{2,3}(?:-[A-Za-z0-9]{2,8}){0,3}$")
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def resolve_review_language(value: str | None = None) -> str:
|
|
47
|
+
"""Return one safe language label for OCR config and review context."""
|
|
48
|
+
|
|
49
|
+
raw = (os.environ.get(REVIEW_LANGUAGE_ENV, "") if value is None else value).strip()
|
|
50
|
+
if not raw:
|
|
51
|
+
return DEFAULT_REVIEW_LANGUAGE
|
|
52
|
+
if raw.lower() in KNOWN_LANGUAGES or BCP47_RE.fullmatch(raw):
|
|
53
|
+
return raw
|
|
54
|
+
|
|
55
|
+
print(
|
|
56
|
+
f"{REVIEW_LANGUAGE_ENV} is not an allowed language label or BCP-47 tag; "
|
|
57
|
+
f"falling back to {DEFAULT_REVIEW_LANGUAGE}.",
|
|
58
|
+
file=sys.stderr,
|
|
59
|
+
)
|
|
60
|
+
return DEFAULT_REVIEW_LANGUAGE
|
|
@@ -0,0 +1,214 @@
|
|
|
1
|
+
"""Markdown formatting helpers for OCR CI output."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import re
|
|
6
|
+
import unicodedata
|
|
7
|
+
|
|
8
|
+
SAFE_CODE_BLOCK_LANGUAGE_RE = re.compile(r"^[A-Za-z0-9_+.#-]{1,40}$")
|
|
9
|
+
FENCE_LINE_RE = re.compile(r"^ {0,3}(`{3,}|~{3,})([^\n]*)$")
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def markdown_fence_transition(line: str, open_fence: str | None) -> tuple[str | None, bool]:
|
|
13
|
+
"""Return updated fenced-code marker and whether this line is a fence."""
|
|
14
|
+
|
|
15
|
+
match = FENCE_LINE_RE.match(line)
|
|
16
|
+
if not match:
|
|
17
|
+
return open_fence, False
|
|
18
|
+
|
|
19
|
+
fence = match.group(1)
|
|
20
|
+
suffix = match.group(2)
|
|
21
|
+
marker = fence[0]
|
|
22
|
+
if open_fence is None:
|
|
23
|
+
if marker == "`" and "`" in suffix:
|
|
24
|
+
return open_fence, False
|
|
25
|
+
return marker * len(fence), True
|
|
26
|
+
|
|
27
|
+
if marker != open_fence[0] or len(fence) < len(open_fence):
|
|
28
|
+
return open_fence, False
|
|
29
|
+
if suffix.strip():
|
|
30
|
+
return open_fence, False
|
|
31
|
+
return None, True
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def open_markdown_fence(markdown: str) -> str | None:
|
|
35
|
+
"""Return the currently open CommonMark fenced-code marker, if any."""
|
|
36
|
+
|
|
37
|
+
open_fence: str | None = None
|
|
38
|
+
for line in markdown.splitlines():
|
|
39
|
+
open_fence, _ = markdown_fence_transition(line, open_fence)
|
|
40
|
+
return open_fence
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def neutralize_suggestion_fences(text: str) -> str:
|
|
44
|
+
"""Prevent OCR-controlled prose from creating GitLab suggestion blocks."""
|
|
45
|
+
|
|
46
|
+
neutralized: list[str] = []
|
|
47
|
+
for raw_line in text.splitlines(keepends=True):
|
|
48
|
+
line = raw_line.rstrip("\r\n")
|
|
49
|
+
line_ending = raw_line[len(line) :]
|
|
50
|
+
stripped = line.lstrip(" ")
|
|
51
|
+
indent = line[: len(line) - len(stripped)]
|
|
52
|
+
match = FENCE_LINE_RE.match(line)
|
|
53
|
+
if (
|
|
54
|
+
len(indent) <= 3
|
|
55
|
+
and match
|
|
56
|
+
and match.group(1).startswith("`")
|
|
57
|
+
and "`" not in match.group(2)
|
|
58
|
+
and match.group(2).lstrip().lower().startswith("suggestion")
|
|
59
|
+
):
|
|
60
|
+
neutralized.append(f"{indent}{match.group(1)}text{line_ending}")
|
|
61
|
+
else:
|
|
62
|
+
neutralized.append(raw_line)
|
|
63
|
+
return "".join(neutralized)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def max_backtick_run(text: str) -> int:
|
|
67
|
+
"""Return the longest consecutive backtick run in text."""
|
|
68
|
+
|
|
69
|
+
longest = 0
|
|
70
|
+
current = 0
|
|
71
|
+
|
|
72
|
+
for char in text:
|
|
73
|
+
if char == "`":
|
|
74
|
+
current += 1
|
|
75
|
+
longest = max(longest, current)
|
|
76
|
+
else:
|
|
77
|
+
current = 0
|
|
78
|
+
|
|
79
|
+
return longest
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def escape_code_block_control_chars(value: str) -> str:
|
|
83
|
+
"""Escape invisible controls while preserving code-block layout."""
|
|
84
|
+
|
|
85
|
+
escaped: list[str] = []
|
|
86
|
+
for char in value:
|
|
87
|
+
codepoint = ord(char)
|
|
88
|
+
if char in {"\n", "\t"}:
|
|
89
|
+
escaped.append(char)
|
|
90
|
+
continue
|
|
91
|
+
category = unicodedata.category(char)
|
|
92
|
+
if category == "Cc":
|
|
93
|
+
escaped.append(f"\\x{codepoint:02x}")
|
|
94
|
+
elif category in {"Cf", "Zl", "Zp"}:
|
|
95
|
+
if codepoint <= 0xFFFF:
|
|
96
|
+
escaped.append(f"\\u{codepoint:04x}")
|
|
97
|
+
else:
|
|
98
|
+
escaped.append(f"\\U{codepoint:08x}")
|
|
99
|
+
else:
|
|
100
|
+
escaped.append(char)
|
|
101
|
+
|
|
102
|
+
return "".join(escaped)
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def markdown_code_block(language: str, content: str, escape_controls: bool = True) -> str:
|
|
106
|
+
"""Return a Markdown code block safe for embedded fences."""
|
|
107
|
+
|
|
108
|
+
if escape_controls:
|
|
109
|
+
content = escape_code_block_control_chars(content)
|
|
110
|
+
|
|
111
|
+
fence_len = max(3, max_backtick_run(content) + 1)
|
|
112
|
+
fence = "`" * fence_len
|
|
113
|
+
lang = language.strip()
|
|
114
|
+
if not SAFE_CODE_BLOCK_LANGUAGE_RE.fullmatch(lang):
|
|
115
|
+
lang = ""
|
|
116
|
+
|
|
117
|
+
closing_separator = "" if content.endswith("\n") else "\n"
|
|
118
|
+
if lang:
|
|
119
|
+
return f"{fence}{lang}\n{content}{closing_separator}{fence}"
|
|
120
|
+
|
|
121
|
+
return f"{fence}\n{content}{closing_separator}{fence}"
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def escape_control_chars(value: str) -> str:
|
|
125
|
+
"""Return text with control/format characters escaped for Markdown output.
|
|
126
|
+
|
|
127
|
+
Unicode control characters (category ``Cc``), format controls
|
|
128
|
+
(category ``Cf``), and Unicode line/paragraph separators (``Zl``/``Zp``)
|
|
129
|
+
can make rendered Markdown differ from the underlying value. Keep ordinary
|
|
130
|
+
printable Unicode unchanged.
|
|
131
|
+
"""
|
|
132
|
+
|
|
133
|
+
escaped: list[str] = []
|
|
134
|
+
for char in value:
|
|
135
|
+
codepoint = ord(char)
|
|
136
|
+
if char == "\n":
|
|
137
|
+
escaped.append("\\n")
|
|
138
|
+
continue
|
|
139
|
+
elif char == "\r":
|
|
140
|
+
escaped.append("\\r")
|
|
141
|
+
continue
|
|
142
|
+
elif char == "\t":
|
|
143
|
+
escaped.append("\\t")
|
|
144
|
+
continue
|
|
145
|
+
category = unicodedata.category(char)
|
|
146
|
+
if category == "Cc":
|
|
147
|
+
escaped.append(f"\\x{codepoint:02x}")
|
|
148
|
+
elif category in {"Cf", "Zl", "Zp"}:
|
|
149
|
+
if codepoint <= 0xFFFF:
|
|
150
|
+
escaped.append(f"\\u{codepoint:04x}")
|
|
151
|
+
else:
|
|
152
|
+
escaped.append(f"\\U{codepoint:08x}")
|
|
153
|
+
else:
|
|
154
|
+
escaped.append(char)
|
|
155
|
+
|
|
156
|
+
return "".join(escaped)
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def inline_code(value: str, escape_controls: bool = True) -> str:
|
|
160
|
+
"""Return a Markdown inline-code representation safe for backticks.
|
|
161
|
+
|
|
162
|
+
Control/format escaping is the default because most callers render
|
|
163
|
+
MR-controlled paths, CI values, manifest strings, or OCR output. Callers
|
|
164
|
+
that render compile-time literal identifiers may opt out explicitly.
|
|
165
|
+
|
|
166
|
+
An empty input is rendered as an italic ``_(unset)_`` so a missing
|
|
167
|
+
branch name or commit SHA is visually obvious instead of collapsing
|
|
168
|
+
into two adjacent backticks that GitLab renders ambiguously.
|
|
169
|
+
"""
|
|
170
|
+
|
|
171
|
+
if not value:
|
|
172
|
+
return "_(unset)_"
|
|
173
|
+
|
|
174
|
+
if escape_controls:
|
|
175
|
+
value = escape_control_chars(value)
|
|
176
|
+
|
|
177
|
+
padded_span = bool(value.strip()) and value.startswith(" ") and value.endswith(" ")
|
|
178
|
+
if "`" not in value and not padded_span:
|
|
179
|
+
return f"`{value}`"
|
|
180
|
+
|
|
181
|
+
fence = "`" * (max_backtick_run(value) + 1)
|
|
182
|
+
return f"{fence} {value} {fence}"
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def neutralize_quick_actions(text: str) -> str:
|
|
186
|
+
"""Prevent GitLab quick actions from executing in bot-created notes.
|
|
187
|
+
|
|
188
|
+
GitLab treats lines starting with '/' as quick actions in comments.
|
|
189
|
+
Prefix such lines with a backslash so the text remains readable but
|
|
190
|
+
is not executed. This is a raw GitLab-note safety boundary, so it
|
|
191
|
+
intentionally escapes slash-prefixed lines even inside Markdown code
|
|
192
|
+
blocks rather than relying on GitLab quick-action parsing to honor
|
|
193
|
+
the same Markdown boundaries as this local parser.
|
|
194
|
+
"""
|
|
195
|
+
|
|
196
|
+
neutralized_lines: list[str] = []
|
|
197
|
+
open_fence: str | None = None # currently open fence marker, or None
|
|
198
|
+
for raw_line in text.splitlines(keepends=True):
|
|
199
|
+
line = raw_line.rstrip("\r\n")
|
|
200
|
+
line_ending = raw_line[len(line) :]
|
|
201
|
+
next_open_fence, is_fence = markdown_fence_transition(line, open_fence)
|
|
202
|
+
if is_fence:
|
|
203
|
+
open_fence = next_open_fence
|
|
204
|
+
neutralized_lines.append(line + line_ending)
|
|
205
|
+
continue
|
|
206
|
+
|
|
207
|
+
stripped = line.lstrip()
|
|
208
|
+
if stripped.startswith("/"):
|
|
209
|
+
leading = line[: len(line) - len(stripped)]
|
|
210
|
+
neutralized_lines.append(f"{leading}\\{stripped}{line_ending}")
|
|
211
|
+
else:
|
|
212
|
+
neutralized_lines.append(line + line_ending)
|
|
213
|
+
|
|
214
|
+
return "".join(neutralized_lines)
|
|
@@ -0,0 +1,324 @@
|
|
|
1
|
+
"""Redaction helpers for data sent to logs, GitLab, or LLM context."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
import re
|
|
7
|
+
import unicodedata
|
|
8
|
+
import urllib.parse
|
|
9
|
+
from typing import Any
|
|
10
|
+
|
|
11
|
+
SENSITIVE_KEY_PATTERN = (
|
|
12
|
+
r"x-api-key|api[_-]?key|access[_-]?token|auth[_-]?token|"
|
|
13
|
+
r"secret[_-]?key|refresh[_-]?token|private[_-]?token|"
|
|
14
|
+
r"client[_-]?secret|aws[_-]?secret[_-]?access[_-]?key|password"
|
|
15
|
+
)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
SENSITIVE_NAMED_KEY_PATTERN = (
|
|
19
|
+
rf"(?:(?:[A-Za-z0-9]+[_.-])*(?:{SENSITIVE_KEY_PATTERN})|"
|
|
20
|
+
r"(?:[A-Za-z0-9]+[_.-])*token)"
|
|
21
|
+
)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
SENSITIVE_ENV_NAMES = (
|
|
25
|
+
"OCR_LLM_TOKEN",
|
|
26
|
+
"OCR_LLM_AUTH_TOKEN",
|
|
27
|
+
"OCR_LLM_EXTRA_HEADERS",
|
|
28
|
+
"GITLAB_API_TOKEN",
|
|
29
|
+
"ANTHROPIC_API_KEY",
|
|
30
|
+
"OPENAI_API_KEY",
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
SECRET_SHAPED_ENV_NAME_RE = re.compile(
|
|
34
|
+
r"(?:^|_)(?:API_?KEY|AUTH_?TOKEN|ACCESS_?TOKEN|BEARER|CREDENTIAL|"
|
|
35
|
+
r"PASSWORD|PRIVATE_?KEY|SECRET|TOKEN)(?:$|_)",
|
|
36
|
+
flags=re.IGNORECASE,
|
|
37
|
+
)
|
|
38
|
+
MIN_DISCOVERED_SECRET_LENGTH = 16
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
REDACTION_TOKEN_SEPARATOR = r"[-_\s\u200b-\u200f\u202a-\u202e\u2060-\u206f]*"
|
|
42
|
+
REDACTION_TOKEN_ALIASES = {
|
|
43
|
+
"authorization": "authorization",
|
|
44
|
+
"proxyauthorization": "proxy-authorization",
|
|
45
|
+
"bearer": "bearer",
|
|
46
|
+
"xapikey": "x-api-key",
|
|
47
|
+
"apikey": "api_key",
|
|
48
|
+
"accesstoken": "access_token",
|
|
49
|
+
"authtoken": "auth_token",
|
|
50
|
+
"secretkey": "secret_key",
|
|
51
|
+
"refreshtoken": "refresh_token",
|
|
52
|
+
"privatetoken": "private_token",
|
|
53
|
+
"clientsecret": "client_secret",
|
|
54
|
+
"password": "password",
|
|
55
|
+
}
|
|
56
|
+
REDACTION_TOKEN_PATTERNS = tuple(
|
|
57
|
+
(
|
|
58
|
+
re.compile(
|
|
59
|
+
r"(?i)(?<![A-Za-z0-9])"
|
|
60
|
+
+ REDACTION_TOKEN_SEPARATOR.join(re.escape(char) for char in compact)
|
|
61
|
+
+ r"(?![A-Za-z0-9])"
|
|
62
|
+
),
|
|
63
|
+
canonical,
|
|
64
|
+
)
|
|
65
|
+
for compact, canonical in REDACTION_TOKEN_ALIASES.items()
|
|
66
|
+
)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def normalize_redaction_tokens(text: str) -> str:
|
|
70
|
+
"""Collapse invisible separators only inside sensitive token names.
|
|
71
|
+
|
|
72
|
+
Preserving normal line breaks keeps diagnostics readable and prevents an
|
|
73
|
+
``Authorization`` header sweep from consuming subsequent Markdown lines.
|
|
74
|
+
"""
|
|
75
|
+
|
|
76
|
+
normalized = text
|
|
77
|
+
for pattern, canonical in REDACTION_TOKEN_PATTERNS:
|
|
78
|
+
normalized = pattern.sub(
|
|
79
|
+
lambda match: (
|
|
80
|
+
canonical
|
|
81
|
+
if any(
|
|
82
|
+
char.isspace() or unicodedata.category(char) in {"Cf", "Zl", "Zp"}
|
|
83
|
+
for char in match.group(0)
|
|
84
|
+
)
|
|
85
|
+
else match.group(0)
|
|
86
|
+
),
|
|
87
|
+
normalized,
|
|
88
|
+
)
|
|
89
|
+
return normalized
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def strip_redaction_bypass_controls(text: str) -> str:
|
|
93
|
+
"""Remove invisible controls that can split secret keys or values."""
|
|
94
|
+
|
|
95
|
+
return "".join(
|
|
96
|
+
char
|
|
97
|
+
for char in text
|
|
98
|
+
if unicodedata.category(char) not in {"Cc", "Cf", "Zl", "Zp"} or char in "\n\r\t"
|
|
99
|
+
)
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def strip_path_controls(text: str) -> str:
|
|
103
|
+
"""Remove every Unicode control/format separator from a display path."""
|
|
104
|
+
|
|
105
|
+
return "".join(
|
|
106
|
+
char for char in text if unicodedata.category(char) not in {"Cc", "Cf", "Cs", "Zl", "Zp"}
|
|
107
|
+
)
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def redact_env_secret_values(text: str) -> str:
|
|
111
|
+
"""Replace configured secret values without generic key/value sweeps."""
|
|
112
|
+
|
|
113
|
+
if not text:
|
|
114
|
+
return text
|
|
115
|
+
|
|
116
|
+
redacted = text
|
|
117
|
+
variants: set[str] = set()
|
|
118
|
+
candidate_names = set(SENSITIVE_ENV_NAMES)
|
|
119
|
+
candidate_names.update(
|
|
120
|
+
name
|
|
121
|
+
for name, value in os.environ.items()
|
|
122
|
+
if len(value.strip()) >= MIN_DISCOVERED_SECRET_LENGTH
|
|
123
|
+
and SECRET_SHAPED_ENV_NAME_RE.search(name)
|
|
124
|
+
)
|
|
125
|
+
for name in candidate_names:
|
|
126
|
+
value = os.environ.get(name)
|
|
127
|
+
if not value or len(value) < 4:
|
|
128
|
+
continue
|
|
129
|
+
normalized_value = strip_redaction_bypass_controls(value)
|
|
130
|
+
for variant in (value, normalized_value):
|
|
131
|
+
if not variant:
|
|
132
|
+
continue
|
|
133
|
+
variants.update(
|
|
134
|
+
{
|
|
135
|
+
variant,
|
|
136
|
+
urllib.parse.quote(variant, safe=""),
|
|
137
|
+
urllib.parse.quote_plus(variant),
|
|
138
|
+
}
|
|
139
|
+
)
|
|
140
|
+
|
|
141
|
+
for variant in sorted(variants, key=len, reverse=True):
|
|
142
|
+
redacted = redacted.replace(variant, "***")
|
|
143
|
+
|
|
144
|
+
return redacted
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def redact_sensitive(text: str) -> str:
|
|
148
|
+
"""Replace known secret values from the process environment with `***`.
|
|
149
|
+
|
|
150
|
+
Used before publishing OCR-controlled text to the merge request, including
|
|
151
|
+
stderr excerpts from `OCR_POST_ERROR_DETAILS=1` and structured OCR JSON
|
|
152
|
+
fields. Performs two passes:
|
|
153
|
+
|
|
154
|
+
1. Replace each known env value (length >= 4). This is exact-match
|
|
155
|
+
replacement over a small allowlist of secret-bearing environment
|
|
156
|
+
variables, so short bot/MCP secrets still get redacted without
|
|
157
|
+
broadening arbitrary text rewrites. URL-encoded forms are covered.
|
|
158
|
+
2. Sweep generic patterns: whole Authorization header values regardless
|
|
159
|
+
of auth scheme, `Bearer ...`, `x-api-key: ...`, JSON/key-value secrets,
|
|
160
|
+
JWT-shaped values, and `token=...`. This catches values OCR may have
|
|
161
|
+
logged or copied into structured JSON that did not come straight from
|
|
162
|
+
one of our env vars.
|
|
163
|
+
"""
|
|
164
|
+
|
|
165
|
+
if not text:
|
|
166
|
+
return text
|
|
167
|
+
|
|
168
|
+
redacted = normalize_redaction_tokens(strip_redaction_bypass_controls(text))
|
|
169
|
+
redacted = redact_url_userinfo(redacted)
|
|
170
|
+
redacted = redact_env_secret_values(redacted)
|
|
171
|
+
|
|
172
|
+
# Generic high-entropy sweeps. Order matters: stricter patterns
|
|
173
|
+
# first so the looser key/value pass doesn't double-redact.
|
|
174
|
+
# Authorization headers are credentials regardless of auth scheme
|
|
175
|
+
# (`Basic`, `ApiKey`, `Digest`, custom gateway schemes, or bare values),
|
|
176
|
+
# so redact the complete header value rather than trying to parse it.
|
|
177
|
+
redacted = re.sub(
|
|
178
|
+
r'(?i)("(?:proxy-)?authorization"\s*:\s*")(?:[^"\\]|\\.)*(")',
|
|
179
|
+
r"\1***\2",
|
|
180
|
+
redacted,
|
|
181
|
+
)
|
|
182
|
+
redacted = re.sub(
|
|
183
|
+
r"(?i)('(?:proxy-)?authorization'\s*:\s*')(?:[^'\\]|\\.)*(')",
|
|
184
|
+
r"\1***\2",
|
|
185
|
+
redacted,
|
|
186
|
+
)
|
|
187
|
+
redacted = re.sub(
|
|
188
|
+
r"(?im)(\b(?:proxy-)?authorization\s*[:=]\s*)[^\r\n]+",
|
|
189
|
+
r"\1***",
|
|
190
|
+
redacted,
|
|
191
|
+
)
|
|
192
|
+
redacted = re.sub(
|
|
193
|
+
r"(?i)\bbearer\s+[A-Za-z0-9_.\-+/=~]{12,}",
|
|
194
|
+
"Bearer ***",
|
|
195
|
+
redacted,
|
|
196
|
+
)
|
|
197
|
+
# JSON-quoted form: `"token": "value"`, `"api_key": "value"`. OCR or
|
|
198
|
+
# SDK error logs often serialize requests/responses this way. Redact even
|
|
199
|
+
# short values because the key name already identifies the field as secret.
|
|
200
|
+
redacted = re.sub(
|
|
201
|
+
rf'(?i)("{SENSITIVE_NAMED_KEY_PATTERN}"\s*:\s*")' r'(?:[^"\\]|\\.)*(")',
|
|
202
|
+
r"\1***\2",
|
|
203
|
+
redacted,
|
|
204
|
+
)
|
|
205
|
+
# Named key/value form: `password=...`, `client_secret: ...`,
|
|
206
|
+
# `private_token=...`. Do not require high entropy for these keys: the
|
|
207
|
+
# field name itself carries enough signal, and this path is used only for
|
|
208
|
+
# failure diagnostics. Handle quoted values before unquoted values so
|
|
209
|
+
# spaces inside `password="..."` are not leaked.
|
|
210
|
+
redacted = re.sub(
|
|
211
|
+
rf'(?i)\b({SENSITIVE_NAMED_KEY_PATTERN})(\s*[:=]\s*)"(?:[^"\\]|\\.)*"',
|
|
212
|
+
r'\1\2"***"',
|
|
213
|
+
redacted,
|
|
214
|
+
)
|
|
215
|
+
redacted = re.sub(
|
|
216
|
+
rf"(?i)\b({SENSITIVE_NAMED_KEY_PATTERN})(\s*[:=]\s*)'(?:[^'\\]|\\.)*'",
|
|
217
|
+
r"\1\2'***'",
|
|
218
|
+
redacted,
|
|
219
|
+
)
|
|
220
|
+
redacted = re.sub(
|
|
221
|
+
rf"(?i)\b({SENSITIVE_NAMED_KEY_PATTERN})(\s*[:=]\s*)[^\"'\s,;&]+",
|
|
222
|
+
r"\1\2***",
|
|
223
|
+
redacted,
|
|
224
|
+
)
|
|
225
|
+
# Structured logs and URLs often use a bare `token=...` key. Keep it
|
|
226
|
+
# bounded to query/CLI-like contexts so ordinary prose mentioning
|
|
227
|
+
# "token" is not aggressively rewritten.
|
|
228
|
+
redacted = re.sub(
|
|
229
|
+
r"(?i)(^|[?&;\s])(token=)[A-Za-z0-9_.\-+/=~]{8,}",
|
|
230
|
+
r"\1\2***",
|
|
231
|
+
redacted,
|
|
232
|
+
)
|
|
233
|
+
# JWT pattern: three base64url segments separated by dots. The
|
|
234
|
+
# header always starts with `eyJ` (base64 of `{"`); the other two
|
|
235
|
+
# segments are bounded loosely — real-world JWTs always exceed this.
|
|
236
|
+
redacted = re.sub(
|
|
237
|
+
r"\beyJ[A-Za-z0-9_\-]{5,}\.[A-Za-z0-9_\-]{5,}\.[A-Za-z0-9_\-]{5,}\b",
|
|
238
|
+
"***",
|
|
239
|
+
redacted,
|
|
240
|
+
)
|
|
241
|
+
return redacted
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
def sanitize_ocr_value(value: Any) -> Any:
|
|
245
|
+
"""Recursively redact OCR-controlled strings before MR publication.
|
|
246
|
+
|
|
247
|
+
OCR JSON is model/provider-controlled output. Treat every string as
|
|
248
|
+
publishable until proven otherwise: comments, warnings, tool metadata,
|
|
249
|
+
and future fields all pass through this sanitizer before any formatting,
|
|
250
|
+
fingerprinting, or GitLab posting logic consumes them.
|
|
251
|
+
"""
|
|
252
|
+
|
|
253
|
+
if isinstance(value, str):
|
|
254
|
+
return redact_sensitive(value)
|
|
255
|
+
if isinstance(value, list):
|
|
256
|
+
return [sanitize_ocr_value(item) for item in value]
|
|
257
|
+
if isinstance(value, dict):
|
|
258
|
+
sanitized: dict[Any, Any] = {}
|
|
259
|
+
for key, item in value.items():
|
|
260
|
+
sanitized_key = sanitize_ocr_value(key) if isinstance(key, str) else key
|
|
261
|
+
sanitized[sanitized_key] = sanitize_ocr_value(item)
|
|
262
|
+
return sanitized
|
|
263
|
+
return value
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
def redact_url_userinfo_only(value: str) -> str:
|
|
267
|
+
"""Redact only URL userinfo, leaving path/query text unchanged."""
|
|
268
|
+
|
|
269
|
+
def replace(match: re.Match[str]) -> str:
|
|
270
|
+
prefix, userinfo = match.group(1), match.group(2)
|
|
271
|
+
if not _is_url_userinfo_candidate(userinfo):
|
|
272
|
+
return match.group(0)
|
|
273
|
+
return f"{prefix}***@"
|
|
274
|
+
|
|
275
|
+
return re.sub(
|
|
276
|
+
r"(?i)([a-z][a-z0-9+.-]*://)([^/?#@]+)@"
|
|
277
|
+
r"(?=[^/?#\s@]+(?:[/?#]|[\s,;.!)]|$))",
|
|
278
|
+
replace,
|
|
279
|
+
value,
|
|
280
|
+
)
|
|
281
|
+
|
|
282
|
+
|
|
283
|
+
def _is_url_userinfo_candidate(userinfo: str) -> bool:
|
|
284
|
+
"""Return whether text before `@host` is plausibly URL userinfo."""
|
|
285
|
+
|
|
286
|
+
normalized = "".join(
|
|
287
|
+
char for char in userinfo if unicodedata.category(char) not in {"Cc", "Cf", "Zl", "Zp"}
|
|
288
|
+
)
|
|
289
|
+
decoded = urllib.parse.unquote(normalized)
|
|
290
|
+
if not decoded or any(char in decoded for char in "/?#@"):
|
|
291
|
+
return False
|
|
292
|
+
if any(char.isspace() for char in userinfo):
|
|
293
|
+
return ":" in decoded
|
|
294
|
+
return True
|
|
295
|
+
|
|
296
|
+
|
|
297
|
+
def _query_key_is_sensitive(raw_key: str) -> bool:
|
|
298
|
+
"""Return whether a URL query key names a credential."""
|
|
299
|
+
|
|
300
|
+
decoded = urllib.parse.unquote_plus(raw_key)
|
|
301
|
+
decoded = "".join(
|
|
302
|
+
char for char in decoded if unicodedata.category(char) not in {"Cc", "Cf", "Zl", "Zp"}
|
|
303
|
+
)
|
|
304
|
+
decoded = normalize_redaction_tokens(decoded)
|
|
305
|
+
return re.fullmatch(SENSITIVE_NAMED_KEY_PATTERN, decoded, re.IGNORECASE) is not None
|
|
306
|
+
|
|
307
|
+
|
|
308
|
+
def _redact_sensitive_query_values(value: str) -> str:
|
|
309
|
+
"""Redact sensitive query-like key values, including encoded key names."""
|
|
310
|
+
|
|
311
|
+
def replace(match: re.Match[str]) -> str:
|
|
312
|
+
prefix, key, suffix = match.group(1), match.group(2), match.group(4) or ""
|
|
313
|
+
if not _query_key_is_sensitive(key):
|
|
314
|
+
return match.group(0)
|
|
315
|
+
return f"{prefix}{key}=***{suffix}"
|
|
316
|
+
|
|
317
|
+
return re.sub(r"(?i)(^|[?&;])([^=?#&;]+)=([^&#;\s]+)(#[^&;\s]*)?", replace, value)
|
|
318
|
+
|
|
319
|
+
|
|
320
|
+
def redact_url_userinfo(value: str) -> str:
|
|
321
|
+
"""Redact credentials embedded in URL-like manifest values."""
|
|
322
|
+
|
|
323
|
+
value = redact_url_userinfo_only(value)
|
|
324
|
+
return _redact_sensitive_query_values(value)
|