modelpin-relay 0.1.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- modelpin_relay/__init__.py +6 -0
- modelpin_relay/ci_env.py +142 -0
- modelpin_relay/cli.py +320 -0
- modelpin_relay/git_context.py +139 -0
- modelpin_relay/insights.py +226 -0
- modelpin_relay/modelpin_files.py +392 -0
- modelpin_relay/payload.py +431 -0
- modelpin_relay/py.typed +0 -0
- modelpin_relay/signing.py +24 -0
- modelpin_relay/transport.py +91 -0
- modelpin_relay/truncate.py +86 -0
- modelpin_relay-0.1.1.dist-info/METADATA +108 -0
- modelpin_relay-0.1.1.dist-info/RECORD +17 -0
- modelpin_relay-0.1.1.dist-info/WHEEL +5 -0
- modelpin_relay-0.1.1.dist-info/entry_points.txt +2 -0
- modelpin_relay-0.1.1.dist-info/licenses/LICENSE +202 -0
- modelpin_relay-0.1.1.dist-info/top_level.txt +1 -0
modelpin_relay/ci_env.py
ADDED
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
"""Which CI the relay runs in, and the event's `source` block (BUILD_BRIEF 8.2).
|
|
2
|
+
|
|
3
|
+
Bitbucket Pipelines is detected by `BITBUCKET_BUILD_NUMBER`, GitHub Actions by `GITHUB_ACTIONS=true`; anything
|
|
4
|
+
else is `local`. URL shapes for Bitbucket were confirmed in spike S3 (`<origin>/pipelines/results/<build>`,
|
|
5
|
+
`<origin>/pull-requests/<id>`). On GitHub `pull_request` events, `GITHUB_SHA` is a merge commit, so the head
|
|
6
|
+
commit, number and URL come from the event payload file.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import json
|
|
12
|
+
import re
|
|
13
|
+
from collections.abc import Mapping
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
from typing import Any
|
|
16
|
+
|
|
17
|
+
from modelpin_relay.git_context import current_branch, head_commit, local_repo_key
|
|
18
|
+
from modelpin_relay.payload import Source
|
|
19
|
+
|
|
20
|
+
CI_CHOICES = ("auto", "bitbucket", "github", "local")
|
|
21
|
+
|
|
22
|
+
_SEGMENT = re.compile(r"[A-Za-z0-9._-]+")
|
|
23
|
+
_FULL_HASH = re.compile(r"[0-9a-f]{40}(?:[0-9a-f]{24})?")
|
|
24
|
+
_HTTPS_URL = re.compile(r"https://[^\s]+")
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class UnsupportedCI(Exception):
|
|
28
|
+
"""The environment doesn't describe the CI it claims to be (a usage problem, exit 2)."""
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def detect_ci(env: Mapping[str, str]) -> str:
|
|
32
|
+
if env.get("BITBUCKET_BUILD_NUMBER"):
|
|
33
|
+
return "bitbucket"
|
|
34
|
+
if env.get("GITHUB_ACTIONS") == "true":
|
|
35
|
+
return "github"
|
|
36
|
+
return "local"
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def resolve_ci(choice: str, env: Mapping[str, str]) -> str:
|
|
40
|
+
return detect_ci(env) if choice == "auto" else choice
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _https(url: str | None) -> str | None:
|
|
44
|
+
"""Bitbucket's documented origin example is http://; event URLs are always https://."""
|
|
45
|
+
if not url:
|
|
46
|
+
return None
|
|
47
|
+
text = url.strip().rstrip("/")
|
|
48
|
+
if text.startswith("http://"):
|
|
49
|
+
text = "https://" + text[len("http://") :]
|
|
50
|
+
return text if _HTTPS_URL.fullmatch(text) and len(text) <= 2000 else None
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _full_hash(value: str | None) -> str | None:
|
|
54
|
+
return value if value and _FULL_HASH.fullmatch(value) else None
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _positive_int(value: object) -> int | None:
|
|
58
|
+
if isinstance(value, bool):
|
|
59
|
+
return None
|
|
60
|
+
if isinstance(value, int):
|
|
61
|
+
return value if value > 0 else None
|
|
62
|
+
if isinstance(value, str) and value.isdigit():
|
|
63
|
+
number = int(value)
|
|
64
|
+
return number if number > 0 else None
|
|
65
|
+
return None
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def bitbucket_source(env: Mapping[str, str]) -> Source:
|
|
69
|
+
workspace = env.get("BITBUCKET_WORKSPACE", "")
|
|
70
|
+
slug = env.get("BITBUCKET_REPO_SLUG", "")
|
|
71
|
+
if not (_SEGMENT.fullmatch(workspace) and _SEGMENT.fullmatch(slug)):
|
|
72
|
+
raise UnsupportedCI("BITBUCKET_WORKSPACE and BITBUCKET_REPO_SLUG are not both set")
|
|
73
|
+
origin = _https(env.get("BITBUCKET_GIT_HTTP_ORIGIN")) or f"https://bitbucket.org/{workspace}/{slug}"
|
|
74
|
+
build = env.get("BITBUCKET_BUILD_NUMBER") or None
|
|
75
|
+
pr_id = _positive_int(env.get("BITBUCKET_PR_ID"))
|
|
76
|
+
return Source(
|
|
77
|
+
ci="bitbucket",
|
|
78
|
+
repo_key=f"bitbucket:{workspace}/{slug}",
|
|
79
|
+
repo_url=origin,
|
|
80
|
+
commit=_full_hash(env.get("BITBUCKET_COMMIT")),
|
|
81
|
+
branch=(env.get("BITBUCKET_BRANCH") or "")[:255] or None,
|
|
82
|
+
pr_id=pr_id,
|
|
83
|
+
pr_url=f"{origin}/pull-requests/{pr_id}" if pr_id else None,
|
|
84
|
+
build_url=f"{origin}/pipelines/results/{build}" if build else None,
|
|
85
|
+
build_number=build[:64] if build else None,
|
|
86
|
+
)
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def _event_payload(env: Mapping[str, str]) -> dict[str, Any]:
|
|
90
|
+
path = env.get("GITHUB_EVENT_PATH")
|
|
91
|
+
if not path:
|
|
92
|
+
return {}
|
|
93
|
+
try:
|
|
94
|
+
file = Path(path)
|
|
95
|
+
if file.stat().st_size > 5_000_000:
|
|
96
|
+
return {}
|
|
97
|
+
data = json.loads(file.read_text(encoding="utf-8"))
|
|
98
|
+
except (OSError, ValueError, UnicodeDecodeError):
|
|
99
|
+
return {}
|
|
100
|
+
return data if isinstance(data, dict) else {}
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def github_source(env: Mapping[str, str]) -> Source:
|
|
104
|
+
repository = env.get("GITHUB_REPOSITORY", "")
|
|
105
|
+
parts = repository.split("/")
|
|
106
|
+
if len(parts) != 2 or not all(_SEGMENT.fullmatch(p) for p in parts):
|
|
107
|
+
raise UnsupportedCI("GITHUB_REPOSITORY is not set to <owner>/<repo>")
|
|
108
|
+
server = _https(env.get("GITHUB_SERVER_URL")) or "https://github.com"
|
|
109
|
+
repo_url = f"{server}/{repository}"
|
|
110
|
+
run_id = env.get("GITHUB_RUN_ID") or None
|
|
111
|
+
commit = _full_hash(env.get("GITHUB_SHA"))
|
|
112
|
+
branch = env.get("GITHUB_REF_NAME") or None
|
|
113
|
+
pr_id: int | None = None
|
|
114
|
+
pr_url: str | None = None
|
|
115
|
+
if env.get("GITHUB_EVENT_NAME", "").startswith("pull_request"):
|
|
116
|
+
pull = _event_payload(env).get("pull_request")
|
|
117
|
+
if isinstance(pull, dict):
|
|
118
|
+
head = pull.get("head")
|
|
119
|
+
if isinstance(head, dict):
|
|
120
|
+
commit = _full_hash(head.get("sha") if isinstance(head.get("sha"), str) else None)
|
|
121
|
+
pr_id = _positive_int(pull.get("number"))
|
|
122
|
+
pr_url = _https(pull.get("html_url") if isinstance(pull.get("html_url"), str) else None)
|
|
123
|
+
branch = env.get("GITHUB_HEAD_REF") or None
|
|
124
|
+
return Source(
|
|
125
|
+
ci="github",
|
|
126
|
+
repo_key=f"github:{repository}",
|
|
127
|
+
repo_url=repo_url,
|
|
128
|
+
commit=commit,
|
|
129
|
+
branch=branch[:255] if branch else None,
|
|
130
|
+
pr_id=pr_id,
|
|
131
|
+
pr_url=pr_url,
|
|
132
|
+
build_url=f"{repo_url}/actions/runs/{run_id}" if run_id else None,
|
|
133
|
+
build_number=run_id[:64] if run_id else None,
|
|
134
|
+
)
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def build_source(ci: str, env: Mapping[str, str], cwd: Path) -> Source:
|
|
138
|
+
if ci == "bitbucket":
|
|
139
|
+
return bitbucket_source(env)
|
|
140
|
+
if ci == "github":
|
|
141
|
+
return github_source(env)
|
|
142
|
+
return Source(ci="local", repo_key=local_repo_key(cwd), commit=head_commit(cwd), branch=current_branch(cwd))
|
modelpin_relay/cli.py
ADDED
|
@@ -0,0 +1,320 @@
|
|
|
1
|
+
"""The `modelpin-relay` command (BUILD_BRIEF 8.1).
|
|
2
|
+
|
|
3
|
+
Exit codes: 0 done (or nothing to send), 2 usage error, 5 a send failed. The relay never changes the check's
|
|
4
|
+
own exit code; the pipeline exits with that (8.3). Notes go to stderr; stdout carries only `--dry-run` output.
|
|
5
|
+
The webhook URL and secret are read from the environment and never printed, logged or written to `--out`.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import argparse
|
|
11
|
+
import json
|
|
12
|
+
import os
|
|
13
|
+
import sys
|
|
14
|
+
from collections.abc import Mapping, Sequence
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
from typing import Any, TextIO
|
|
17
|
+
|
|
18
|
+
from modelpin_relay import TESTED_MODELPIN_VERSIONS, __version__, insights
|
|
19
|
+
from modelpin_relay import git_context as git
|
|
20
|
+
from modelpin_relay.ci_env import CI_CHOICES, UnsupportedCI, build_source, resolve_ci
|
|
21
|
+
from modelpin_relay.modelpin_files import ModelpinFileError, read_config, read_watch_output
|
|
22
|
+
from modelpin_relay.payload import CheckInputs, PayloadError, Source, build_check, build_watch, make_event
|
|
23
|
+
from modelpin_relay.transport import send_event
|
|
24
|
+
from modelpin_relay.truncate import EventTooLarge, fit
|
|
25
|
+
|
|
26
|
+
EXIT_OK = 0
|
|
27
|
+
EXIT_USAGE = 2
|
|
28
|
+
EXIT_SEND_FAILED = 5
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _common(parser: argparse.ArgumentParser) -> None:
|
|
32
|
+
parser.add_argument("--ci", choices=CI_CHOICES, default="auto", help="which CI to describe (default: detect)")
|
|
33
|
+
parser.add_argument("--url-env", default="MODELPIN_WEBHOOK_URL", help="variable holding the web trigger URL")
|
|
34
|
+
parser.add_argument("--secret-env", default="MODELPIN_WEBHOOK_SECRET", help="variable holding the signing secret")
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _event_flags(parser: argparse.ArgumentParser) -> None:
|
|
38
|
+
parser.add_argument("--no-event", action="store_true", help="build the event but do not send it")
|
|
39
|
+
parser.add_argument("--dry-run", action="store_true", help="print the event to stdout and send nothing")
|
|
40
|
+
parser.add_argument("--out", type=Path, help="write event.json into this folder (one folder per event)")
|
|
41
|
+
parser.add_argument("--sample", action="store_true", help='mark the event "sample": true (in-app sample data only)')
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
45
|
+
parser = argparse.ArgumentParser(
|
|
46
|
+
prog="modelpin-relay", description="Send Modelpin check and watch results from CI to Jira and Bitbucket."
|
|
47
|
+
)
|
|
48
|
+
commands = parser.add_subparsers(dest="command", required=True)
|
|
49
|
+
|
|
50
|
+
pending = commands.add_parser("pending", help="post the PENDING Code Insights report (Bitbucket only)")
|
|
51
|
+
pending.add_argument("--config", type=Path, default=Path("modelpin.yaml"))
|
|
52
|
+
_common(pending)
|
|
53
|
+
|
|
54
|
+
skip = commands.add_parser("skip", help="post a PASSED report: nothing Modelpin checks changed (Bitbucket only)")
|
|
55
|
+
skip.add_argument("--reason", required=True, help="the sentence the report shows")
|
|
56
|
+
skip.add_argument("--config", type=Path, default=Path("modelpin.yaml"))
|
|
57
|
+
_common(skip)
|
|
58
|
+
|
|
59
|
+
publish = commands.add_parser("publish", help="send the check event for this step's modelpin check")
|
|
60
|
+
publish.add_argument("--exit-code", required=True, help="the exit code of modelpin check in this step")
|
|
61
|
+
publish.add_argument("--since", required=True, type=float, help="step start, in Unix seconds (date +%%s)")
|
|
62
|
+
publish.add_argument("--to", required=True, dest="to_model", help="the candidate model passed to --to")
|
|
63
|
+
publish.add_argument("--from", dest="from_model", help="the current model, if --from was passed to modelpin")
|
|
64
|
+
publish.add_argument("--provider", help="the --provider passed to modelpin, if any")
|
|
65
|
+
publish.add_argument("--config", type=Path, default=Path("modelpin.yaml"))
|
|
66
|
+
publish.add_argument("--store-dir", type=Path, default=Path(".modelpin"))
|
|
67
|
+
publish.add_argument("--scenarios-dir", type=Path)
|
|
68
|
+
publish.add_argument("--no-insights", action="store_true", help="do not post a Code Insights report")
|
|
69
|
+
publish.add_argument("--outcome-file", type=Path, help="write the outcome (clean, regression, ...) here")
|
|
70
|
+
_event_flags(publish)
|
|
71
|
+
_common(publish)
|
|
72
|
+
|
|
73
|
+
watch = commands.add_parser("publish-watch", help="send the watch event from modelpin watch --json output")
|
|
74
|
+
watch.add_argument("--input", required=True, type=Path, help="file holding modelpin watch --json output")
|
|
75
|
+
_event_flags(watch)
|
|
76
|
+
_common(watch)
|
|
77
|
+
|
|
78
|
+
commands.add_parser("version", help="print the relay version")
|
|
79
|
+
return parser
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def _note(err: TextIO, text: str) -> None:
|
|
83
|
+
print(f"modelpin-relay: {text}", file=err)
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _print_utf8(out: TextIO, text: str) -> None:
|
|
87
|
+
"""Writes UTF-8 whatever the console code page is (Windows defaults to cp1252, which garbles `’` and
|
|
88
|
+
emoji in model output); falls back to text writes for streams without a byte buffer (tests)."""
|
|
89
|
+
buffer = getattr(out, "buffer", None)
|
|
90
|
+
if buffer is None:
|
|
91
|
+
print(text, file=out)
|
|
92
|
+
return
|
|
93
|
+
out.flush()
|
|
94
|
+
buffer.write((text + "\n").encode("utf-8"))
|
|
95
|
+
buffer.flush()
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def _json(value: Any) -> str:
|
|
99
|
+
return json.dumps(value, ensure_ascii=False, indent=2)
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def _write(folder: Path, name: str, value: Any) -> None:
|
|
103
|
+
folder.mkdir(parents=True, exist_ok=True)
|
|
104
|
+
(folder / name).write_text(_json(value) + "\n", encoding="utf-8", newline="\n")
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def _warn_version(version: str | None, err: TextIO) -> None:
|
|
108
|
+
if version is not None and version not in TESTED_MODELPIN_VERSIONS:
|
|
109
|
+
tested = ", ".join(sorted(TESTED_MODELPIN_VERSIONS))
|
|
110
|
+
_note(err, f"warning: Modelpin {version} wrote these files; this relay is tested with {tested}")
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def _installed_modelpin_version() -> str | None:
|
|
114
|
+
"""5.2: when the files don't say, the installed Modelpin's version, read from package metadata only."""
|
|
115
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
116
|
+
|
|
117
|
+
try:
|
|
118
|
+
return version("modelpin")
|
|
119
|
+
except PackageNotFoundError:
|
|
120
|
+
return None
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def _relative(path: Path, root: Path) -> str | None:
|
|
124
|
+
try:
|
|
125
|
+
return path.resolve().relative_to(root.resolve()).as_posix()
|
|
126
|
+
except ValueError:
|
|
127
|
+
return None
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def _insights_target(source: Source) -> insights.Target | None:
|
|
131
|
+
"""Only a Bitbucket Pipelines build with a full commit hash can post a report."""
|
|
132
|
+
if source.ci != "bitbucket" or not source.commit:
|
|
133
|
+
return None
|
|
134
|
+
workspace, _, slug = source.repo_key.removeprefix("bitbucket:").partition("/")
|
|
135
|
+
return insights.Target(workspace, slug, source.commit)
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def _deliver(
|
|
139
|
+
event: dict[str, Any], args: argparse.Namespace, env: Mapping[str, str], out: TextIO, err: TextIO
|
|
140
|
+
) -> int:
|
|
141
|
+
try:
|
|
142
|
+
body = fit(event)
|
|
143
|
+
except EventTooLarge as exc:
|
|
144
|
+
_note(err, f"event not sent: {exc}")
|
|
145
|
+
return EXIT_SEND_FAILED
|
|
146
|
+
if args.out is not None:
|
|
147
|
+
_write(args.out, "event.json", event)
|
|
148
|
+
if args.dry_run:
|
|
149
|
+
_print_utf8(out, _json(event))
|
|
150
|
+
return EXIT_OK
|
|
151
|
+
if args.no_event:
|
|
152
|
+
return EXIT_OK
|
|
153
|
+
url = env.get(args.url_env, "").strip()
|
|
154
|
+
secret = env.get(args.secret_env, "").strip()
|
|
155
|
+
if not url or not secret:
|
|
156
|
+
missing = args.url_env if not url else args.secret_env
|
|
157
|
+
_note(err, f"{missing} is not set, so the event was not sent")
|
|
158
|
+
return EXIT_OK
|
|
159
|
+
if not url.startswith("https://") or any(c.isspace() or ord(c) < 32 or ord(c) == 127 for c in url):
|
|
160
|
+
# Only the variable's name is printed: the URL itself is a secret.
|
|
161
|
+
_note(err, f"{args.url_env} must be one https:// URL with no spaces or line breaks; the event was not sent")
|
|
162
|
+
return EXIT_USAGE
|
|
163
|
+
result = send_event(url, secret, body)
|
|
164
|
+
answer = f"HTTP {result.status}" if result.status is not None else (result.error or "no response")
|
|
165
|
+
if result.app_status:
|
|
166
|
+
answer += f" {result.app_status}"
|
|
167
|
+
tries = "" if result.attempts == 1 else f" after {result.attempts} attempts"
|
|
168
|
+
if result.ok:
|
|
169
|
+
_note(err, f"event {event['event_id']} sent: {answer}{tries}")
|
|
170
|
+
return EXIT_OK
|
|
171
|
+
_note(err, f"event {event['event_id']} was not accepted: {answer}{tries}")
|
|
172
|
+
return EXIT_SEND_FAILED
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def _post_report(
|
|
176
|
+
target: insights.Target,
|
|
177
|
+
report: dict[str, Any],
|
|
178
|
+
notes: list[dict[str, Any]],
|
|
179
|
+
*,
|
|
180
|
+
replace: bool,
|
|
181
|
+
err: TextIO,
|
|
182
|
+
) -> int:
|
|
183
|
+
posted = insights.post(target, report, notes, replace=replace)
|
|
184
|
+
_note(err, "Code Insights: " + "; ".join(posted.steps))
|
|
185
|
+
return EXIT_OK if posted.ok else EXIT_SEND_FAILED
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def _pending_or_skip(args: argparse.Namespace, env: Mapping[str, str], cwd: Path, err: TextIO) -> int:
|
|
189
|
+
ci = resolve_ci(args.ci, env)
|
|
190
|
+
if ci != "bitbucket":
|
|
191
|
+
_note(err, "not running in Bitbucket Pipelines, so no Code Insights report was posted")
|
|
192
|
+
return EXIT_OK
|
|
193
|
+
source = build_source(ci, env, cwd)
|
|
194
|
+
target = _insights_target(source)
|
|
195
|
+
if target is None:
|
|
196
|
+
_note(err, "BITBUCKET_COMMIT is not a full commit hash, so no Code Insights report was posted")
|
|
197
|
+
return EXIT_USAGE
|
|
198
|
+
config = read_config(args.config)
|
|
199
|
+
from_model = config.models[0] if config.models else None
|
|
200
|
+
version = _installed_modelpin_version()
|
|
201
|
+
if args.command == "pending":
|
|
202
|
+
report = insights.pending_report(from_model, version, source.build_url)
|
|
203
|
+
else:
|
|
204
|
+
report = insights.skip_report(args.reason, from_model, version, source.build_url)
|
|
205
|
+
return _post_report(target, report, [], replace=True, err=err)
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
def _publish(args: argparse.Namespace, env: Mapping[str, str], cwd: Path, out: TextIO, err: TextIO) -> int:
|
|
209
|
+
source = build_source(resolve_ci(args.ci, env), env, cwd)
|
|
210
|
+
inputs = CheckInputs(
|
|
211
|
+
exit_code_raw=args.exit_code,
|
|
212
|
+
since=args.since,
|
|
213
|
+
to_model=args.to_model,
|
|
214
|
+
from_model=args.from_model,
|
|
215
|
+
provider=args.provider,
|
|
216
|
+
config_path=args.config,
|
|
217
|
+
store_dir=args.store_dir,
|
|
218
|
+
scenarios_dir=args.scenarios_dir,
|
|
219
|
+
repo_root=cwd,
|
|
220
|
+
)
|
|
221
|
+
built = build_check(inputs, source)
|
|
222
|
+
_warn_version(built.modelpin_version, err)
|
|
223
|
+
for warning in built.warnings:
|
|
224
|
+
_note(err, f"warning: {warning}")
|
|
225
|
+
check = built.payload
|
|
226
|
+
|
|
227
|
+
# Git context (8.2): keys in the branch and commit, the files that name either model, their recent history.
|
|
228
|
+
scenarios_dir = args.scenarios_dir or args.config.parent / built.config.scenarios_dir
|
|
229
|
+
excluded = [p for p in (_relative(args.store_dir, cwd), _relative(scenarios_dir, cwd), _relative(args.config, cwd)) if p]
|
|
230
|
+
truncated = list(built.truncated)
|
|
231
|
+
source.jira_keys = git.jira_keys(source.branch, source.commit, cwd)
|
|
232
|
+
source.code_refs = git.code_refs([check["from_model"], check["to_model"]], cwd, excluded=excluded)
|
|
233
|
+
source.history_keys = git.history_keys(source.code_refs, source.jira_keys, cwd)
|
|
234
|
+
if source.code_refs and git.is_shallow(cwd):
|
|
235
|
+
truncated.append("history_keys")
|
|
236
|
+
|
|
237
|
+
if args.outcome_file is not None:
|
|
238
|
+
args.outcome_file.write_text(check["outcome"] + "\n", encoding="utf-8")
|
|
239
|
+
version = built.modelpin_version or _installed_modelpin_version()
|
|
240
|
+
event = make_event(
|
|
241
|
+
"check", source, check=check, watch=None, sample=args.sample, modelpin_version=version, truncated=truncated
|
|
242
|
+
)
|
|
243
|
+
|
|
244
|
+
insights_code = EXIT_OK
|
|
245
|
+
target = _insights_target(source)
|
|
246
|
+
if target is not None and not args.no_insights:
|
|
247
|
+
runs = insights.runs_per_scenario(check, built.config.runs)
|
|
248
|
+
report = insights.check_report(check, version, runs, source.build_url)
|
|
249
|
+
notes = insights.annotations_for(check)
|
|
250
|
+
if args.out is not None:
|
|
251
|
+
_write(args.out, "insights-report.json", report)
|
|
252
|
+
_write(args.out, "insights-annotations.json", notes)
|
|
253
|
+
if args.dry_run:
|
|
254
|
+
_print_utf8(err, "Code Insights report (not posted):\n" + _json({"report": report, "annotations": notes}))
|
|
255
|
+
else:
|
|
256
|
+
insights_code = _post_report(target, report, notes, replace=False, err=err)
|
|
257
|
+
event_code = _deliver(event, args, env, out, err)
|
|
258
|
+
return event_code if event_code != EXIT_OK else insights_code
|
|
259
|
+
|
|
260
|
+
|
|
261
|
+
def _publish_watch(args: argparse.Namespace, env: Mapping[str, str], cwd: Path, out: TextIO, err: TextIO) -> int:
|
|
262
|
+
source = build_source(resolve_ci(args.ci, env), env, cwd)
|
|
263
|
+
built = build_watch(read_watch_output(args.input))
|
|
264
|
+
_warn_version(built.modelpin_version, err)
|
|
265
|
+
source.jira_keys = git.jira_keys(source.branch, source.commit, cwd)
|
|
266
|
+
excluded = [p for p in (".modelpin", "scenarios", "modelpin.yaml") if p]
|
|
267
|
+
input_rel = _relative(args.input, cwd)
|
|
268
|
+
if input_rel:
|
|
269
|
+
excluded.append(input_rel)
|
|
270
|
+
shallow = git.is_shallow(cwd)
|
|
271
|
+
any_refs = False
|
|
272
|
+
for model in built.payload["models"]:
|
|
273
|
+
model["code_refs"] = git.code_refs([model["id"]], cwd, excluded=excluded)
|
|
274
|
+
model["history_keys"] = git.history_keys(model["code_refs"], source.jira_keys, cwd)
|
|
275
|
+
any_refs = any_refs or bool(model["code_refs"])
|
|
276
|
+
event = make_event(
|
|
277
|
+
"watch",
|
|
278
|
+
source,
|
|
279
|
+
check=None,
|
|
280
|
+
watch=built.payload,
|
|
281
|
+
sample=args.sample,
|
|
282
|
+
modelpin_version=built.modelpin_version or _installed_modelpin_version(),
|
|
283
|
+
truncated=["history_keys"] if shallow and any_refs else [],
|
|
284
|
+
)
|
|
285
|
+
return _deliver(event, args, env, out, err)
|
|
286
|
+
|
|
287
|
+
|
|
288
|
+
def main(
|
|
289
|
+
argv: Sequence[str] | None = None,
|
|
290
|
+
*,
|
|
291
|
+
env: Mapping[str, str] | None = None,
|
|
292
|
+
cwd: Path | None = None,
|
|
293
|
+
out: TextIO | None = None,
|
|
294
|
+
err: TextIO | None = None,
|
|
295
|
+
) -> int:
|
|
296
|
+
stdout = out or sys.stdout
|
|
297
|
+
stderr = err or sys.stderr
|
|
298
|
+
try:
|
|
299
|
+
args = build_parser().parse_args(argv)
|
|
300
|
+
except SystemExit as exc:
|
|
301
|
+
return int(exc.code) if isinstance(exc.code, int) else EXIT_USAGE
|
|
302
|
+
environment = os.environ if env is None else env
|
|
303
|
+
folder = cwd or Path.cwd()
|
|
304
|
+
# Relative paths are taken from the working folder, so `main(cwd=...)` behaves like running there.
|
|
305
|
+
for name in ("config", "store_dir", "scenarios_dir", "out", "outcome_file", "input"):
|
|
306
|
+
value = getattr(args, name, None)
|
|
307
|
+
if isinstance(value, Path) and not value.is_absolute():
|
|
308
|
+
setattr(args, name, folder / value)
|
|
309
|
+
try:
|
|
310
|
+
if args.command == "version":
|
|
311
|
+
print(f"modelpin-relay {__version__}", file=stdout)
|
|
312
|
+
return EXIT_OK
|
|
313
|
+
if args.command in ("pending", "skip"):
|
|
314
|
+
return _pending_or_skip(args, environment, folder, stderr)
|
|
315
|
+
if args.command == "publish":
|
|
316
|
+
return _publish(args, environment, folder, stdout, stderr)
|
|
317
|
+
return _publish_watch(args, environment, folder, stdout, stderr)
|
|
318
|
+
except (PayloadError, ModelpinFileError, UnsupportedCI, OSError) as exc:
|
|
319
|
+
_note(stderr, f"error: {exc}")
|
|
320
|
+
return EXIT_USAGE
|
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
"""Read-only git commands (BUILD_BRIEF 8.2). Skipped quietly when git or the repository is unavailable.
|
|
2
|
+
|
|
3
|
+
The repository's identity for `--ci local` (origin, commit, branch), and the git context of every event: Jira keys
|
|
4
|
+
in the branch and the commit's subject, the files that name the model (`code_refs`), and the Jira keys in the
|
|
5
|
+
last 180 days of those files' folders (`history_keys`). Never authors, emails or messages beyond the keys.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import re
|
|
11
|
+
import subprocess
|
|
12
|
+
from collections.abc import Iterable
|
|
13
|
+
from pathlib import Path, PurePosixPath
|
|
14
|
+
|
|
15
|
+
JIRA_KEY = re.compile(r"\b[A-Z][A-Z0-9_]+-[1-9][0-9]*\b")
|
|
16
|
+
MAX_JIRA_KEYS = 10
|
|
17
|
+
MAX_CODE_REFS = 20
|
|
18
|
+
MAX_HISTORY_KEYS = 20
|
|
19
|
+
HISTORY_DAYS = 180
|
|
20
|
+
#: CI configuration files never count as code that calls the model (8.2).
|
|
21
|
+
CI_PATHS = ("bitbucket-pipelines.yml", ".github")
|
|
22
|
+
|
|
23
|
+
_SEGMENT = re.compile(r"[A-Za-z0-9._-]+")
|
|
24
|
+
_URL_REMOTE = re.compile(r"^(?:https?|ssh|git)://(?:[^@/]+@)?(?P<host>[^/:]+)(?::[0-9]+)?/(?P<path>.+)$")
|
|
25
|
+
_SCP_REMOTE = re.compile(r"^(?:[^@/]+@)?(?P<host>[^:/]+):(?P<path>[^/].*)$")
|
|
26
|
+
_HOSTS = {"bitbucket.org": "bitbucket", "github.com": "github"}
|
|
27
|
+
_FULL_HASH = re.compile(r"[0-9a-f]{40}(?:[0-9a-f]{24})?")
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def run_git(args: list[str], cwd: Path) -> str | None:
|
|
31
|
+
"""stdout of `git <args>` in `cwd`, or None if git is missing or the command fails."""
|
|
32
|
+
try:
|
|
33
|
+
done = subprocess.run(
|
|
34
|
+
["git", *args],
|
|
35
|
+
cwd=cwd,
|
|
36
|
+
capture_output=True,
|
|
37
|
+
text=True,
|
|
38
|
+
encoding="utf-8",
|
|
39
|
+
errors="replace",
|
|
40
|
+
timeout=15,
|
|
41
|
+
check=False,
|
|
42
|
+
)
|
|
43
|
+
except (OSError, subprocess.SubprocessError):
|
|
44
|
+
return None
|
|
45
|
+
return done.stdout.strip() if done.returncode == 0 else None
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def repo_key_from_remote(url: str | None) -> str | None:
|
|
49
|
+
"""`bitbucket:<workspace>/<slug>` or `github:<owner>/<repo>` from a remote URL, else None (5.2)."""
|
|
50
|
+
if not url:
|
|
51
|
+
return None
|
|
52
|
+
text = url.strip()
|
|
53
|
+
found = _URL_REMOTE.match(text) or _SCP_REMOTE.match(text)
|
|
54
|
+
if found is None:
|
|
55
|
+
return None
|
|
56
|
+
prefix = _HOSTS.get(found.group("host").lower())
|
|
57
|
+
if prefix is None:
|
|
58
|
+
return None
|
|
59
|
+
path = found.group("path").rstrip("/")
|
|
60
|
+
path = path.removesuffix(".git")
|
|
61
|
+
parts = path.split("/")
|
|
62
|
+
if len(parts) != 2 or not all(_SEGMENT.fullmatch(p) for p in parts):
|
|
63
|
+
return None
|
|
64
|
+
return f"{prefix}:{parts[0]}/{parts[1]}"
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def local_repo_key(cwd: Path) -> str:
|
|
68
|
+
"""5.2 `ci: local`: the normalised origin, else `local:<folder name>`."""
|
|
69
|
+
from_remote = repo_key_from_remote(run_git(["remote", "get-url", "origin"], cwd))
|
|
70
|
+
if from_remote:
|
|
71
|
+
return from_remote
|
|
72
|
+
top = run_git(["rev-parse", "--show-toplevel"], cwd)
|
|
73
|
+
name = Path(top).name if top else cwd.resolve().name
|
|
74
|
+
name = re.sub(r"[/\\]", "_", name)[:200] or "repository"
|
|
75
|
+
return f"local:{name}"
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def head_commit(cwd: Path) -> str | None:
|
|
79
|
+
commit = run_git(["rev-parse", "HEAD"], cwd)
|
|
80
|
+
return commit if commit and _FULL_HASH.fullmatch(commit) else None
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def current_branch(cwd: Path) -> str | None:
|
|
84
|
+
branch = run_git(["rev-parse", "--abbrev-ref", "HEAD"], cwd)
|
|
85
|
+
return branch[:255] if branch and branch != "HEAD" else None
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _unique(items: Iterable[str]) -> list[str]:
|
|
89
|
+
return list(dict.fromkeys(items))
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def jira_keys(branch: str | None, commit: str | None, cwd: Path) -> list[str]:
|
|
93
|
+
"""Keys in the branch name and in the subject of the checked commit (never HEAD, a merge in PR builds)."""
|
|
94
|
+
texts = [branch or ""]
|
|
95
|
+
if commit:
|
|
96
|
+
texts.append(run_git(["log", "-1", "--format=%s", commit], cwd) or "")
|
|
97
|
+
keys = [k for k in JIRA_KEY.findall("\n".join(texts)) if len(k) <= 64]
|
|
98
|
+
return _unique(keys)[:MAX_JIRA_KEYS]
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def code_refs(model_ids: Iterable[str], cwd: Path, *, excluded: Iterable[str] = ()) -> list[str]:
|
|
102
|
+
"""Tracked files that contain any of the model ids, minus `excluded` paths and CI configuration (8.2)."""
|
|
103
|
+
patterns: list[str] = []
|
|
104
|
+
for model_id in _unique(m for m in model_ids if m):
|
|
105
|
+
patterns += ["-e", model_id]
|
|
106
|
+
if not patterns:
|
|
107
|
+
return []
|
|
108
|
+
exclusions = [f":(exclude){p}" for p in (*excluded, *CI_PATHS) if p and p != "."]
|
|
109
|
+
found = run_git(["-c", "core.quotePath=false", "grep", "-l", "-z", "-F", *patterns, "--", ".", *exclusions], cwd)
|
|
110
|
+
if not found:
|
|
111
|
+
return []
|
|
112
|
+
names = {name.replace("\\", "/")[:500] for name in found.split("\0") if name.strip()}
|
|
113
|
+
return sorted(names)[:MAX_CODE_REFS]
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def history_folders(refs: Iterable[str]) -> list[str]:
|
|
117
|
+
"""The folder of each file; a file at the root stands for itself, never for the whole repository."""
|
|
118
|
+
folders = set()
|
|
119
|
+
for ref in refs:
|
|
120
|
+
path = PurePosixPath(ref)
|
|
121
|
+
folders.add(str(path.parent) if str(path.parent) != "." else str(path))
|
|
122
|
+
return sorted(folders)
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def is_shallow(cwd: Path) -> bool:
|
|
126
|
+
return run_git(["rev-parse", "--is-shallow-repository"], cwd) == "true"
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def history_keys(refs: Iterable[str], exclude: Iterable[str], cwd: Path) -> list[str]:
|
|
130
|
+
"""Jira keys in the subjects of the last 180 days of commits that touched the folders, newest first (8.2)."""
|
|
131
|
+
folders = history_folders(refs)
|
|
132
|
+
if not folders:
|
|
133
|
+
return []
|
|
134
|
+
subjects = run_git(["log", f"--since={HISTORY_DAYS} days ago", "--format=%s", "--", *folders], cwd)
|
|
135
|
+
if not subjects:
|
|
136
|
+
return []
|
|
137
|
+
skip = set(exclude)
|
|
138
|
+
keys = [key for key in _unique(JIRA_KEY.findall(subjects)) if key not in skip and len(key) <= 64]
|
|
139
|
+
return keys[:MAX_HISTORY_KEYS]
|