patchsight 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- patchsight/__init__.py +5 -0
- patchsight/cli.py +130 -0
- patchsight/db.py +221 -0
- patchsight/eval/__init__.py +1 -0
- patchsight/eval/confidence.py +169 -0
- patchsight/eval/metrics.py +134 -0
- patchsight/eval/parity.py +96 -0
- patchsight/eval/safety.py +54 -0
- patchsight/names.py +76 -0
- patchsight/pipeline/__init__.py +4 -0
- patchsight/pipeline/__main__.py +863 -0
- patchsight/pipeline/benchmarks/__init__.py +1 -0
- patchsight/pipeline/benchmarks/bundle.py +108 -0
- patchsight/pipeline/benchmarks/changedfuncs.py +275 -0
- patchsight/pipeline/benchmarks/dunlap.py +135 -0
- patchsight/pipeline/benchmarks/extraction.py +242 -0
- patchsight/pipeline/benchmarks/replay.py +187 -0
- patchsight/pipeline/benchmarks/splits.py +121 -0
- patchsight/pipeline/build_db.py +227 -0
- patchsight/pipeline/diffs/__init__.py +1 -0
- patchsight/pipeline/diffs/commits.py +109 -0
- patchsight/pipeline/diffs/github.py +177 -0
- patchsight/pipeline/diffs/hunks.py +120 -0
- patchsight/pipeline/diffs/mapping.py +203 -0
- patchsight/pipeline/diffs/paths.py +127 -0
- patchsight/pipeline/extract/__init__.py +1 -0
- patchsight/pipeline/extract/extractor.py +156 -0
- patchsight/pipeline/extract/prompt.py +219 -0
- patchsight/pipeline/extract/schema.py +40 -0
- patchsight/pipeline/extract/validate.py +81 -0
- patchsight/pipeline/fix_sample.py +384 -0
- patchsight/pipeline/http.py +143 -0
- patchsight/pipeline/ingest/__init__.py +1 -0
- patchsight/pipeline/ingest/aliases.py +76 -0
- patchsight/pipeline/ingest/ghsa_repo.py +80 -0
- patchsight/pipeline/ingest/osv.py +253 -0
- patchsight/pipeline/ingest/references.py +189 -0
- patchsight/pipeline/ingest/snapshot.py +146 -0
- patchsight/pipeline/llm/__init__.py +1 -0
- patchsight/pipeline/llm/client.py +288 -0
- patchsight/pipeline/llm/config.py +68 -0
- patchsight/pipeline/llm/tokens.py +74 -0
- patchsight/pipeline/pypi.py +64 -0
- patchsight/pipeline/week1.py +314 -0
- patchsight/py.typed +0 -0
- patchsight/scan/__init__.py +4 -0
- patchsight/scan/ast_scanner.py +270 -0
- patchsight/scan/dbcache.py +119 -0
- patchsight/scan/lockfiles/__init__.py +1 -0
- patchsight/scan/lockfiles/requirements.py +116 -0
- patchsight/scan/osv_client.py +80 -0
- patchsight/scan/report.py +90 -0
- patchsight/scan/run.py +43 -0
- patchsight/scan/verdicts.py +178 -0
- patchsight-0.1.0.dist-info/METADATA +104 -0
- patchsight-0.1.0.dist-info/RECORD +59 -0
- patchsight-0.1.0.dist-info/WHEEL +4 -0
- patchsight-0.1.0.dist-info/entry_points.txt +3 -0
- patchsight-0.1.0.dist-info/licenses/LICENSE +202 -0
|
@@ -0,0 +1,288 @@
|
|
|
1
|
+
"""An OpenAI-compatible chat client with a response cache and a token-bucket limiter.
|
|
2
|
+
|
|
3
|
+
* **Cache:** keyed by SHA-256 over the whole request: model, messages, schema, decoding
|
|
4
|
+
parameters and prompt version. Changing any of them misses the cache, so a model swap
|
|
5
|
+
never reuses stale answers. Hits are free and make re-runs deterministic.
|
|
6
|
+
* **Limiter:** a sliding 60-second window over estimated tokens. It keeps each provider
|
|
7
|
+
under its tokens-per-minute cap and stops at the requests-per-day cap.
|
|
8
|
+
* **Backend:** injectable, so tests never call a provider.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
import hashlib
|
|
12
|
+
import json
|
|
13
|
+
import os
|
|
14
|
+
import re
|
|
15
|
+
import time
|
|
16
|
+
from collections import deque
|
|
17
|
+
from collections.abc import Callable
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
from typing import Any, cast
|
|
20
|
+
|
|
21
|
+
from pydantic import BaseModel, ConfigDict
|
|
22
|
+
|
|
23
|
+
from patchsight.pipeline.llm.config import LLMConfig
|
|
24
|
+
from patchsight.pipeline.llm.tokens import CharEstimate, TokenCounter
|
|
25
|
+
|
|
26
|
+
_ACCOUNT_ID = re.compile(r"\b(org|user|proj|acct)_[A-Za-z0-9]{6,}")
|
|
27
|
+
_SECRET = re.compile(r"(?i)\b(api[_-]?key|token|secret)\b(\W{1,3})\S+")
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def redact(text: str) -> str:
|
|
31
|
+
"""Remove account identifiers and anything key-like from provider messages."""
|
|
32
|
+
text = _ACCOUNT_ID.sub(lambda m: f"{m.group(1)}_[redacted]", text)
|
|
33
|
+
return _SECRET.sub(lambda m: f"{m.group(1)}{m.group(2)}[redacted]", text)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class QuotaExhaustedError(RuntimeError):
|
|
37
|
+
"""The provider's daily request cap for this run is used up."""
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class RateLimitedError(RuntimeError):
|
|
41
|
+
"""The provider kept refusing for rate limits after the SDK's own retries."""
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
class ProviderRejectedError(RuntimeError):
|
|
45
|
+
"""The provider refused this one generation, for example output that failed its schema
|
|
46
|
+
check or a request too large. Worth one retry; it does not mean the provider is down."""
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
class LLMRequest(BaseModel):
|
|
50
|
+
model_config = ConfigDict(frozen=True)
|
|
51
|
+
|
|
52
|
+
model: str
|
|
53
|
+
prompt_version: str
|
|
54
|
+
messages: tuple[dict[str, str], ...]
|
|
55
|
+
schema_name: str
|
|
56
|
+
response_schema: dict[str, Any]
|
|
57
|
+
max_tokens: int
|
|
58
|
+
seed: int
|
|
59
|
+
temperature: float = 0.0
|
|
60
|
+
extra: dict[str, Any] = {}
|
|
61
|
+
|
|
62
|
+
def cache_key(self) -> str:
|
|
63
|
+
canonical = json.dumps(self.model_dump(mode="json"), sort_keys=True, ensure_ascii=False)
|
|
64
|
+
return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
class LLMResponse(BaseModel):
|
|
68
|
+
model_config = ConfigDict(frozen=True)
|
|
69
|
+
|
|
70
|
+
content: str
|
|
71
|
+
prompt_tokens: int
|
|
72
|
+
completion_tokens: int
|
|
73
|
+
cached: bool
|
|
74
|
+
seconds: float
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
Backend = Callable[[LLMRequest], tuple[str, int, int]]
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
class RateLimiter:
|
|
81
|
+
"""Token limits at two scales.
|
|
82
|
+
|
|
83
|
+
* Per minute: a sliding 60-second window over estimated tokens, held in memory.
|
|
84
|
+
* Per day: a rolling 24-hour ledger of actual usage, kept on disk when ``ledger`` is
|
|
85
|
+
given, so separate runs share one daily budget. When the next call would exceed the
|
|
86
|
+
daily token or request cap, the run stops with ``QuotaExhaustedError``; everything
|
|
87
|
+
already answered is cached, so the next run resumes where this one stopped.
|
|
88
|
+
"""
|
|
89
|
+
|
|
90
|
+
def __init__(
|
|
91
|
+
self,
|
|
92
|
+
tokens_per_minute: int,
|
|
93
|
+
requests_per_day: int,
|
|
94
|
+
tokens_per_day: int | None = None,
|
|
95
|
+
ledger: Path | None = None,
|
|
96
|
+
clock: Callable[[], float] = time.monotonic,
|
|
97
|
+
sleep: Callable[[float], None] = time.sleep,
|
|
98
|
+
wall: Callable[[], float] = time.time,
|
|
99
|
+
) -> None:
|
|
100
|
+
self.tokens_per_minute = tokens_per_minute
|
|
101
|
+
self.requests_per_day = requests_per_day
|
|
102
|
+
self.tokens_per_day = tokens_per_day
|
|
103
|
+
self._ledger = ledger
|
|
104
|
+
self._clock = clock
|
|
105
|
+
self._sleep = sleep
|
|
106
|
+
self._wall = wall
|
|
107
|
+
self._window: deque[tuple[float, int]] = deque()
|
|
108
|
+
self._day: list[tuple[float, int]] = self._load_ledger()
|
|
109
|
+
self.requests = 0
|
|
110
|
+
self.waited = 0.0
|
|
111
|
+
|
|
112
|
+
def _load_ledger(self) -> list[tuple[float, int]]:
|
|
113
|
+
if self._ledger is None or not self._ledger.exists():
|
|
114
|
+
return []
|
|
115
|
+
entries = []
|
|
116
|
+
for line in self._ledger.read_text(encoding="utf-8").splitlines():
|
|
117
|
+
if line.strip():
|
|
118
|
+
record = json.loads(line)
|
|
119
|
+
entries.append((float(record["t"]), int(record["tokens"])))
|
|
120
|
+
return entries
|
|
121
|
+
|
|
122
|
+
def _last_day(self) -> list[tuple[float, int]]:
|
|
123
|
+
cutoff = self._wall() - 86_400.0
|
|
124
|
+
self._day = [entry for entry in self._day if entry[0] > cutoff]
|
|
125
|
+
return self._day
|
|
126
|
+
|
|
127
|
+
def _used(self, now: float) -> int:
|
|
128
|
+
while self._window and now - self._window[0][0] >= 60.0:
|
|
129
|
+
self._window.popleft()
|
|
130
|
+
return sum(tokens for _, tokens in self._window)
|
|
131
|
+
|
|
132
|
+
def acquire(self, tokens: int) -> None:
|
|
133
|
+
day = self._last_day()
|
|
134
|
+
requests_today = len(day) if self._ledger is not None else self.requests
|
|
135
|
+
if requests_today >= self.requests_per_day:
|
|
136
|
+
raise QuotaExhaustedError(f"daily request cap of {self.requests_per_day} reached")
|
|
137
|
+
if self.tokens_per_day is not None:
|
|
138
|
+
used_today = sum(t for _, t in day)
|
|
139
|
+
if used_today + tokens > self.tokens_per_day:
|
|
140
|
+
raise QuotaExhaustedError(
|
|
141
|
+
f"daily token cap: {used_today:,} used in the last 24 h, this call needs "
|
|
142
|
+
f"about {tokens:,}, cap {self.tokens_per_day:,}. Cached work resumes later."
|
|
143
|
+
)
|
|
144
|
+
tokens = min(tokens, self.tokens_per_minute)
|
|
145
|
+
while self._used(self._clock()) + tokens > self.tokens_per_minute:
|
|
146
|
+
wait = 60.0 - (self._clock() - self._window[0][0]) + 0.05
|
|
147
|
+
self.waited += wait
|
|
148
|
+
self._sleep(wait)
|
|
149
|
+
self._window.append((self._clock(), tokens))
|
|
150
|
+
self.requests += 1
|
|
151
|
+
|
|
152
|
+
def settle(self, actual: int) -> None:
|
|
153
|
+
"""Record the tokens the provider reported for the most recent call."""
|
|
154
|
+
if self._window:
|
|
155
|
+
when, _ = self._window.pop()
|
|
156
|
+
self._window.append((when, min(actual, self.tokens_per_minute)))
|
|
157
|
+
entry = (self._wall(), actual)
|
|
158
|
+
self._day.append(entry)
|
|
159
|
+
if self._ledger is not None:
|
|
160
|
+
self._ledger.parent.mkdir(parents=True, exist_ok=True)
|
|
161
|
+
with self._ledger.open("a", encoding="utf-8") as handle:
|
|
162
|
+
handle.write(json.dumps({"t": entry[0], "tokens": actual}) + "\n")
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def openai_backend(base_url: str, api_key: str) -> Backend:
|
|
166
|
+
"""A backend that calls any OpenAI-compatible endpoint with strict JSON-schema output."""
|
|
167
|
+
# Imported lazily: offline tests never need the SDK.
|
|
168
|
+
from openai import APIStatusError, BadRequestError, OpenAI, RateLimitError
|
|
169
|
+
|
|
170
|
+
client = OpenAI(base_url=base_url, api_key=api_key, max_retries=4, timeout=180.0)
|
|
171
|
+
|
|
172
|
+
def call(request: LLMRequest) -> tuple[str, int, int]:
|
|
173
|
+
try:
|
|
174
|
+
return _create(request)
|
|
175
|
+
except RateLimitError as error:
|
|
176
|
+
# Per-minute limits are retried by the SDK; a daily cap cannot be waited out.
|
|
177
|
+
message = redact(str(error))
|
|
178
|
+
if "per day" in message.lower():
|
|
179
|
+
raise QuotaExhaustedError(message[:300]) from None
|
|
180
|
+
raise RateLimitedError(message[:300]) from None
|
|
181
|
+
except (BadRequestError, APIStatusError) as error:
|
|
182
|
+
if isinstance(error, BadRequestError) or error.status_code == 413:
|
|
183
|
+
raise ProviderRejectedError(redact(str(error))[:300]) from error
|
|
184
|
+
raise
|
|
185
|
+
|
|
186
|
+
def _create(request: LLMRequest) -> tuple[str, int, int]:
|
|
187
|
+
completion = client.chat.completions.create(
|
|
188
|
+
model=request.model,
|
|
189
|
+
messages=cast(Any, list(request.messages)),
|
|
190
|
+
temperature=request.temperature,
|
|
191
|
+
seed=request.seed,
|
|
192
|
+
max_tokens=request.max_tokens,
|
|
193
|
+
response_format=cast(
|
|
194
|
+
Any,
|
|
195
|
+
{
|
|
196
|
+
"type": "json_schema",
|
|
197
|
+
"json_schema": {
|
|
198
|
+
"name": request.schema_name,
|
|
199
|
+
"schema": request.response_schema,
|
|
200
|
+
"strict": True,
|
|
201
|
+
},
|
|
202
|
+
},
|
|
203
|
+
),
|
|
204
|
+
extra_body=request.extra or None,
|
|
205
|
+
)
|
|
206
|
+
usage = completion.usage
|
|
207
|
+
content = completion.choices[0].message.content or ""
|
|
208
|
+
return (
|
|
209
|
+
content,
|
|
210
|
+
usage.prompt_tokens if usage else 0,
|
|
211
|
+
usage.completion_tokens if usage else 0,
|
|
212
|
+
)
|
|
213
|
+
|
|
214
|
+
return call
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
class LLMClient:
|
|
218
|
+
def __init__(
|
|
219
|
+
self,
|
|
220
|
+
backend: Backend,
|
|
221
|
+
limiter: RateLimiter,
|
|
222
|
+
cache_dir: Path,
|
|
223
|
+
counter: TokenCounter | None = None,
|
|
224
|
+
) -> None:
|
|
225
|
+
self._backend = backend
|
|
226
|
+
self._limiter = limiter
|
|
227
|
+
self._cache_dir = cache_dir
|
|
228
|
+
self._counter: TokenCounter = counter or CharEstimate()
|
|
229
|
+
self.network_calls = 0
|
|
230
|
+
|
|
231
|
+
def estimate_tokens(self, text: str) -> int:
|
|
232
|
+
return self._counter.count(text)
|
|
233
|
+
|
|
234
|
+
def complete(self, request: LLMRequest) -> LLMResponse:
|
|
235
|
+
key = request.cache_key()
|
|
236
|
+
path = self._cache_dir / key[:2] / f"{key}.json"
|
|
237
|
+
if path.exists():
|
|
238
|
+
cached = json.loads(path.read_text(encoding="utf-8"))
|
|
239
|
+
if "rejected" in cached:
|
|
240
|
+
# Rejections are cached too, so a re-run gives the same outcome for free.
|
|
241
|
+
raise ProviderRejectedError(cached["rejected"])
|
|
242
|
+
return LLMResponse(**cached, cached=True, seconds=0.0)
|
|
243
|
+
|
|
244
|
+
prompt_text = "".join(m["content"] for m in request.messages)
|
|
245
|
+
estimate = self.estimate_tokens(prompt_text) + request.max_tokens
|
|
246
|
+
self._limiter.acquire(estimate)
|
|
247
|
+
started = time.monotonic()
|
|
248
|
+
try:
|
|
249
|
+
content, prompt_tokens, completion_tokens = self._backend(request)
|
|
250
|
+
except ProviderRejectedError as rejected:
|
|
251
|
+
self.network_calls += 1
|
|
252
|
+
self._limiter.settle(estimate) # the provider may still count it; stay conservative
|
|
253
|
+
message = redact(str(rejected))
|
|
254
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
255
|
+
path.write_text(json.dumps({"rejected": message}), encoding="utf-8")
|
|
256
|
+
raise ProviderRejectedError(message) from rejected
|
|
257
|
+
seconds = time.monotonic() - started
|
|
258
|
+
self.network_calls += 1
|
|
259
|
+
self._limiter.settle(prompt_tokens + completion_tokens)
|
|
260
|
+
|
|
261
|
+
record = {
|
|
262
|
+
"content": content,
|
|
263
|
+
"prompt_tokens": prompt_tokens,
|
|
264
|
+
"completion_tokens": completion_tokens,
|
|
265
|
+
}
|
|
266
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
267
|
+
path.write_text(json.dumps(record, ensure_ascii=False), encoding="utf-8")
|
|
268
|
+
return LLMResponse(**record, cached=False, seconds=seconds)
|
|
269
|
+
|
|
270
|
+
|
|
271
|
+
def client_for(
|
|
272
|
+
config: LLMConfig, role: str, cache_dir: Path, counter: TokenCounter
|
|
273
|
+
) -> tuple[LLMClient, str]:
|
|
274
|
+
"""A live client for one configured model role, and that model's ID."""
|
|
275
|
+
model = config.models[role]
|
|
276
|
+
provider = config.providers[model.provider]
|
|
277
|
+
api_key = os.environ.get(provider.api_key_env)
|
|
278
|
+
if not api_key:
|
|
279
|
+
raise RuntimeError(f"set {provider.api_key_env} to call {model.provider}")
|
|
280
|
+
ledger = cache_dir / "usage" / f"{model.model.replace('/', '__')}.jsonl"
|
|
281
|
+
limiter = RateLimiter(
|
|
282
|
+
provider.tokens_per_minute,
|
|
283
|
+
provider.requests_per_day,
|
|
284
|
+
tokens_per_day=provider.tokens_per_day,
|
|
285
|
+
ledger=ledger,
|
|
286
|
+
)
|
|
287
|
+
backend = openai_backend(provider.base_url, api_key)
|
|
288
|
+
return LLMClient(backend, limiter, cache_dir, counter), model.model
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
"""Load provider, model and budget settings from ``configs/llm.toml``."""
|
|
2
|
+
|
|
3
|
+
import tomllib
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
from pydantic import BaseModel, ConfigDict
|
|
8
|
+
|
|
9
|
+
from patchsight.pipeline.llm.tokens import TokenizerConfig
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class ProviderConfig(BaseModel):
|
|
13
|
+
model_config = ConfigDict(frozen=True)
|
|
14
|
+
|
|
15
|
+
base_url: str
|
|
16
|
+
api_key_env: str
|
|
17
|
+
tokens_per_minute: int
|
|
18
|
+
requests_per_day: int
|
|
19
|
+
tokens_per_day: int | None = None
|
|
20
|
+
# A separate cap on input tokens per minute, which one request must also fit under.
|
|
21
|
+
input_tokens_per_minute: int | None = None
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class ModelConfig(BaseModel):
|
|
25
|
+
model_config = ConfigDict(frozen=True)
|
|
26
|
+
|
|
27
|
+
provider: str
|
|
28
|
+
model: str
|
|
29
|
+
max_output_tokens: int
|
|
30
|
+
seed: int
|
|
31
|
+
extra: dict[str, Any] = {}
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class BudgetConfig(BaseModel):
|
|
35
|
+
model_config = ConfigDict(frozen=True)
|
|
36
|
+
|
|
37
|
+
max_prompt_tokens: int
|
|
38
|
+
max_hunk_lines: int
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class LLMConfig(BaseModel):
|
|
42
|
+
model_config = ConfigDict(frozen=True)
|
|
43
|
+
|
|
44
|
+
providers: dict[str, ProviderConfig]
|
|
45
|
+
models: dict[str, ModelConfig]
|
|
46
|
+
budget: BudgetConfig
|
|
47
|
+
tokenizer: TokenizerConfig
|
|
48
|
+
|
|
49
|
+
def prompt_budget(self, role: str) -> int:
|
|
50
|
+
"""Largest prompt, in exact tokens, that still fits the repair round under the caps.
|
|
51
|
+
|
|
52
|
+
A repair round sends the prompt, the previous answer (up to ``max_output_tokens``)
|
|
53
|
+
and a short repair note, and asks for another ``max_output_tokens``.
|
|
54
|
+
"""
|
|
55
|
+
model = self.models[role]
|
|
56
|
+
provider = self.providers[model.provider]
|
|
57
|
+
repair_note = 400
|
|
58
|
+
limits = [
|
|
59
|
+
self.budget.max_prompt_tokens,
|
|
60
|
+
provider.tokens_per_minute - 2 * model.max_output_tokens - repair_note,
|
|
61
|
+
]
|
|
62
|
+
if provider.input_tokens_per_minute is not None:
|
|
63
|
+
limits.append(provider.input_tokens_per_minute - model.max_output_tokens - repair_note)
|
|
64
|
+
return max(0, min(limits))
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def load_config(path: Path) -> LLMConfig:
|
|
68
|
+
return LLMConfig.model_validate(tomllib.loads(path.read_text(encoding="utf-8")))
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
"""Exact token counts with the model's own tokenizer.
|
|
2
|
+
|
|
3
|
+
Budgets are hard limits on Groq (8,000 tokens a minute, 7,000 input tokens a minute), and
|
|
4
|
+
a character-based estimate was off by up to a third on code-heavy prompts. Counting with
|
|
5
|
+
the real tokenizer makes the budget exact, and the same count serves the self-hosted
|
|
6
|
+
model, which shares the tokenizer.
|
|
7
|
+
|
|
8
|
+
The tokenizer file is downloaded once from Hugging Face at a pinned revision and refused
|
|
9
|
+
unless its SHA-256 matches the pin.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
import hashlib
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
from typing import Protocol
|
|
15
|
+
|
|
16
|
+
import httpx
|
|
17
|
+
from pydantic import BaseModel, ConfigDict
|
|
18
|
+
|
|
19
|
+
# Chat formatting adds a few tokens per message (role markers, separators).
|
|
20
|
+
MESSAGE_OVERHEAD = 8
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class TokenizerConfig(BaseModel):
|
|
24
|
+
model_config = ConfigDict(frozen=True)
|
|
25
|
+
|
|
26
|
+
repo: str
|
|
27
|
+
revision: str
|
|
28
|
+
sha256: str
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class TokenCounter(Protocol):
|
|
32
|
+
def count(self, text: str) -> int: ...
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class CharEstimate:
|
|
36
|
+
"""A rough estimate for tests and offline use; never used to enforce real budgets."""
|
|
37
|
+
|
|
38
|
+
def __init__(self, chars_per_token: float = 3.0) -> None:
|
|
39
|
+
self.chars_per_token = chars_per_token
|
|
40
|
+
|
|
41
|
+
def count(self, text: str) -> int:
|
|
42
|
+
return int(len(text) / self.chars_per_token) + 1
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
class TokenizerMismatchError(RuntimeError):
|
|
46
|
+
"""The downloaded tokenizer file does not match the pinned SHA-256."""
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
class HfTokenizer:
|
|
50
|
+
def __init__(self, config: TokenizerConfig, cache_dir: Path) -> None:
|
|
51
|
+
from tokenizers import Tokenizer # pipeline extra; imported lazily
|
|
52
|
+
|
|
53
|
+
path = cache_dir / config.repo.replace("/", "__") / config.revision / "tokenizer.json"
|
|
54
|
+
if not path.exists():
|
|
55
|
+
url = f"https://huggingface.co/{config.repo}/resolve/{config.revision}/tokenizer.json"
|
|
56
|
+
response = httpx.get(url, follow_redirects=True, timeout=120.0)
|
|
57
|
+
response.raise_for_status()
|
|
58
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
59
|
+
path.write_bytes(response.content)
|
|
60
|
+
actual = hashlib.sha256(path.read_bytes()).hexdigest()
|
|
61
|
+
if actual != config.sha256:
|
|
62
|
+
raise TokenizerMismatchError(
|
|
63
|
+
f"{path} has SHA-256 {actual}, but the config pins {config.sha256}"
|
|
64
|
+
)
|
|
65
|
+
self._tokenizer = Tokenizer.from_file(str(path))
|
|
66
|
+
|
|
67
|
+
def count(self, text: str) -> int:
|
|
68
|
+
return len(self._tokenizer.encode(text, add_special_tokens=False).ids)
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def count_messages(
|
|
72
|
+
counter: TokenCounter, messages: list[dict[str, str]] | tuple[dict[str, str], ...]
|
|
73
|
+
) -> int:
|
|
74
|
+
return sum(counter.count(m["content"]) + MESSAGE_OVERHEAD for m in messages)
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
"""Find a package's declared source repositories from its PyPI metadata.
|
|
2
|
+
|
|
3
|
+
Offline pipeline only, limited to ``pypi.org``. Used to check that a fix candidate sits on
|
|
4
|
+
the package's own repository rather than on a fork, a dependency or an unrelated project.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
import re
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from typing import Any
|
|
10
|
+
|
|
11
|
+
import httpx
|
|
12
|
+
from tenacity.wait import wait_base
|
|
13
|
+
|
|
14
|
+
from patchsight.pipeline.http import CachedJsonClient, NotFoundError
|
|
15
|
+
|
|
16
|
+
PYPI = "https://pypi.org"
|
|
17
|
+
_GITHUB_REPO = re.compile(
|
|
18
|
+
r"github\.com[/:](?P<owner>[A-Za-z0-9_.-]+)/(?P<repo>[A-Za-z0-9_.-]+)", re.IGNORECASE
|
|
19
|
+
)
|
|
20
|
+
# github.com/<these>/... are GitHub pages, not repositories.
|
|
21
|
+
_NOT_OWNERS = frozenset(
|
|
22
|
+
{"sponsors", "orgs", "marketplace", "features", "topics", "apps", "about", "settings"}
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def github_repos_in(urls: list[str]) -> tuple[str, ...]:
|
|
27
|
+
"""Every distinct ``owner/name`` GitHub repository mentioned in ``urls``, lower-cased."""
|
|
28
|
+
found: set[str] = set()
|
|
29
|
+
for url in urls:
|
|
30
|
+
for match in _GITHUB_REPO.finditer(url):
|
|
31
|
+
owner, repo = match["owner"].lower(), match["repo"].lower().removesuffix(".git")
|
|
32
|
+
if owner not in _NOT_OWNERS and repo:
|
|
33
|
+
found.add(f"{owner}/{repo}")
|
|
34
|
+
return tuple(sorted(found))
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class PyPIClient:
|
|
38
|
+
def __init__(
|
|
39
|
+
self,
|
|
40
|
+
cache_dir: Path,
|
|
41
|
+
transport: httpx.BaseTransport | None = None,
|
|
42
|
+
wait: wait_base | None = None,
|
|
43
|
+
) -> None:
|
|
44
|
+
self.http = CachedJsonClient(
|
|
45
|
+
PYPI,
|
|
46
|
+
cache_dir,
|
|
47
|
+
headers={"Accept": "application/json", "User-Agent": "patchsight-pipeline"},
|
|
48
|
+
transport=transport,
|
|
49
|
+
wait=wait,
|
|
50
|
+
)
|
|
51
|
+
|
|
52
|
+
def close(self) -> None:
|
|
53
|
+
self.http.close()
|
|
54
|
+
|
|
55
|
+
def declared_github_repos(self, package: str) -> tuple[str, ...] | None:
|
|
56
|
+
"""GitHub repositories named in the package's metadata, or None if PyPI has no record."""
|
|
57
|
+
try:
|
|
58
|
+
data: dict[str, Any] = self.http.get_json(f"/pypi/{package}/json")
|
|
59
|
+
except NotFoundError:
|
|
60
|
+
return None
|
|
61
|
+
info = data.get("info") or {}
|
|
62
|
+
urls = [str(u) for u in (info.get("project_urls") or {}).values() if u]
|
|
63
|
+
urls += [str(info[k]) for k in ("home_page", "download_url") if info.get(k)]
|
|
64
|
+
return github_repos_in(urls)
|