gitfence 0.1.0__py3-none-any.whl

fence/git/log.py

@@ -0,0 +1,60 @@
+"""`git log -L` for walking further back than blame when a commit explains nothing."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from fence.git.runner import GitError, Repo, UnsafeInput
+
+DEFAULT_MAX_COUNT = 20
+MAX_COUNT_LIMIT = 200
+
+_FIELD = "\x1f"
+_RECORD = "\x1e"
+
+
+@dataclass(frozen=True)
+class LogCommit:
+    sha: str
+    author: str
+    author_mail: str
+    date: str  # strict ISO 8601 author date
+    subject: str
+
+
+def log_line_range(
+    repo: Repo, rev: str, path: str, start: int, end: int, *, max_count: int = DEFAULT_MAX_COUNT
+) -> list[LogCommit]:
+    """Commits that touched lines `start`..`end` of `path` as of `rev`, newest first."""
+    sha = repo.verify_ref(rev)
+    rel = repo.safe_relpath(path)
+    if start < 1 or end < start:
+        raise UnsafeInput(f"bad line range: {start},{end}")
+    if not 1 <= max_count <= MAX_COUNT_LIMIT:
+        raise UnsafeInput(f"max_count must be 1..{MAX_COUNT_LIMIT}")
+    out = repo.run(
+        [
+            "log",
+            f"-L{start},{end}:{rel}",
+            "-s",  # metadata only; the patch text is never needed here
+            "--no-color",
+            "--no-ext-diff",
+            f"--max-count={max_count}",
+            f"--format=%H{_FIELD}%an{_FIELD}%ae{_FIELD}%aI{_FIELD}%s{_RECORD}",
+            sha,
+        ]
+    )
+    return parse_log(out)
+
+
+def parse_log(output: str) -> list[LogCommit]:
+    commits: list[LogCommit] = []
+    for record in output.split(_RECORD):
+        record = record.strip("\n")
+        if not record:
+            continue
+        parts = record.split(_FIELD)
+        if len(parts) != 5 or len(parts[0]) not in (40, 64):
+            raise GitError(f"unexpected git log record: {record[:80]!r}")
+        commits.append(LogCommit(*parts))
+    return commits

fence/git/runner.py

@@ -0,0 +1,248 @@
+"""The only place Fence starts a git process.
+
+Every argument that comes from outside (a ref, a path) is untrusted: it may come
+from a diff, a CLI argument, or later from model output. So:
+- arguments are always a list, never a shell string;
+- refs are syntax-checked, then resolved by git itself to a commit SHA;
+- paths must stay inside the repo root after resolving symlinks;
+- every call has a timeout.
+"""
+
+from __future__ import annotations
+
+import codecs
+import os
+import re
+import subprocess
+import tempfile
+import threading
+from collections.abc import Iterator, Sequence
+from dataclasses import dataclass
+from pathlib import Path, PurePosixPath
+
+DEFAULT_TIMEOUT_SECONDS = 10.0
+MAX_STDERR_CHARS = 500
+_VERIFIED: set[tuple[str, str]] = set()
+
+# Characters git allows in refs that we also accept: names, SHAs, HEAD~2, main^, origin/main.
+_REF_PATTERN = re.compile(r"^[A-Za-z0-9_][A-Za-z0-9._/~^@{}-]*$")
+
+
+class GitError(Exception):
+    """git failed, or Fence refused to run it."""
+
+
+class GitTimeout(GitError):
+    """git did not finish within the timeout."""
+
+
+class UnsafeInput(GitError):
+    """A ref or path was rejected before reaching git."""
+
+
+def _git_env() -> dict[str, str]:
+    env = dict(os.environ)
+    env.update(
+        {
+            "GIT_TERMINAL_PROMPT": "0",  # never block on a credential prompt
+            "GIT_PAGER": "cat",
+            "GIT_OPTIONAL_LOCKS": "0",  # read-only: don't take index locks
+            "LC_ALL": "C",  # stable, parseable output
+        }
+    )
+    return env
+
+
+@dataclass(frozen=True)
+class Repo:
+    root: Path
+    timeout: float = DEFAULT_TIMEOUT_SECONDS
+
+    @classmethod
+    def open(cls, path: str | Path, timeout: float = DEFAULT_TIMEOUT_SECONDS) -> Repo:
+        start = Path(path).resolve()
+        if not start.is_dir():
+            raise GitError(f"not a directory: {start}")
+        top = _run(["rev-parse", "--show-toplevel"], cwd=start, timeout=timeout).strip()
+        return cls(root=Path(top).resolve(), timeout=timeout)
+
+    def run(self, args: Sequence[str], *, timeout: float | None = None) -> str:
+        """Run `git <args>` in the repo root and return stdout."""
+        return _run(args, cwd=self.root, timeout=timeout or self.timeout)
+
+    def verify_ref(self, ref: str) -> str:
+        """Return the full commit SHA for `ref`, or raise UnsafeInput / GitError."""
+        if not ref or not _REF_PATTERN.match(ref) or ".." in ref:
+            raise UnsafeInput(f"rejected ref: {ref!r}")
+        # A full SHA that resolved to a commit once always will: memoize those only.
+        key = (str(self.root), ref)
+        if key in _VERIFIED:
+            return ref
+        out = self.run(
+            ["rev-parse", "--verify", "--quiet", "--end-of-options", f"{ref}^{{commit}}"]
+        )
+        sha = out.strip()
+        if not re.fullmatch(r"[0-9a-f]{40}|[0-9a-f]{64}", sha):
+            raise GitError(f"could not resolve ref: {ref!r}")
+        if sha == ref and len(_VERIFIED) < 100_000:
+            _VERIFIED.add(key)
+        return sha
+
+    def safe_relpath(self, path: str | Path) -> str:
+        """Return `path` as a repo-relative POSIX path, refusing anything that escapes the root.
+
+        Works for files that no longer exist in the working tree (deleted in a diff),
+        but any symlink that does exist on the way is resolved and checked.
+        """
+        raw = str(path)
+        if not raw or "\x00" in raw or raw.startswith("-"):
+            raise UnsafeInput(f"rejected path: {raw!r}")
+        candidate = Path(raw)
+        if not candidate.is_absolute():
+            candidate = self.root / candidate
+        resolved = candidate.resolve(strict=False)
+        try:
+            rel = resolved.relative_to(self.root)
+        except ValueError:
+            raise UnsafeInput(f"path escapes repo root: {raw!r}") from None
+        rel_posix = PurePosixPath(*rel.parts)
+        if str(rel_posix) in ("", "."):
+            raise UnsafeInput(f"path is the repo root: {raw!r}")
+        return str(rel_posix)
+
+    def read_blob(self, ref: str, path: str, *, max_bytes: int = 1_000_000) -> str:
+        """Return the contents of `path` at `ref` (`git show ref:path`)."""
+        sha = self.verify_ref(ref)
+        rel = self.safe_relpath(path)
+        # Check the size before reading, so a huge blob in a hostile repo is never loaded.
+        size = self.run(["cat-file", "-s", f"{sha}:{rel}"]).strip()
+        if not size.isdigit() or int(size) > max_bytes:
+            raise GitError(f"file too large to read: {rel} ({size} bytes)")
+        return self.run(["show", f"{sha}:{rel}"])
+
+    def read_blobs(
+        self, specs: Sequence[tuple[str, str]], *, max_bytes: int = 1_000_000
+    ) -> dict[tuple[str, str], str | None]:
+        """Read many `(commit_sha, path)` blobs with two git processes total.
+
+        Missing, binary-looking or oversized blobs map to None. Sizes are checked with
+        --batch-check first, so oversized content is never transferred.
+        """
+        ok = [
+            (sha, path) for sha, path in dict.fromkeys(specs)
+            if re.fullmatch(r"[0-9a-f]{40}|[0-9a-f]{64}", sha)
+            and path and "\n" not in path and "\r" not in path
+        ]
+        result: dict[tuple[str, str], str | None] = {s: None for s in specs}
+        if not ok:
+            return result
+        query = "".join(f"{sha}:{path}\n" for sha, path in ok).encode("utf-8")
+        checks = _run_bytes(["cat-file", "--batch-check"], cwd=self.root, timeout=self.timeout,
+                            input=query).decode("utf-8", "replace").split("\n")
+        small = []
+        for spec, line in zip(ok, checks, strict=False):
+            parts = line.split(" ")
+            if len(parts) == 3 and parts[1] == "blob" and parts[2].isdigit():
+                if int(parts[2]) <= max_bytes:
+                    small.append(spec)
+        if not small:
+            return result
+        query = "".join(f"{sha}:{path}\n" for sha, path in small).encode("utf-8")
+        out = _run_bytes(["cat-file", "--batch"], cwd=self.root, timeout=self.timeout,
+                         input=query)
+        pos = 0
+        for spec in small:
+            nl = out.index(b"\n", pos)
+            header = out[pos:nl].decode("utf-8", "replace").split(" ")
+            pos = nl + 1
+            if len(header) != 3 or not header[2].isdigit():
+                continue
+            size = int(header[2])
+            data = out[pos : pos + size]
+            pos += size + 1  # content is followed by a newline
+            if b"\x00" not in data[:8000]:
+                result[spec] = data.decode("utf-8", "replace")
+        return result
+
+    def stream(
+        self, args: Sequence[str], *, timeout: float | None = None, chunk_size: int = 65536
+    ) -> Iterator[str]:
+        """Run `git <args>` and yield stdout in text chunks, for output too big to hold.
+
+        `timeout` bounds the whole run. Stopping iteration early kills git.
+        """
+        return _stream(args, cwd=self.root, timeout=timeout or self.timeout, chunk_size=chunk_size)
+
+
+def _stream(args: Sequence[str], *, cwd: Path, timeout: float, chunk_size: int) -> Iterator[str]:
+    if isinstance(args, str):
+        raise TypeError("git args must be a list, not a string")
+    argv = ["git", "-c", "core.fsmonitor=false", "-c", "core.quotePath=false", *args]
+    # stderr goes to a file: a PIPE nobody reads can fill up and deadlock git.
+    with tempfile.TemporaryFile() as err:
+        try:
+            proc = subprocess.Popen(
+                argv, cwd=cwd, env=_git_env(), stdout=subprocess.PIPE, stderr=err, shell=False
+            )
+        except FileNotFoundError:
+            raise GitError("git executable not found") from None
+        timed_out = threading.Event()
+
+        def _kill() -> None:
+            timed_out.set()
+            proc.kill()
+
+        timer = threading.Timer(timeout, _kill)
+        timer.daemon = True
+        timer.start()
+        decoder = codecs.getincrementaldecoder("utf-8")(errors="replace")
+        try:
+            assert proc.stdout is not None  # noqa: S101 - set by stdout=PIPE
+            while data := proc.stdout.read1(chunk_size):
+                yield decoder.decode(data)
+            tail = decoder.decode(b"", final=True)
+            if tail:
+                yield tail
+            returncode = proc.wait()
+        finally:
+            timer.cancel()
+            if proc.poll() is None:
+                proc.kill()
+                proc.wait()
+            if proc.stdout is not None:
+                proc.stdout.close()
+        if timed_out.is_set():
+            raise GitTimeout(f"git {args[0] if args else ''} timed out after {timeout}s")
+        if returncode != 0:
+            err.seek(0)
+            stderr = err.read(MAX_STDERR_CHARS * 4).decode("utf-8", "replace").strip()
+            raise GitError(
+                f"git {args[0] if args else ''} failed ({returncode}): {stderr[:MAX_STDERR_CHARS]}"
+            )
+
+
+def _run_bytes(
+    args: Sequence[str], *, cwd: Path, timeout: float, input: bytes | None = None
+) -> bytes:
+    if isinstance(args, str):
+        raise TypeError("git args must be a list, not a string")
+    argv = ["git", "-c", "core.fsmonitor=false", "-c", "core.quotePath=false", *args]
+    try:
+        proc = subprocess.run(
+            argv, cwd=cwd, env=_git_env(), input=input, capture_output=True, timeout=timeout,
+            check=False, shell=False,
+        )
+    except subprocess.TimeoutExpired:
+        raise GitTimeout(f"git {args[0] if args else ''} timed out after {timeout}s") from None
+    except FileNotFoundError:
+        raise GitError("git executable not found") from None
+    if proc.returncode != 0:
+        stderr = proc.stderr.decode("utf-8", "replace").strip()[:MAX_STDERR_CHARS]
+        raise GitError(f"git {args[0] if args else ''} failed ({proc.returncode}): {stderr}")
+    return proc.stdout
+
+
+def _run(args: Sequence[str], *, cwd: Path, timeout: float) -> str:
+    # Bytes in, decoded here: text mode would translate CRLF to LF and hand back
+    # file content that differs from the blob (and from any diff of it).
+    return _run_bytes(args, cwd=cwd, timeout=timeout).decode("utf-8", errors="replace")

fence/github/client.py

@@ -0,0 +1,122 @@
+"""Minimal GitHub REST client: ETag revalidation, rate-limit respect, pagination.
+
+Stdlib only. The token is sent only to the configured https base URL and is
+never logged: every error message passes through `_redact`.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+import time
+import urllib.error
+import urllib.request
+from dataclasses import dataclass
+
+from fence.index.store import Store
+
+MAX_BODY = 2_000_000
+_LINK_NEXT = re.compile(r'<([^>]+)>;\s*rel="next"')
+
+
+class GithubError(Exception):
+    pass
+
+
+class RateLimited(GithubError):
+    pass
+
+
+@dataclass
+class Response:
+    status: int
+    data: object
+    next_url: str | None
+
+
+class Client:
+    def __init__(self, base_url: str, token: str | None, store: Store, *, min_remaining: int = 50,
+                 policy: str = "stop", max_requests: int = 2000, timeout: float = 20.0,
+                 opener=None) -> None:
+        self.base = base_url.rstrip("/")
+        self.token = token
+        self.store = store
+        self.min_remaining = min_remaining
+        self.policy = policy
+        self.max_requests = max_requests
+        self.timeout = timeout
+        self.requests = 0
+        self.remaining: int | None = None
+        self._open = opener or urllib.request.urlopen
+
+    def _redact(self, text: str) -> str:
+        return text.replace(self.token, "***") if self.token else text
+
+    def get(self, path_or_url: str) -> Response:
+        url = path_or_url if path_or_url.startswith(self.base) else self.base + path_or_url
+        if not url.startswith(self.base + "/"):
+            raise GithubError("refusing to send the token to another host")
+        if self.requests >= self.max_requests:
+            raise RateLimited(f"request budget of {self.max_requests} per run used up")
+        if self.remaining is not None and self.remaining <= self.min_remaining:
+            raise RateLimited(f"GitHub rate limit nearly exhausted ({self.remaining} left)")
+        cached = self.store.conn.execute(
+            "SELECT etag, body FROM http_cache WHERE url = ?", (url,)).fetchone()
+        req = urllib.request.Request(url, headers={  # noqa: S310 - base is validated https
+            "Accept": "application/vnd.github+json", "X-GitHub-Api-Version": "2022-11-28",
+            "User-Agent": "fence",
+            **({"Authorization": f"Bearer {self.token}"} if self.token else {}),
+            **({"If-None-Match": cached["etag"]} if cached and cached["etag"] else {}),
+        })
+        self.requests += 1
+        try:
+            with self._open(req, timeout=self.timeout) as resp:  # noqa: S310 - https only
+                self._note_limits(resp.headers)
+                body = resp.read(MAX_BODY + 1)
+                if len(body) > MAX_BODY:
+                    raise GithubError(f"response too large: {url}")
+                etag = resp.headers.get("ETag")
+                text = body.decode("utf-8", "replace")
+                with self.store.transaction():
+                    self.store.conn.execute(
+                        "INSERT OR REPLACE INTO http_cache VALUES(?,?,?,?)",
+                        (url, etag, text, int(time.time())))
+                return Response(resp.status, json.loads(text), _next(resp.headers))
+        except urllib.error.HTTPError as e:
+            self._note_limits(e.headers)
+            if e.code == 304 and cached:
+                return Response(200, json.loads(cached["body"]), _next(e.headers))
+            if e.code in (403, 429) and (e.headers.get("X-RateLimit-Remaining") == "0"
+                                         or e.code == 429):
+                if self.policy == "wait":
+                    reset = int(e.headers.get("X-RateLimit-Reset") or 0)
+                    delay = max(1, min(900, reset - int(time.time())))
+                    time.sleep(delay)
+                    return self.get(url)
+                raise RateLimited("GitHub rate limit reached; re-run later to resume") from None
+            return Response(e.code, None, None)
+        except (urllib.error.URLError, TimeoutError, OSError) as e:
+            raise GithubError(self._redact(f"GitHub unreachable: {e}")) from None
+        except ValueError:
+            raise GithubError(f"invalid JSON from {url}") from None
+
+    def paginate(self, path: str, limit: int = 300) -> list:
+        out: list = []
+        url: str | None = path + ("&" if "?" in path else "?") + "per_page=100"
+        while url and len(out) < limit:
+            r = self.get(url)
+            if r.status != 200 or not isinstance(r.data, list):
+                break
+            out += r.data
+            url = r.next_url
+        return out[:limit]
+
+    def _note_limits(self, headers) -> None:
+        rem = headers.get("X-RateLimit-Remaining") if headers else None
+        if rem and rem.isdigit():
+            self.remaining = int(rem)
+
+
+def _next(headers) -> str | None:
+    m = _LINK_NEXT.search(headers.get("Link", "") if headers else "")
+    return m.group(1) if m else None

fence/github/comment.py

@@ -0,0 +1,50 @@
+"""One sticky PR comment, found by a hidden marker and updated in place.
+
+Fork PRs get a read-only token: posting then fails with 403, and the caller falls
+back to the job summary. Fence never asks for pull_request_target.
+"""
+
+from __future__ import annotations
+
+import json
+import urllib.error
+import urllib.request
+
+from fence.render import COMMENT_MARKER
+
+MAX_COMMENT = 60_000
+
+
+def upsert(api: str, repo: str, pr: int, body: str, token: str, *, opener=None) -> str:
+    """Create or update the sticky comment. Returns 'created', 'updated' or 'forbidden'."""
+    if not api.startswith("https://"):
+        raise ValueError("api base must be https")
+    opener = opener or urllib.request.urlopen
+    body = body[:MAX_COMMENT]
+
+    def call(method: str, url: str, data: dict | None = None):
+        req = urllib.request.Request(  # noqa: S310 - https only, checked above
+            url, method=method, data=json.dumps(data).encode() if data is not None else None,
+            headers={"Authorization": f"Bearer {token}", "Accept": "application/vnd.github+json",
+                     "User-Agent": "fence", "Content-Type": "application/json"})
+        with opener(req, timeout=20) as resp:
+            return json.loads(resp.read().decode("utf-8") or "null")
+
+    base = f"{api.rstrip('/')}/repos/{repo}/issues"
+    try:
+        page = 1
+        while page <= 10:
+            comments = call("GET", f"{base}/{pr}/comments?per_page=100&page={page}") or []
+            for c in comments:
+                if COMMENT_MARKER in (c.get("body") or ""):
+                    call("PATCH", f"{base}/comments/{c['id']}", {"body": body})
+                    return "updated"
+            if len(comments) < 100:
+                break
+            page += 1
+        call("POST", f"{base}/{pr}/comments", {"body": body})
+        return "created"
+    except urllib.error.HTTPError as e:
+        if e.code in (403, 404):
+            return "forbidden"
+        raise

fence/github/sync.py

@@ -0,0 +1,123 @@
+"""Fetch PRs, linked issues and review comments for indexed commits.
+
+Resumable: every item is committed as soon as it is fetched, and already-stored
+PRs are skipped, so an interrupted or rate-limited run continues next time.
+Text is stored verbatim and stays untrusted.
+"""
+
+from __future__ import annotations
+
+import json
+import time
+from dataclasses import dataclass, field
+
+from fence.context import Context
+from fence.github.client import Client, GithubError, RateLimited
+from fence.signals.catalog import extract_text
+
+
+@dataclass
+class SyncReport:
+    prs: int = 0
+    issues: int = 0
+    review_comments: int = 0
+    unreachable: int = 0
+    requests: int = 0
+    stopped: str | None = None
+    notes: list[str] = field(default_factory=list)
+
+
+def sync(ctx: Context, *, client: Client | None = None, max_prs: int = 500) -> SyncReport:
+    rep = SyncReport()
+    cfg = ctx.cfg.github
+    if cfg.enabled == "off" or not ctx.slug:
+        rep.notes.append("GitHub sync skipped (disabled or no GitHub remote)")
+        return rep
+    client = client or Client(cfg.api_base_url, ctx.github_token, ctx.store,
+                              min_remaining=cfg.min_remaining, policy=cfg.rate_limit_policy,
+                              max_requests=cfg.max_requests_per_run)
+    owner, name = ctx.slug
+    base = f"/repos/{owner}/{name}"
+    min_time = int(time.time()) - cfg.max_age_days * 86400 if cfg.max_age_days else 0
+    todo = [r["pr_number"] for r in ctx.store.conn.execute(
+        "SELECT DISTINCT c.pr_number FROM commits c LEFT JOIN prs p ON p.number = c.pr_number "
+        "WHERE c.pr_number IS NOT NULL AND p.number IS NULL AND c.orphaned = 0 "
+        "AND c.committer_time >= ? ORDER BY c.committer_time DESC LIMIT ?",
+        (min_time, max_prs))]
+    try:
+        for n in todo if cfg.sync_prs else []:
+            _sync_pr(ctx, client, base, n, rep)
+    except RateLimited as e:
+        rep.stopped = str(e)
+    except GithubError as e:
+        rep.stopped = str(e)
+    rep.requests = client.requests
+    return rep
+
+
+def _sync_pr(ctx: Context, client: Client, base: str, n: int, rep: SyncReport) -> None:
+    r = client.get(f"{base}/pulls/{n}")
+    if r.status != 200 or not isinstance(r.data, dict):
+        with ctx.store.transaction():
+            ctx.store.conn.execute(
+                "INSERT OR REPLACE INTO prs(number, state, fetched_at) VALUES(?, 'unreachable', ?)",
+                (n, int(time.time())))
+        rep.unreachable += 1
+        return
+    d = r.data
+    labels = [lb.get("name", "") for lb in d.get("labels") or [] if isinstance(lb, dict)]
+    reviewers: set[str] = set()
+    comments = []
+    if ctx.cfg.github.sync_review_comments:
+        for c in client.paginate(f"{base}/pulls/{n}/comments", 200):
+            comments.append(c)
+        for rv in client.paginate(f"{base}/pulls/{n}/reviews", 100):
+            if isinstance(rv, dict) and rv.get("user"):
+                reviewers.add(rv["user"].get("login", ""))
+                if rv.get("body"):
+                    comments.append({**rv, "path": None, "line": None})
+    with ctx.store.transaction():
+        ctx.store.conn.execute(
+            "INSERT OR REPLACE INTO prs VALUES(?,?,?,?,?,?,?,?,?,?,?)",
+            (n, d.get("title"), d.get("body") or "", json.dumps(labels), d.get("state"),
+             d.get("merged_at"), d.get("merge_commit_sha"), d.get("html_url"),
+             (d.get("user") or {}).get("login"), json.dumps(sorted(reviewers)),
+             int(time.time())))
+        for c in comments:
+            if not isinstance(c, dict) or "id" not in c:
+                continue
+            ctx.store.conn.execute(
+                "INSERT OR REPLACE INTO review_comments VALUES(?,?,?,?,?,?,?,?)",
+                (c["id"], n, c.get("path"), c.get("line"), c.get("body") or "",
+                 (c.get("user") or {}).get("login"), c.get("html_url"),
+                 c.get("created_at") or c.get("submitted_at")))
+    rep.prs += 1
+    rep.review_comments += len(comments)
+    if ctx.cfg.github.sync_issues:
+        refs = {d for s, d in extract_text(d.get("body") or "")
+                if s in ("text.closes_issue", "text.issue_ref") and d.startswith("#")}
+        for ref in sorted(refs)[:10]:
+            _sync_issue(ctx, client, base, int(ref[1:]), n, rep)
+
+
+def _sync_issue(ctx: Context, client: Client, base: str, n: int, pr: int,
+                rep: SyncReport) -> None:
+    if ctx.store.conn.execute("SELECT 1 FROM issues WHERE number=?", (n,)).fetchone():
+        return
+    r = client.get(f"{base}/issues/{n}")
+    with ctx.store.transaction():
+        if r.status != 200 or not isinstance(r.data, dict):
+            ctx.store.conn.execute(
+                "INSERT OR REPLACE INTO issues(number, fetched_at, retrieval_status) "
+                "VALUES(?, ?, 'unreachable')", (n, int(time.time())))
+            rep.unreachable += 1
+            return
+        d = r.data
+        labels = [lb.get("name", "") for lb in d.get("labels") or [] if isinstance(lb, dict)]
+        ctx.store.conn.execute(
+            "INSERT OR REPLACE INTO issues VALUES(?,?,?,?,?,?,?,?,'ok')",
+            (n, d.get("title"), d.get("body") or "", json.dumps(labels), d.get("state"),
+             d.get("closed_at"), d.get("html_url"), int(time.time())))
+        ctx.store.conn.execute("INSERT OR IGNORE INTO pr_issue_links VALUES(?,?,?)",
+                               (pr, n, "references"))
+    rep.issues += 1

fence/globs.py

@@ -0,0 +1,43 @@
+"""Gitignore-style globs for include/exclude/ignore_paths: `*`, `?`, `**`."""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Iterable
+from functools import lru_cache
+
+
+@lru_cache(maxsize=512)
+def _compile(pattern: str) -> re.Pattern[str]:
+    p = pattern.strip().lstrip("/")
+    if p.endswith("/"):
+        p += "**"
+    out = []
+    i = 0
+    while i < len(p):
+        if p.startswith("**/", i):
+            out.append("(?:.*/)?")
+            i += 3
+        elif p.startswith("**", i):
+            out.append(".*")
+            i += 2
+        elif p[i] == "*":
+            out.append("[^/]*")
+            i += 1
+        elif p[i] == "?":
+            out.append("[^/]")
+            i += 1
+        else:
+            out.append(re.escape(p[i]))
+            i += 1
+    # A pattern without a slash matches at any depth, like .gitignore.
+    prefix = "" if "/" in pattern.strip().rstrip("/") else "(?:.*/)?"
+    return re.compile(prefix + "".join(out) + r"\Z")
+
+
+def match_any(path: str, patterns: Iterable[str]) -> bool:
+    return any(_compile(p).match(path) for p in patterns)
+
+
+def included(path: str, include: Iterable[str], exclude: Iterable[str]) -> bool:
+    return match_any(path, include) and not match_any(path, exclude)