commitguardian 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- commitguard/__init__.py +26 -0
- commitguard/__main__.py +6 -0
- commitguard/api/__init__.py +18 -0
- commitguard/api/app.py +1376 -0
- commitguard/api/governance.py +1085 -0
- commitguard/api/hosting.py +196 -0
- commitguard/api/http.py +252 -0
- commitguard/api/settings.py +169 -0
- commitguard/audit/__init__.py +13 -0
- commitguard/audit/logger.py +34 -0
- commitguard/audit/models.py +222 -0
- commitguard/audit/storage.py +59 -0
- commitguard/ci/__init__.py +7 -0
- commitguard/ci/context.py +60 -0
- commitguard/cli/__init__.py +6 -0
- commitguard/cli/app.py +74 -0
- commitguard/cli/commands/__init__.py +1 -0
- commitguard/cli/commands/benchmark.py +441 -0
- commitguard/cli/commands/check.py +100 -0
- commitguard/cli/commands/ci.py +165 -0
- commitguard/cli/commands/dashboard.py +141 -0
- commitguard/cli/commands/doctor.py +533 -0
- commitguard/cli/commands/github.py +449 -0
- commitguard/cli/commands/hook.py +156 -0
- commitguard/cli/commands/init.py +137 -0
- commitguard/cli/commands/install.py +152 -0
- commitguard/cli/commands/policy.py +36 -0
- commitguard/cli/commands/report.py +39 -0
- commitguard/cli/commands/reproduce.py +123 -0
- commitguard/cli/commands/scan.py +47 -0
- commitguard/cli/common.py +44 -0
- commitguard/cli/output.py +89 -0
- commitguard/cli/render.py +367 -0
- commitguard/config/__init__.py +6 -0
- commitguard/config/defaults.py +53 -0
- commitguard/config/enforcement.py +53 -0
- commitguard/config/loader.py +174 -0
- commitguard/config/schema.py +105 -0
- commitguard/config/sources.py +183 -0
- commitguard/controlplane/__init__.py +24 -0
- commitguard/controlplane/access.py +231 -0
- commitguard/controlplane/commands.py +393 -0
- commitguard/controlplane/errors.py +88 -0
- commitguard/controlplane/identity.py +478 -0
- commitguard/controlplane/members.py +219 -0
- commitguard/controlplane/notifications.py +787 -0
- commitguard/controlplane/pagination.py +146 -0
- commitguard/controlplane/policies.py +1204 -0
- commitguard/controlplane/queries.py +1814 -0
- commitguard/controlplane/results.py +909 -0
- commitguard/controlplane/rules.py +184 -0
- commitguard/controlplane/views.py +799 -0
- commitguard/core/__init__.py +6 -0
- commitguard/core/context.py +31 -0
- commitguard/core/decision.py +58 -0
- commitguard/core/engine.py +82 -0
- commitguard/core/result.py +177 -0
- commitguard/detectors/__init__.py +6 -0
- commitguard/detectors/base.py +58 -0
- commitguard/detectors/bot.py +87 -0
- commitguard/detectors/coauthor.py +86 -0
- commitguard/detectors/identity.py +76 -0
- commitguard/detectors/registry.py +72 -0
- commitguard/detectors/trailer.py +211 -0
- commitguard/exceptions/__init__.py +33 -0
- commitguard/exceptions/base.py +9 -0
- commitguard/exceptions/configuration.py +22 -0
- commitguard/exceptions/detection.py +11 -0
- commitguard/exceptions/git.py +41 -0
- commitguard/exceptions/service.py +25 -0
- commitguard/git/__init__.py +12 -0
- commitguard/git/commands.py +101 -0
- commitguard/git/commit.py +97 -0
- commitguard/git/diff.py +36 -0
- commitguard/git/hooks.py +527 -0
- commitguard/git/push.py +93 -0
- commitguard/git/ranges.py +71 -0
- commitguard/git/repository.py +447 -0
- commitguard/github/__init__.py +34 -0
- commitguard/github/actions.py +163 -0
- commitguard/github/app.py +935 -0
- commitguard/github/auth.py +217 -0
- commitguard/github/check_runs.py +172 -0
- commitguard/github/checks.py +210 -0
- commitguard/github/client.py +844 -0
- commitguard/github/enforcement_status.py +209 -0
- commitguard/github/errors.py +129 -0
- commitguard/github/events.py +563 -0
- commitguard/github/identifiers.py +90 -0
- commitguard/github/installations.py +566 -0
- commitguard/github/markdown.py +19 -0
- commitguard/github/permissions.py +70 -0
- commitguard/github/pull_requests.py +53 -0
- commitguard/github/queue.py +47 -0
- commitguard/github/recovery.py +124 -0
- commitguard/github/repositories.py +305 -0
- commitguard/github/server.py +52 -0
- commitguard/github/settings.py +174 -0
- commitguard/github/storage.py +2315 -0
- commitguard/github/webhooks.py +129 -0
- commitguard/github/worker.py +628 -0
- commitguard/github/workflow.py +286 -0
- commitguard/governance/__init__.py +26 -0
- commitguard/governance/bulk.py +765 -0
- commitguard/governance/cache.py +88 -0
- commitguard/governance/common.py +216 -0
- commitguard/governance/exceptions.py +861 -0
- commitguard/governance/groups.py +448 -0
- commitguard/governance/inventory.py +386 -0
- commitguard/governance/posture.py +1272 -0
- commitguard/governance/resolver.py +632 -0
- commitguard/governance/rollouts.py +760 -0
- commitguard/governance/rules.py +371 -0
- commitguard/governance/schedules.py +663 -0
- commitguard/governance/service.py +120 -0
- commitguard/governance/settings.py +365 -0
- commitguard/governance/simulation.py +618 -0
- commitguard/governance/workflow.py +734 -0
- commitguard/notifications/__init__.py +2 -0
- commitguard/notifications/channels/__init__.py +1 -0
- commitguard/notifications/channels/base.py +22 -0
- commitguard/notifications/channels/email.py +110 -0
- commitguard/notifications/channels/in_app.py +74 -0
- commitguard/notifications/channels/sink.py +58 -0
- commitguard/notifications/channels/webhook.py +233 -0
- commitguard/notifications/deduplication.py +57 -0
- commitguard/notifications/dispatcher.py +201 -0
- commitguard/notifications/models.py +439 -0
- commitguard/notifications/outbox.py +106 -0
- commitguard/notifications/preferences.py +224 -0
- commitguard/notifications/retry.py +282 -0
- commitguard/notifications/service.py +128 -0
- commitguard/notifications/settings.py +167 -0
- commitguard/notifications/templates.py +108 -0
- commitguard/observability/__init__.py +5 -0
- commitguard/observability/logging.py +161 -0
- commitguard/observability/metrics.py +105 -0
- commitguard/policies/__init__.py +6 -0
- commitguard/policies/defaults.py +48 -0
- commitguard/policies/evaluator.py +66 -0
- commitguard/policies/governance.py +498 -0
- commitguard/policies/loader.py +23 -0
- commitguard/policies/mandatory.py +52 -0
- commitguard/policies/model.py +46 -0
- commitguard/provenance/__init__.py +9 -0
- commitguard/provenance/author.py +146 -0
- commitguard/provenance/committer.py +16 -0
- commitguard/provenance/normalization.py +158 -0
- commitguard/provenance/signatures.py +34 -0
- commitguard/provenance/trailers.py +256 -0
- commitguard/research/__init__.py +26 -0
- commitguard/research/compare.py +231 -0
- commitguard/research/datasets.py +1484 -0
- commitguard/research/detection.py +183 -0
- commitguard/research/environment.py +185 -0
- commitguard/research/gitenv.py +108 -0
- commitguard/research/hooks.py +247 -0
- commitguard/research/metrics.py +85 -0
- commitguard/research/performance.py +194 -0
- commitguard/research/platform.py +288 -0
- commitguard/research/report.py +372 -0
- commitguard/research/repository.py +111 -0
- commitguard/research/reproduction.py +297 -0
- commitguard/research/results.py +94 -0
- commitguard/rules/__init__.py +11 -0
- commitguard/rules/data/ai-domains.yaml +51 -0
- commitguard/rules/data/ai-identities.yaml +131 -0
- commitguard/rules/data/bot-identities.yaml +53 -0
- commitguard/rules/data/patterns.yaml +52 -0
- commitguard/rules/loader.py +102 -0
- commitguard/rules/matcher.py +212 -0
- commitguard/rules/models.py +269 -0
- commitguard/security/__init__.py +5 -0
- commitguard/security/hashing.py +30 -0
- commitguard/security/rate_limit.py +33 -0
- commitguard/security/safe_yaml.py +69 -0
- commitguard/security/sanitization.py +85 -0
- commitguard/security/secrets.py +169 -0
- commitguard/security/validation.py +89 -0
- commitguard/services/__init__.py +15 -0
- commitguard/services/analysis.py +119 -0
- commitguard/services/audit.py +95 -0
- commitguard/services/ci.py +383 -0
- commitguard/services/enforcement.py +102 -0
- commitguard/services/hooks.py +254 -0
- commitguard/services/remediation.py +99 -0
- commitguard/services/reports.py +146 -0
- commitguard/services/scan.py +172 -0
- commitguard/utils/__init__.py +1 -0
- commitguard/utils/filesystem.py +72 -0
- commitguard/utils/platform.py +35 -0
- commitguard/utils/subprocess.py +84 -0
- commitguardian-0.1.0.dist-info/METADATA +694 -0
- commitguardian-0.1.0.dist-info/RECORD +197 -0
- commitguardian-0.1.0.dist-info/WHEEL +4 -0
- commitguardian-0.1.0.dist-info/entry_points.txt +2 -0
- commitguardian-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,2315 @@
|
|
|
1
|
+
"""Durable state for the GitHub App.
|
|
2
|
+
|
|
3
|
+
Interfaces (so PostgreSQL or another backend can replace SQLite later):
|
|
4
|
+
|
|
5
|
+
* :class:`DeliveryRepository` - webhook delivery IDs (replay protection);
|
|
6
|
+
* :class:`InstallationRepository` - installations and the repositories they grant;
|
|
7
|
+
* :class:`ScanRepository` - scan jobs, their states and Check Run ownership;
|
|
8
|
+
* :class:`~commitguard.audit.storage.AuditStorage` - audit events.
|
|
9
|
+
|
|
10
|
+
The dashboard's control plane (:mod:`commitguard.controlplane.store`) keeps its
|
|
11
|
+
tables - findings, violations, users, sessions, memberships, organisation
|
|
12
|
+
policy versions - in the same database, through :meth:`SqliteStateStore.transaction`
|
|
13
|
+
and :meth:`SqliteStateStore.query`.
|
|
14
|
+
|
|
15
|
+
Schema changes are ordered migrations (:data:`_MIGRATIONS`); a database written
|
|
16
|
+
by an older version is upgraded in place inside one transaction, and a database
|
|
17
|
+
from a newer version is refused.
|
|
18
|
+
|
|
19
|
+
:class:`SqliteStateStore` implements all four with the standard-library
|
|
20
|
+
``sqlite3`` module (no new dependency). It is safe for many threads in one
|
|
21
|
+
process and for several processes on one host (WAL mode, ``BEGIN IMMEDIATE``
|
|
22
|
+
for read-modify-write). Deployments with several hosts need a shared database
|
|
23
|
+
implementation of the same interfaces.
|
|
24
|
+
|
|
25
|
+
Data minimisation: the store holds IDs, repository names, commit SHAs, states,
|
|
26
|
+
counts, rule IDs and - for commits with a finding only - the finding's evidence
|
|
27
|
+
and the commit's author and committer identity. It never stores commit
|
|
28
|
+
messages, file contents, tokens or keys. Every table has a timestamp used by
|
|
29
|
+
:meth:`SqliteStateStore.purge_expired` for retention.
|
|
30
|
+
|
|
31
|
+
Tenant isolation: all lookups that could be exposed later take the
|
|
32
|
+
installation ID (and repository ID) as mandatory filters.
|
|
33
|
+
"""
|
|
34
|
+
|
|
35
|
+
import json
|
|
36
|
+
import os
|
|
37
|
+
import re
|
|
38
|
+
import sqlite3
|
|
39
|
+
import threading
|
|
40
|
+
import uuid
|
|
41
|
+
from collections.abc import Iterator, Sequence
|
|
42
|
+
from contextlib import contextmanager
|
|
43
|
+
from datetime import UTC, datetime
|
|
44
|
+
from enum import StrEnum
|
|
45
|
+
from pathlib import Path
|
|
46
|
+
from typing import Any, Protocol
|
|
47
|
+
|
|
48
|
+
from pydantic import BaseModel, ConfigDict
|
|
49
|
+
|
|
50
|
+
from commitguard.audit.models import AuditEvent
|
|
51
|
+
from commitguard.audit.storage import AuditStorage
|
|
52
|
+
from commitguard.ci.context import CIContext
|
|
53
|
+
from commitguard.exceptions.service import InfrastructureError
|
|
54
|
+
from commitguard.github.identifiers import AccountType, RepositoryRef
|
|
55
|
+
|
|
56
|
+
SCHEMA_VERSION = 4
|
|
57
|
+
DATABASE_FILENAME = "commitguard-app.sqlite3"
|
|
58
|
+
DEFAULT_MAX_ATTEMPTS = 3
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def _ts(value: datetime) -> float:
|
|
62
|
+
return value.timestamp()
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _dt(value: float) -> datetime:
|
|
66
|
+
return datetime.fromtimestamp(value, UTC)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _opt_dt(value: float | None) -> datetime | None:
|
|
70
|
+
return None if value is None else _dt(value)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
# --------------------------------------------------------------------------- #
|
|
74
|
+
# Records
|
|
75
|
+
# --------------------------------------------------------------------------- #
|
|
76
|
+
class DeliveryStatus(StrEnum):
|
|
77
|
+
NEW = "new"
|
|
78
|
+
DUPLICATE = "duplicate" # same delivery ID, same payload: already handled
|
|
79
|
+
CONFLICT = "conflict" # same delivery ID, different payload: never processed
|
|
80
|
+
RETRY = "retry" # same delivery ID and payload, earlier processing failed: process again
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
class EventProcessingStatus(StrEnum):
|
|
84
|
+
"""Processing state of a received webhook delivery (the event record)."""
|
|
85
|
+
|
|
86
|
+
RECEIVED = "received"
|
|
87
|
+
PROCESSING = "processing"
|
|
88
|
+
PROCESSED = "processed"
|
|
89
|
+
FAILED = "failed"
|
|
90
|
+
IGNORED = "ignored"
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
#: A delivery still "processing" after this long is treated as abandoned (process crash).
|
|
94
|
+
STUCK_DELIVERY_SECONDS = 600.0
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
class ScanTrigger(StrEnum):
|
|
98
|
+
"""Why a scan execution exists."""
|
|
99
|
+
|
|
100
|
+
PUSH = "push"
|
|
101
|
+
PULL_REQUEST = "pull_request"
|
|
102
|
+
MERGE_GROUP = "merge_group"
|
|
103
|
+
MANUAL = "manual" # dashboard "Scan again"
|
|
104
|
+
RERUN = "rerun" # GitHub "Re-run" on the CommitGuard check
|
|
105
|
+
RETRY = "retry" # automatic recovery after an infrastructure failure
|
|
106
|
+
SCHEDULED = "scheduled" # an organization scan schedule (default branch)
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
class MergeGroupState(StrEnum):
|
|
110
|
+
CHECKS_REQUESTED = "checks_requested"
|
|
111
|
+
DESTROYED = "destroyed"
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
class MergeGroupRecord(BaseModel):
|
|
115
|
+
model_config = ConfigDict(frozen=True, extra="forbid")
|
|
116
|
+
|
|
117
|
+
installation_id: int
|
|
118
|
+
repository_id: int
|
|
119
|
+
head_sha: str
|
|
120
|
+
head_ref: str
|
|
121
|
+
base_sha: str
|
|
122
|
+
base_ref: str
|
|
123
|
+
pull_requests: tuple[int, ...]
|
|
124
|
+
state: MergeGroupState
|
|
125
|
+
destroyed_reason: str | None
|
|
126
|
+
job_id: str | None
|
|
127
|
+
created_at: datetime
|
|
128
|
+
updated_at: datetime
|
|
129
|
+
destroyed_at: datetime | None
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
class InstallationState(StrEnum):
|
|
133
|
+
ACTIVE = "active"
|
|
134
|
+
SUSPENDED = "suspended"
|
|
135
|
+
DELETED = "deleted"
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
class InstallationRecord(BaseModel):
|
|
139
|
+
model_config = ConfigDict(frozen=True, extra="forbid")
|
|
140
|
+
|
|
141
|
+
installation_id: int
|
|
142
|
+
account_id: int
|
|
143
|
+
account_login: str
|
|
144
|
+
account_type: AccountType
|
|
145
|
+
repository_selection: str
|
|
146
|
+
state: InstallationState
|
|
147
|
+
permissions: dict[str, str] = {}
|
|
148
|
+
created_at: datetime
|
|
149
|
+
updated_at: datetime
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
class JobState(StrEnum):
|
|
153
|
+
QUEUED = "queued"
|
|
154
|
+
RUNNING = "running"
|
|
155
|
+
PASSED = "passed" # scan completed, policy allowed (possibly with warnings)
|
|
156
|
+
FAILED = "failed" # scan completed, policy blocked
|
|
157
|
+
ERROR = "error" # scan could not be completed (system failure)
|
|
158
|
+
CANCELLED = "cancelled" # superseded, stale or no longer relevant
|
|
159
|
+
|
|
160
|
+
@property
|
|
161
|
+
def terminal(self) -> bool:
|
|
162
|
+
return self not in (JobState.QUEUED, JobState.RUNNING)
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
class ScanJob(BaseModel):
|
|
166
|
+
model_config = ConfigDict(frozen=True, extra="forbid")
|
|
167
|
+
|
|
168
|
+
job_id: str
|
|
169
|
+
sequence: int
|
|
170
|
+
job_key: str
|
|
171
|
+
installation_id: int
|
|
172
|
+
repository: RepositoryRef
|
|
173
|
+
delivery_id: str | None
|
|
174
|
+
event: str
|
|
175
|
+
group_key: str
|
|
176
|
+
head_sha: str
|
|
177
|
+
check_name: str
|
|
178
|
+
pull_request_number: int | None
|
|
179
|
+
context: CIContext
|
|
180
|
+
state: JobState
|
|
181
|
+
attempts: int
|
|
182
|
+
created_at: datetime
|
|
183
|
+
updated_at: datetime
|
|
184
|
+
lease_expires_at: datetime | None = None
|
|
185
|
+
scan_id: str | None = None
|
|
186
|
+
check_run_id: int | None = None
|
|
187
|
+
result_action: str | None = None
|
|
188
|
+
conclusion: str | None = None
|
|
189
|
+
failure_kind: str | None = None
|
|
190
|
+
message: str | None = None
|
|
191
|
+
commits_scanned: int | None = None
|
|
192
|
+
violations: int | None = None
|
|
193
|
+
warnings: int | None = None
|
|
194
|
+
base_sha: str | None = None
|
|
195
|
+
ref: str | None = None
|
|
196
|
+
started_at: datetime | None = None
|
|
197
|
+
completed_at: datetime | None = None
|
|
198
|
+
tool_version: str | None = None
|
|
199
|
+
rules_version: str | None = None
|
|
200
|
+
policy_version: str | None = None
|
|
201
|
+
policy_source: str | None = None
|
|
202
|
+
organization_policy_version: int | None = None
|
|
203
|
+
effective_policies: tuple[dict[str, str | bool], ...] = ()
|
|
204
|
+
findings_count: int | None = None
|
|
205
|
+
detector_failures: int | None = None
|
|
206
|
+
notices: tuple[str, ...] = ()
|
|
207
|
+
requested_by: str | None = None
|
|
208
|
+
scan_key: str = ""
|
|
209
|
+
execution: int = 1
|
|
210
|
+
trigger: ScanTrigger = ScanTrigger.PUSH
|
|
211
|
+
previous_job_id: str | None = None
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
class NewScanJob(BaseModel):
|
|
215
|
+
model_config = ConfigDict(frozen=True, extra="forbid")
|
|
216
|
+
|
|
217
|
+
job_key: str
|
|
218
|
+
installation_id: int
|
|
219
|
+
repository: RepositoryRef
|
|
220
|
+
delivery_id: str | None
|
|
221
|
+
event: str
|
|
222
|
+
group_key: str
|
|
223
|
+
head_sha: str
|
|
224
|
+
check_name: str
|
|
225
|
+
pull_request_number: int | None
|
|
226
|
+
context: CIContext
|
|
227
|
+
requested_by: str | None = None # dashboard login for a manual re-scan
|
|
228
|
+
|
|
229
|
+
@property
|
|
230
|
+
def trigger(self) -> ScanTrigger:
|
|
231
|
+
return ScanTrigger(self.event)
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
class ClaimOutcome(BaseModel):
|
|
235
|
+
"""Result of claiming a job: the running job, or the job that ran out of attempts."""
|
|
236
|
+
|
|
237
|
+
model_config = ConfigDict(frozen=True, extra="forbid")
|
|
238
|
+
|
|
239
|
+
job: ScanJob | None = None
|
|
240
|
+
exhausted: ScanJob | None = None
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
class CheckClaim(BaseModel):
|
|
244
|
+
model_config = ConfigDict(frozen=True, extra="forbid")
|
|
245
|
+
|
|
246
|
+
owned: bool
|
|
247
|
+
owner_sequence: int
|
|
248
|
+
check_run_id: int | None
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
# --------------------------------------------------------------------------- #
|
|
252
|
+
# Interfaces
|
|
253
|
+
# --------------------------------------------------------------------------- #
|
|
254
|
+
class DeliveryRepository(Protocol):
|
|
255
|
+
def record_delivery(
|
|
256
|
+
self, delivery_id: str, event: str, body_sha256: str, received_at: datetime
|
|
257
|
+
) -> DeliveryStatus: ...
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
class InstallationRepository(Protocol):
|
|
261
|
+
def upsert_installation(self, record: InstallationRecord) -> None: ...
|
|
262
|
+
def get_installation(self, installation_id: int) -> InstallationRecord | None: ...
|
|
263
|
+
def set_installation_state(
|
|
264
|
+
self, installation_id: int, state: InstallationState, now: datetime
|
|
265
|
+
) -> None: ...
|
|
266
|
+
def replace_repositories(
|
|
267
|
+
self, installation_id: int, repositories: Sequence[RepositoryRef], now: datetime
|
|
268
|
+
) -> None: ...
|
|
269
|
+
def add_repositories(
|
|
270
|
+
self, installation_id: int, repositories: Sequence[RepositoryRef], now: datetime
|
|
271
|
+
) -> None: ...
|
|
272
|
+
def remove_repositories(
|
|
273
|
+
self, installation_id: int, repository_ids: Sequence[int], now: datetime | None = None
|
|
274
|
+
) -> None: ...
|
|
275
|
+
def repository_listed(self, installation_id: int, repository_id: int) -> bool: ...
|
|
276
|
+
def list_repositories(self, installation_id: int) -> list[RepositoryRef]: ...
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
class ScanRepository(Protocol):
|
|
280
|
+
def create_job(self, job: NewScanJob, now: datetime) -> tuple[ScanJob, bool]: ...
|
|
281
|
+
def get_job(self, job_id: str) -> ScanJob | None: ...
|
|
282
|
+
def list_jobs(
|
|
283
|
+
self, *, installation_id: int, repository_id: int | None = None, limit: int = 100
|
|
284
|
+
) -> list[ScanJob]: ...
|
|
285
|
+
def claim_job(
|
|
286
|
+
self, job_id: str, now: datetime, lease_seconds: float, max_attempts: int
|
|
287
|
+
) -> ScanJob | None: ...
|
|
288
|
+
def update_job(self, job_id: str, now: datetime, **fields: Any) -> None: ...
|
|
289
|
+
def recoverable_jobs(
|
|
290
|
+
self, now: datetime, *, queued_before: datetime, limit: int = 100
|
|
291
|
+
) -> list[str]: ...
|
|
292
|
+
def latest_group_sequence(
|
|
293
|
+
self, installation_id: int, repository_id: int, group_key: str
|
|
294
|
+
) -> int: ...
|
|
295
|
+
def cancel_queued_group(
|
|
296
|
+
self, installation_id: int, repository_id: int, group_key: str, now: datetime
|
|
297
|
+
) -> int: ...
|
|
298
|
+
def claim_check(
|
|
299
|
+
self,
|
|
300
|
+
installation_id: int,
|
|
301
|
+
repository_id: int,
|
|
302
|
+
head_sha: str,
|
|
303
|
+
check_name: str,
|
|
304
|
+
sequence: int,
|
|
305
|
+
now: datetime,
|
|
306
|
+
) -> CheckClaim: ...
|
|
307
|
+
def set_check_run_id(
|
|
308
|
+
self,
|
|
309
|
+
installation_id: int,
|
|
310
|
+
repository_id: int,
|
|
311
|
+
head_sha: str,
|
|
312
|
+
check_name: str,
|
|
313
|
+
sequence: int,
|
|
314
|
+
check_run_id: int,
|
|
315
|
+
) -> bool: ...
|
|
316
|
+
def check_owner(
|
|
317
|
+
self, installation_id: int, repository_id: int, head_sha: str, check_name: str
|
|
318
|
+
) -> int | None: ...
|
|
319
|
+
|
|
320
|
+
|
|
321
|
+
# --------------------------------------------------------------------------- #
|
|
322
|
+
# SQLite implementation
|
|
323
|
+
# --------------------------------------------------------------------------- #
|
|
324
|
+
_SCHEMA_V1 = """
|
|
325
|
+
CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT NOT NULL);
|
|
326
|
+
CREATE TABLE IF NOT EXISTS deliveries (
|
|
327
|
+
delivery_id TEXT PRIMARY KEY,
|
|
328
|
+
event TEXT NOT NULL,
|
|
329
|
+
body_sha256 TEXT NOT NULL,
|
|
330
|
+
received_at REAL NOT NULL
|
|
331
|
+
);
|
|
332
|
+
CREATE INDEX IF NOT EXISTS deliveries_received ON deliveries (received_at);
|
|
333
|
+
CREATE TABLE IF NOT EXISTS installations (
|
|
334
|
+
installation_id INTEGER PRIMARY KEY,
|
|
335
|
+
account_id INTEGER NOT NULL,
|
|
336
|
+
account_login TEXT NOT NULL,
|
|
337
|
+
account_type TEXT NOT NULL,
|
|
338
|
+
repository_selection TEXT NOT NULL,
|
|
339
|
+
state TEXT NOT NULL,
|
|
340
|
+
permissions TEXT NOT NULL,
|
|
341
|
+
created_at REAL NOT NULL,
|
|
342
|
+
updated_at REAL NOT NULL
|
|
343
|
+
);
|
|
344
|
+
CREATE TABLE IF NOT EXISTS installation_repositories (
|
|
345
|
+
installation_id INTEGER NOT NULL,
|
|
346
|
+
repository_id INTEGER NOT NULL,
|
|
347
|
+
owner TEXT NOT NULL,
|
|
348
|
+
name TEXT NOT NULL,
|
|
349
|
+
added_at REAL NOT NULL,
|
|
350
|
+
PRIMARY KEY (installation_id, repository_id)
|
|
351
|
+
);
|
|
352
|
+
CREATE TABLE IF NOT EXISTS scan_jobs (
|
|
353
|
+
sequence INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
354
|
+
job_id TEXT NOT NULL UNIQUE,
|
|
355
|
+
job_key TEXT NOT NULL,
|
|
356
|
+
installation_id INTEGER NOT NULL,
|
|
357
|
+
repository_id INTEGER NOT NULL,
|
|
358
|
+
owner TEXT NOT NULL,
|
|
359
|
+
name TEXT NOT NULL,
|
|
360
|
+
delivery_id TEXT,
|
|
361
|
+
event TEXT NOT NULL,
|
|
362
|
+
group_key TEXT NOT NULL,
|
|
363
|
+
head_sha TEXT NOT NULL,
|
|
364
|
+
check_name TEXT NOT NULL,
|
|
365
|
+
pull_request_number INTEGER,
|
|
366
|
+
context TEXT NOT NULL,
|
|
367
|
+
state TEXT NOT NULL,
|
|
368
|
+
attempts INTEGER NOT NULL DEFAULT 0,
|
|
369
|
+
created_at REAL NOT NULL,
|
|
370
|
+
updated_at REAL NOT NULL,
|
|
371
|
+
lease_expires_at REAL,
|
|
372
|
+
scan_id TEXT,
|
|
373
|
+
check_run_id INTEGER,
|
|
374
|
+
result_action TEXT,
|
|
375
|
+
conclusion TEXT,
|
|
376
|
+
failure_kind TEXT,
|
|
377
|
+
message TEXT,
|
|
378
|
+
commits_scanned INTEGER,
|
|
379
|
+
violations INTEGER,
|
|
380
|
+
warnings INTEGER
|
|
381
|
+
);
|
|
382
|
+
CREATE INDEX IF NOT EXISTS scan_jobs_key ON scan_jobs (installation_id, repository_id, job_key);
|
|
383
|
+
CREATE INDEX IF NOT EXISTS scan_jobs_group
|
|
384
|
+
ON scan_jobs (installation_id, repository_id, group_key, sequence);
|
|
385
|
+
CREATE INDEX IF NOT EXISTS scan_jobs_state ON scan_jobs (state, updated_at);
|
|
386
|
+
CREATE TABLE IF NOT EXISTS check_runs (
|
|
387
|
+
installation_id INTEGER NOT NULL,
|
|
388
|
+
repository_id INTEGER NOT NULL,
|
|
389
|
+
head_sha TEXT NOT NULL,
|
|
390
|
+
check_name TEXT NOT NULL,
|
|
391
|
+
owner_sequence INTEGER NOT NULL,
|
|
392
|
+
check_run_id INTEGER,
|
|
393
|
+
updated_at REAL NOT NULL,
|
|
394
|
+
PRIMARY KEY (installation_id, repository_id, head_sha, check_name)
|
|
395
|
+
);
|
|
396
|
+
CREATE TABLE IF NOT EXISTS audit_events (
|
|
397
|
+
event_id TEXT PRIMARY KEY,
|
|
398
|
+
occurred_at REAL NOT NULL,
|
|
399
|
+
type TEXT NOT NULL,
|
|
400
|
+
installation_id INTEGER,
|
|
401
|
+
repository_id INTEGER,
|
|
402
|
+
document TEXT NOT NULL
|
|
403
|
+
);
|
|
404
|
+
CREATE INDEX IF NOT EXISTS audit_tenant
|
|
405
|
+
ON audit_events (installation_id, repository_id, occurred_at);
|
|
406
|
+
"""
|
|
407
|
+
|
|
408
|
+
# Version 2: the dashboard control plane.
|
|
409
|
+
_SCHEMA_V2 = """
|
|
410
|
+
ALTER TABLE scan_jobs ADD COLUMN base_sha TEXT;
|
|
411
|
+
ALTER TABLE scan_jobs ADD COLUMN ref TEXT;
|
|
412
|
+
ALTER TABLE scan_jobs ADD COLUMN started_at REAL;
|
|
413
|
+
ALTER TABLE scan_jobs ADD COLUMN completed_at REAL;
|
|
414
|
+
ALTER TABLE scan_jobs ADD COLUMN tool_version TEXT;
|
|
415
|
+
ALTER TABLE scan_jobs ADD COLUMN rules_version TEXT;
|
|
416
|
+
ALTER TABLE scan_jobs ADD COLUMN policy_version TEXT;
|
|
417
|
+
ALTER TABLE scan_jobs ADD COLUMN policy_source TEXT;
|
|
418
|
+
ALTER TABLE scan_jobs ADD COLUMN organization_policy_version INTEGER;
|
|
419
|
+
ALTER TABLE scan_jobs ADD COLUMN effective_policies TEXT;
|
|
420
|
+
ALTER TABLE scan_jobs ADD COLUMN findings_count INTEGER;
|
|
421
|
+
ALTER TABLE scan_jobs ADD COLUMN detector_failures INTEGER;
|
|
422
|
+
ALTER TABLE scan_jobs ADD COLUMN notices TEXT;
|
|
423
|
+
ALTER TABLE scan_jobs ADD COLUMN requested_by TEXT;
|
|
424
|
+
CREATE INDEX scan_jobs_repository_sequence
|
|
425
|
+
ON scan_jobs (installation_id, repository_id, sequence);
|
|
426
|
+
CREATE INDEX scan_jobs_installation_created ON scan_jobs (installation_id, created_at);
|
|
427
|
+
|
|
428
|
+
CREATE TABLE known_repositories (
|
|
429
|
+
installation_id INTEGER NOT NULL,
|
|
430
|
+
repository_id INTEGER NOT NULL,
|
|
431
|
+
owner TEXT NOT NULL,
|
|
432
|
+
name TEXT NOT NULL,
|
|
433
|
+
default_branch TEXT,
|
|
434
|
+
first_seen_at REAL NOT NULL,
|
|
435
|
+
last_seen_at REAL NOT NULL,
|
|
436
|
+
removed_at REAL,
|
|
437
|
+
PRIMARY KEY (installation_id, repository_id)
|
|
438
|
+
);
|
|
439
|
+
INSERT OR IGNORE INTO known_repositories
|
|
440
|
+
(installation_id, repository_id, owner, name, first_seen_at, last_seen_at)
|
|
441
|
+
SELECT installation_id, repository_id, owner, name, added_at, added_at
|
|
442
|
+
FROM installation_repositories;
|
|
443
|
+
INSERT OR IGNORE INTO known_repositories
|
|
444
|
+
(installation_id, repository_id, owner, name, first_seen_at, last_seen_at, removed_at)
|
|
445
|
+
SELECT installation_id, repository_id, owner, name, MIN(created_at), MAX(updated_at),
|
|
446
|
+
MAX(updated_at)
|
|
447
|
+
FROM scan_jobs GROUP BY installation_id, repository_id;
|
|
448
|
+
|
|
449
|
+
ALTER TABLE audit_events ADD COLUMN account_id INTEGER;
|
|
450
|
+
ALTER TABLE audit_events ADD COLUMN actor_login TEXT;
|
|
451
|
+
UPDATE audit_events SET account_id = (
|
|
452
|
+
SELECT account_id FROM installations i WHERE i.installation_id = audit_events.installation_id
|
|
453
|
+
);
|
|
454
|
+
CREATE INDEX audit_account ON audit_events (account_id, occurred_at);
|
|
455
|
+
|
|
456
|
+
CREATE TABLE findings (
|
|
457
|
+
finding_id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
458
|
+
job_id TEXT NOT NULL,
|
|
459
|
+
installation_id INTEGER NOT NULL,
|
|
460
|
+
repository_id INTEGER NOT NULL,
|
|
461
|
+
violation_id TEXT,
|
|
462
|
+
fingerprint TEXT NOT NULL,
|
|
463
|
+
commit_sha TEXT,
|
|
464
|
+
rule_id TEXT NOT NULL,
|
|
465
|
+
detector TEXT NOT NULL,
|
|
466
|
+
severity TEXT NOT NULL,
|
|
467
|
+
severity_rank INTEGER NOT NULL,
|
|
468
|
+
confidence TEXT NOT NULL,
|
|
469
|
+
action TEXT NOT NULL,
|
|
470
|
+
policy_id TEXT,
|
|
471
|
+
reason TEXT NOT NULL,
|
|
472
|
+
title TEXT NOT NULL,
|
|
473
|
+
message TEXT NOT NULL,
|
|
474
|
+
remediation TEXT NOT NULL,
|
|
475
|
+
evidence TEXT NOT NULL,
|
|
476
|
+
author TEXT,
|
|
477
|
+
committer TEXT,
|
|
478
|
+
created_at REAL NOT NULL
|
|
479
|
+
);
|
|
480
|
+
CREATE INDEX findings_job ON findings (job_id, finding_id);
|
|
481
|
+
CREATE INDEX findings_violation ON findings (violation_id, finding_id);
|
|
482
|
+
CREATE INDEX findings_rule ON findings (installation_id, rule_id, job_id);
|
|
483
|
+
CREATE INDEX findings_created ON findings (created_at);
|
|
484
|
+
|
|
485
|
+
CREATE TABLE violations (
|
|
486
|
+
violation_id TEXT PRIMARY KEY,
|
|
487
|
+
installation_id INTEGER NOT NULL,
|
|
488
|
+
repository_id INTEGER NOT NULL,
|
|
489
|
+
fingerprint TEXT NOT NULL,
|
|
490
|
+
rule_id TEXT NOT NULL,
|
|
491
|
+
detector TEXT NOT NULL,
|
|
492
|
+
severity TEXT NOT NULL,
|
|
493
|
+
severity_rank INTEGER NOT NULL,
|
|
494
|
+
action TEXT NOT NULL,
|
|
495
|
+
title TEXT NOT NULL,
|
|
496
|
+
commit_sha TEXT,
|
|
497
|
+
author TEXT,
|
|
498
|
+
status TEXT NOT NULL,
|
|
499
|
+
first_detected_at REAL NOT NULL,
|
|
500
|
+
last_detected_at REAL NOT NULL,
|
|
501
|
+
first_job_id TEXT NOT NULL,
|
|
502
|
+
last_job_id TEXT NOT NULL,
|
|
503
|
+
detections INTEGER NOT NULL,
|
|
504
|
+
resolved_at REAL,
|
|
505
|
+
resolution TEXT,
|
|
506
|
+
acknowledged_at REAL,
|
|
507
|
+
acknowledged_by_id INTEGER,
|
|
508
|
+
acknowledged_by_login TEXT,
|
|
509
|
+
acknowledgement_note TEXT,
|
|
510
|
+
updated_at REAL NOT NULL,
|
|
511
|
+
UNIQUE (installation_id, repository_id, fingerprint)
|
|
512
|
+
);
|
|
513
|
+
CREATE INDEX violations_detected ON violations (installation_id, last_detected_at);
|
|
514
|
+
CREATE INDEX violations_repository ON violations (installation_id, repository_id, status);
|
|
515
|
+
CREATE INDEX violations_status ON violations (installation_id, status, severity_rank);
|
|
516
|
+
CREATE INDEX violations_rule ON violations (installation_id, rule_id);
|
|
517
|
+
|
|
518
|
+
CREATE TABLE violation_exposures (
|
|
519
|
+
violation_id TEXT NOT NULL,
|
|
520
|
+
installation_id INTEGER NOT NULL,
|
|
521
|
+
repository_id INTEGER NOT NULL,
|
|
522
|
+
group_key TEXT NOT NULL,
|
|
523
|
+
kind TEXT NOT NULL,
|
|
524
|
+
label TEXT NOT NULL,
|
|
525
|
+
active INTEGER NOT NULL,
|
|
526
|
+
first_job_id TEXT,
|
|
527
|
+
last_job_id TEXT,
|
|
528
|
+
opened_at REAL NOT NULL,
|
|
529
|
+
closed_at REAL,
|
|
530
|
+
closed_reason TEXT,
|
|
531
|
+
PRIMARY KEY (violation_id, group_key)
|
|
532
|
+
);
|
|
533
|
+
CREATE INDEX exposures_group
|
|
534
|
+
ON violation_exposures (installation_id, repository_id, group_key, active);
|
|
535
|
+
|
|
536
|
+
CREATE TABLE users (
|
|
537
|
+
user_id INTEGER PRIMARY KEY,
|
|
538
|
+
login TEXT NOT NULL,
|
|
539
|
+
created_at REAL NOT NULL,
|
|
540
|
+
last_login_at REAL
|
|
541
|
+
);
|
|
542
|
+
CREATE TABLE memberships (
|
|
543
|
+
account_id INTEGER NOT NULL,
|
|
544
|
+
user_id INTEGER NOT NULL,
|
|
545
|
+
role TEXT NOT NULL,
|
|
546
|
+
granted_by TEXT NOT NULL,
|
|
547
|
+
created_at REAL NOT NULL,
|
|
548
|
+
updated_at REAL NOT NULL,
|
|
549
|
+
PRIMARY KEY (account_id, user_id)
|
|
550
|
+
);
|
|
551
|
+
CREATE INDEX memberships_user ON memberships (user_id);
|
|
552
|
+
CREATE TABLE sessions (
|
|
553
|
+
session_hash TEXT PRIMARY KEY,
|
|
554
|
+
public_id TEXT NOT NULL UNIQUE,
|
|
555
|
+
user_id INTEGER NOT NULL,
|
|
556
|
+
created_at REAL NOT NULL,
|
|
557
|
+
authenticated_at REAL NOT NULL,
|
|
558
|
+
last_seen_at REAL NOT NULL,
|
|
559
|
+
expires_at REAL NOT NULL,
|
|
560
|
+
user_agent TEXT NOT NULL
|
|
561
|
+
);
|
|
562
|
+
CREATE INDEX sessions_user ON sessions (user_id);
|
|
563
|
+
CREATE INDEX sessions_expiry ON sessions (expires_at);
|
|
564
|
+
CREATE TABLE session_installations (
|
|
565
|
+
session_hash TEXT NOT NULL REFERENCES sessions (session_hash) ON DELETE CASCADE,
|
|
566
|
+
installation_id INTEGER NOT NULL,
|
|
567
|
+
PRIMARY KEY (session_hash, installation_id)
|
|
568
|
+
);
|
|
569
|
+
CREATE TABLE session_repositories (
|
|
570
|
+
session_hash TEXT NOT NULL REFERENCES sessions (session_hash) ON DELETE CASCADE,
|
|
571
|
+
installation_id INTEGER NOT NULL,
|
|
572
|
+
repository_id INTEGER NOT NULL,
|
|
573
|
+
PRIMARY KEY (session_hash, installation_id, repository_id)
|
|
574
|
+
);
|
|
575
|
+
CREATE TABLE oauth_states (
|
|
576
|
+
state_hash TEXT PRIMARY KEY,
|
|
577
|
+
verifier TEXT NOT NULL,
|
|
578
|
+
return_to TEXT NOT NULL,
|
|
579
|
+
expires_at REAL NOT NULL
|
|
580
|
+
);
|
|
581
|
+
|
|
582
|
+
CREATE TABLE organization_policy_versions (
|
|
583
|
+
account_id INTEGER NOT NULL,
|
|
584
|
+
version INTEGER NOT NULL,
|
|
585
|
+
document TEXT NOT NULL,
|
|
586
|
+
fingerprint TEXT NOT NULL,
|
|
587
|
+
created_at REAL NOT NULL,
|
|
588
|
+
created_by_id INTEGER,
|
|
589
|
+
created_by_login TEXT,
|
|
590
|
+
reason TEXT,
|
|
591
|
+
PRIMARY KEY (account_id, version)
|
|
592
|
+
);
|
|
593
|
+
|
|
594
|
+
CREATE TABLE repository_settings (
|
|
595
|
+
installation_id INTEGER NOT NULL,
|
|
596
|
+
repository_id INTEGER NOT NULL,
|
|
597
|
+
monitoring_enabled INTEGER NOT NULL,
|
|
598
|
+
updated_at REAL NOT NULL,
|
|
599
|
+
updated_by_login TEXT,
|
|
600
|
+
PRIMARY KEY (installation_id, repository_id)
|
|
601
|
+
);
|
|
602
|
+
CREATE TABLE enforcement_status (
|
|
603
|
+
installation_id INTEGER NOT NULL,
|
|
604
|
+
repository_id INTEGER NOT NULL,
|
|
605
|
+
checked_at REAL NOT NULL,
|
|
606
|
+
branch TEXT,
|
|
607
|
+
actions TEXT NOT NULL,
|
|
608
|
+
actions_detail TEXT NOT NULL,
|
|
609
|
+
branch_protection TEXT NOT NULL,
|
|
610
|
+
required_checks TEXT NOT NULL,
|
|
611
|
+
branch_protection_detail TEXT NOT NULL,
|
|
612
|
+
PRIMARY KEY (installation_id, repository_id)
|
|
613
|
+
);
|
|
614
|
+
"""
|
|
615
|
+
|
|
616
|
+
# Version 3: scan executions, event processing, merge queue, policy history, notifications.
|
|
617
|
+
_SCHEMA_V3 = """
|
|
618
|
+
ALTER TABLE scan_jobs ADD COLUMN scan_key TEXT;
|
|
619
|
+
ALTER TABLE scan_jobs ADD COLUMN execution INTEGER NOT NULL DEFAULT 1;
|
|
620
|
+
ALTER TABLE scan_jobs ADD COLUMN trigger_kind TEXT;
|
|
621
|
+
ALTER TABLE scan_jobs ADD COLUMN previous_job_id TEXT;
|
|
622
|
+
UPDATE scan_jobs SET scan_key = job_key, trigger_kind = event;
|
|
623
|
+
UPDATE scan_jobs SET trigger_kind = 'manual', scan_key = COALESCE((
|
|
624
|
+
SELECT o.job_key FROM scan_jobs o
|
|
625
|
+
WHERE o.installation_id = scan_jobs.installation_id
|
|
626
|
+
AND o.repository_id = scan_jobs.repository_id AND o.group_key = scan_jobs.group_key
|
|
627
|
+
AND o.head_sha = scan_jobs.head_sha AND o.check_name = scan_jobs.check_name
|
|
628
|
+
AND o.requested_by IS NULL
|
|
629
|
+
ORDER BY o.sequence LIMIT 1), job_key)
|
|
630
|
+
WHERE requested_by IS NOT NULL;
|
|
631
|
+
UPDATE scan_jobs SET execution = (
|
|
632
|
+
SELECT COUNT(*) FROM scan_jobs o
|
|
633
|
+
WHERE o.installation_id = scan_jobs.installation_id
|
|
634
|
+
AND o.repository_id = scan_jobs.repository_id AND o.scan_key = scan_jobs.scan_key
|
|
635
|
+
AND o.sequence <= scan_jobs.sequence);
|
|
636
|
+
CREATE INDEX scan_jobs_scan_key ON scan_jobs (installation_id, repository_id, scan_key, sequence);
|
|
637
|
+
CREATE INDEX scan_jobs_head ON scan_jobs (installation_id, repository_id, head_sha);
|
|
638
|
+
CREATE UNIQUE INDEX scan_jobs_rerun_event
|
|
639
|
+
ON scan_jobs (installation_id, repository_id, scan_key, delivery_id)
|
|
640
|
+
WHERE delivery_id IS NOT NULL AND trigger_kind = 'rerun';
|
|
641
|
+
|
|
642
|
+
ALTER TABLE deliveries ADD COLUMN provider TEXT NOT NULL DEFAULT 'github';
|
|
643
|
+
ALTER TABLE deliveries ADD COLUMN action TEXT;
|
|
644
|
+
ALTER TABLE deliveries ADD COLUMN status TEXT NOT NULL DEFAULT 'processed';
|
|
645
|
+
ALTER TABLE deliveries ADD COLUMN installation_id INTEGER;
|
|
646
|
+
ALTER TABLE deliveries ADD COLUMN repository_id INTEGER;
|
|
647
|
+
ALTER TABLE deliveries ADD COLUMN attempts INTEGER NOT NULL DEFAULT 1;
|
|
648
|
+
ALTER TABLE deliveries ADD COLUMN updated_at REAL;
|
|
649
|
+
ALTER TABLE deliveries ADD COLUMN detail TEXT;
|
|
650
|
+
UPDATE deliveries SET updated_at = received_at;
|
|
651
|
+
CREATE UNIQUE INDEX deliveries_provider_event ON deliveries (provider, delivery_id);
|
|
652
|
+
CREATE INDEX deliveries_status ON deliveries (status, updated_at);
|
|
653
|
+
|
|
654
|
+
CREATE TABLE merge_groups (
|
|
655
|
+
installation_id INTEGER NOT NULL,
|
|
656
|
+
repository_id INTEGER NOT NULL,
|
|
657
|
+
head_sha TEXT NOT NULL,
|
|
658
|
+
head_ref TEXT NOT NULL,
|
|
659
|
+
base_sha TEXT NOT NULL,
|
|
660
|
+
base_ref TEXT NOT NULL,
|
|
661
|
+
pull_requests TEXT NOT NULL,
|
|
662
|
+
state TEXT NOT NULL,
|
|
663
|
+
destroyed_reason TEXT,
|
|
664
|
+
job_id TEXT,
|
|
665
|
+
delivery_id TEXT,
|
|
666
|
+
created_at REAL NOT NULL,
|
|
667
|
+
updated_at REAL NOT NULL,
|
|
668
|
+
destroyed_at REAL,
|
|
669
|
+
PRIMARY KEY (installation_id, repository_id, head_sha)
|
|
670
|
+
);
|
|
671
|
+
CREATE INDEX merge_groups_recent ON merge_groups (installation_id, repository_id, created_at);
|
|
672
|
+
ALTER TABLE enforcement_status ADD COLUMN merge_queue TEXT;
|
|
673
|
+
ALTER TABLE enforcement_status ADD COLUMN merge_queue_detail TEXT;
|
|
674
|
+
|
|
675
|
+
ALTER TABLE organization_policy_versions ADD COLUMN kind TEXT NOT NULL DEFAULT 'change';
|
|
676
|
+
ALTER TABLE organization_policy_versions ADD COLUMN rollback_of INTEGER;
|
|
677
|
+
ALTER TABLE organization_policy_versions ADD COLUMN restored_version INTEGER;
|
|
678
|
+
CREATE TRIGGER organization_policy_versions_immutable
|
|
679
|
+
BEFORE UPDATE ON organization_policy_versions
|
|
680
|
+
BEGIN SELECT RAISE(ABORT, 'policy versions are immutable'); END;
|
|
681
|
+
CREATE TRIGGER organization_policy_versions_permanent
|
|
682
|
+
BEFORE DELETE ON organization_policy_versions
|
|
683
|
+
BEGIN SELECT RAISE(ABORT, 'policy versions are immutable'); END;
|
|
684
|
+
|
|
685
|
+
CREATE TABLE notification_events (
|
|
686
|
+
event_id TEXT PRIMARY KEY,
|
|
687
|
+
account_id INTEGER NOT NULL,
|
|
688
|
+
type TEXT NOT NULL,
|
|
689
|
+
severity TEXT NOT NULL,
|
|
690
|
+
installation_id INTEGER,
|
|
691
|
+
repository_id INTEGER,
|
|
692
|
+
resource_type TEXT NOT NULL,
|
|
693
|
+
resource_id TEXT NOT NULL,
|
|
694
|
+
dedup_key TEXT NOT NULL,
|
|
695
|
+
title TEXT NOT NULL,
|
|
696
|
+
body TEXT NOT NULL,
|
|
697
|
+
metadata TEXT NOT NULL,
|
|
698
|
+
occurrences INTEGER NOT NULL DEFAULT 1,
|
|
699
|
+
created_at REAL NOT NULL,
|
|
700
|
+
last_occurred_at REAL NOT NULL,
|
|
701
|
+
dispatched_at REAL,
|
|
702
|
+
request_id TEXT,
|
|
703
|
+
delivery_id TEXT,
|
|
704
|
+
job_id TEXT,
|
|
705
|
+
UNIQUE (account_id, dedup_key)
|
|
706
|
+
);
|
|
707
|
+
CREATE INDEX notification_events_outbox ON notification_events (dispatched_at, created_at);
|
|
708
|
+
CREATE INDEX notification_events_created ON notification_events (created_at);
|
|
709
|
+
|
|
710
|
+
CREATE TABLE notifications (
|
|
711
|
+
notification_id TEXT PRIMARY KEY,
|
|
712
|
+
event_id TEXT NOT NULL REFERENCES notification_events (event_id) ON DELETE CASCADE,
|
|
713
|
+
account_id INTEGER NOT NULL,
|
|
714
|
+
user_id INTEGER NOT NULL,
|
|
715
|
+
state TEXT NOT NULL,
|
|
716
|
+
severity TEXT NOT NULL,
|
|
717
|
+
created_at REAL NOT NULL,
|
|
718
|
+
updated_at REAL NOT NULL,
|
|
719
|
+
sort_at REAL NOT NULL,
|
|
720
|
+
read_at REAL,
|
|
721
|
+
archived_at REAL,
|
|
722
|
+
UNIQUE (event_id, user_id)
|
|
723
|
+
);
|
|
724
|
+
CREATE INDEX notifications_user ON notifications (user_id, sort_at);
|
|
725
|
+
CREATE INDEX notifications_user_state ON notifications (user_id, state, sort_at);
|
|
726
|
+
CREATE INDEX notifications_event ON notifications (event_id);
|
|
727
|
+
CREATE INDEX notifications_user_severity ON notifications (user_id, severity, state);
|
|
728
|
+
|
|
729
|
+
CREATE TABLE notification_deliveries (
|
|
730
|
+
delivery_id TEXT PRIMARY KEY,
|
|
731
|
+
event_id TEXT NOT NULL REFERENCES notification_events (event_id) ON DELETE CASCADE,
|
|
732
|
+
account_id INTEGER NOT NULL,
|
|
733
|
+
channel TEXT NOT NULL,
|
|
734
|
+
destination TEXT NOT NULL,
|
|
735
|
+
idempotency_key TEXT NOT NULL UNIQUE,
|
|
736
|
+
status TEXT NOT NULL,
|
|
737
|
+
attempt_count INTEGER NOT NULL DEFAULT 0,
|
|
738
|
+
provider TEXT NOT NULL,
|
|
739
|
+
provider_message_id TEXT,
|
|
740
|
+
failure_code TEXT,
|
|
741
|
+
last_attempt_at REAL,
|
|
742
|
+
next_retry_at REAL,
|
|
743
|
+
lease_expires_at REAL,
|
|
744
|
+
created_at REAL NOT NULL,
|
|
745
|
+
updated_at REAL NOT NULL
|
|
746
|
+
);
|
|
747
|
+
CREATE INDEX notification_deliveries_due ON notification_deliveries (status, next_retry_at);
|
|
748
|
+
CREATE INDEX notification_deliveries_account ON notification_deliveries (account_id, created_at);
|
|
749
|
+
|
|
750
|
+
CREATE TABLE notification_settings (
|
|
751
|
+
account_id INTEGER PRIMARY KEY,
|
|
752
|
+
version INTEGER NOT NULL,
|
|
753
|
+
document TEXT NOT NULL,
|
|
754
|
+
email_recipients TEXT NOT NULL,
|
|
755
|
+
updated_at REAL NOT NULL,
|
|
756
|
+
updated_by_login TEXT
|
|
757
|
+
);
|
|
758
|
+
CREATE TABLE notification_user_preferences (
|
|
759
|
+
user_id INTEGER NOT NULL,
|
|
760
|
+
account_id INTEGER NOT NULL,
|
|
761
|
+
type TEXT NOT NULL,
|
|
762
|
+
in_app INTEGER NOT NULL,
|
|
763
|
+
updated_at REAL NOT NULL,
|
|
764
|
+
PRIMARY KEY (user_id, account_id, type)
|
|
765
|
+
);
|
|
766
|
+
CREATE TABLE notification_webhooks (
|
|
767
|
+
endpoint_id TEXT PRIMARY KEY,
|
|
768
|
+
account_id INTEGER NOT NULL,
|
|
769
|
+
url TEXT NOT NULL,
|
|
770
|
+
created_at REAL NOT NULL,
|
|
771
|
+
created_by_login TEXT,
|
|
772
|
+
removed_at REAL
|
|
773
|
+
);
|
|
774
|
+
CREATE INDEX notification_webhooks_account ON notification_webhooks (account_id, removed_at);
|
|
775
|
+
"""
|
|
776
|
+
|
|
777
|
+
# Version 4: organization governance (Phase 8). Governance rows are keyed by the
|
|
778
|
+
# account (tenant) and GitHub's repository ID, which survives a reinstallation.
|
|
779
|
+
_SCHEMA_V4 = """
|
|
780
|
+
ALTER TABLE known_repositories ADD COLUMN private INTEGER;
|
|
781
|
+
ALTER TABLE known_repositories ADD COLUMN archived INTEGER NOT NULL DEFAULT 0;
|
|
782
|
+
ALTER TABLE scan_jobs ADD COLUMN governance TEXT;
|
|
783
|
+
ALTER TABLE scan_jobs ADD COLUMN governance_fingerprint TEXT;
|
|
784
|
+
ALTER TABLE scan_jobs ADD COLUMN repository_policies TEXT;
|
|
785
|
+
ALTER TABLE scan_jobs ADD COLUMN schedule_id TEXT;
|
|
786
|
+
CREATE INDEX scan_jobs_repository_completed ON scan_jobs (repository_id, completed_at);
|
|
787
|
+
CREATE TRIGGER audit_events_immutable
|
|
788
|
+
BEFORE UPDATE ON audit_events
|
|
789
|
+
BEGIN SELECT RAISE(ABORT, 'audit events are immutable'); END;
|
|
790
|
+
|
|
791
|
+
CREATE TABLE organization_settings (
|
|
792
|
+
account_id INTEGER PRIMARY KEY,
|
|
793
|
+
version INTEGER NOT NULL,
|
|
794
|
+
document TEXT NOT NULL,
|
|
795
|
+
updated_at REAL NOT NULL,
|
|
796
|
+
updated_by_id INTEGER,
|
|
797
|
+
updated_by_login TEXT
|
|
798
|
+
);
|
|
799
|
+
|
|
800
|
+
CREATE TABLE repository_governance (
|
|
801
|
+
account_id INTEGER NOT NULL,
|
|
802
|
+
repository_id INTEGER NOT NULL,
|
|
803
|
+
onboarding TEXT NOT NULL CHECK (onboarding IN ('discovered', 'onboarded', 'excluded')),
|
|
804
|
+
mode TEXT NOT NULL CHECK (mode IN ('monitor', 'enforce')),
|
|
805
|
+
discovered_at REAL NOT NULL,
|
|
806
|
+
onboarded_at REAL,
|
|
807
|
+
onboarded_by TEXT,
|
|
808
|
+
mode_changed_at REAL,
|
|
809
|
+
mode_changed_by TEXT,
|
|
810
|
+
updated_at REAL NOT NULL,
|
|
811
|
+
PRIMARY KEY (account_id, repository_id)
|
|
812
|
+
);
|
|
813
|
+
CREATE INDEX repository_governance_state ON repository_governance (account_id, onboarding, mode);
|
|
814
|
+
INSERT OR IGNORE INTO repository_governance
|
|
815
|
+
(account_id, repository_id, onboarding, mode, discovered_at, onboarded_at, onboarded_by,
|
|
816
|
+
updated_at)
|
|
817
|
+
SELECT i.account_id, k.repository_id, 'onboarded', 'enforce', MIN(k.first_seen_at),
|
|
818
|
+
MIN(k.first_seen_at), 'migration', MIN(k.first_seen_at)
|
|
819
|
+
FROM known_repositories k JOIN installations i ON i.installation_id = k.installation_id
|
|
820
|
+
GROUP BY i.account_id, k.repository_id;
|
|
821
|
+
|
|
822
|
+
CREATE TABLE repository_groups (
|
|
823
|
+
group_id TEXT PRIMARY KEY,
|
|
824
|
+
account_id INTEGER NOT NULL,
|
|
825
|
+
name TEXT NOT NULL,
|
|
826
|
+
name_key TEXT NOT NULL,
|
|
827
|
+
description TEXT,
|
|
828
|
+
created_at REAL NOT NULL,
|
|
829
|
+
created_by TEXT,
|
|
830
|
+
updated_at REAL NOT NULL,
|
|
831
|
+
archived_at REAL,
|
|
832
|
+
archived_by TEXT
|
|
833
|
+
);
|
|
834
|
+
CREATE UNIQUE INDEX repository_groups_name
|
|
835
|
+
ON repository_groups (account_id, name_key) WHERE archived_at IS NULL;
|
|
836
|
+
CREATE UNIQUE INDEX repository_groups_tenant ON repository_groups (group_id, account_id);
|
|
837
|
+
CREATE TABLE repository_group_members (
|
|
838
|
+
group_id TEXT NOT NULL,
|
|
839
|
+
account_id INTEGER NOT NULL,
|
|
840
|
+
repository_id INTEGER NOT NULL,
|
|
841
|
+
added_at REAL NOT NULL,
|
|
842
|
+
added_by TEXT,
|
|
843
|
+
PRIMARY KEY (group_id, repository_id),
|
|
844
|
+
FOREIGN KEY (group_id, account_id) REFERENCES repository_groups (group_id, account_id)
|
|
845
|
+
);
|
|
846
|
+
CREATE INDEX repository_group_members_repository
|
|
847
|
+
ON repository_group_members (account_id, repository_id);
|
|
848
|
+
|
|
849
|
+
ALTER TABLE organization_policy_versions ADD COLUMN draft_id TEXT;
|
|
850
|
+
ALTER TABLE organization_policy_versions ADD COLUMN emergency INTEGER NOT NULL DEFAULT 0;
|
|
851
|
+
CREATE TABLE scoped_policy_versions (
|
|
852
|
+
account_id INTEGER NOT NULL,
|
|
853
|
+
target_type TEXT NOT NULL CHECK (target_type IN ('group', 'repository')),
|
|
854
|
+
target_id TEXT NOT NULL,
|
|
855
|
+
version INTEGER NOT NULL,
|
|
856
|
+
document TEXT NOT NULL,
|
|
857
|
+
fingerprint TEXT NOT NULL,
|
|
858
|
+
created_at REAL NOT NULL,
|
|
859
|
+
created_by_id INTEGER,
|
|
860
|
+
created_by_login TEXT,
|
|
861
|
+
reason TEXT,
|
|
862
|
+
kind TEXT NOT NULL DEFAULT 'change',
|
|
863
|
+
rollback_of INTEGER,
|
|
864
|
+
restored_version INTEGER,
|
|
865
|
+
draft_id TEXT,
|
|
866
|
+
emergency INTEGER NOT NULL DEFAULT 0,
|
|
867
|
+
PRIMARY KEY (account_id, target_type, target_id, version)
|
|
868
|
+
);
|
|
869
|
+
CREATE TRIGGER scoped_policy_versions_immutable
|
|
870
|
+
BEFORE UPDATE ON scoped_policy_versions
|
|
871
|
+
BEGIN SELECT RAISE(ABORT, 'policy versions are immutable'); END;
|
|
872
|
+
CREATE TRIGGER scoped_policy_versions_permanent
|
|
873
|
+
BEFORE DELETE ON scoped_policy_versions
|
|
874
|
+
BEGIN SELECT RAISE(ABORT, 'policy versions are immutable'); END;
|
|
875
|
+
|
|
876
|
+
CREATE TABLE policy_drafts (
|
|
877
|
+
draft_id TEXT PRIMARY KEY,
|
|
878
|
+
account_id INTEGER NOT NULL,
|
|
879
|
+
target_type TEXT NOT NULL CHECK (target_type IN ('organization', 'group', 'repository')),
|
|
880
|
+
target_id TEXT NOT NULL,
|
|
881
|
+
base_version INTEGER NOT NULL,
|
|
882
|
+
document TEXT NOT NULL,
|
|
883
|
+
fingerprint TEXT NOT NULL,
|
|
884
|
+
title TEXT NOT NULL,
|
|
885
|
+
reason TEXT,
|
|
886
|
+
state TEXT NOT NULL,
|
|
887
|
+
revision INTEGER NOT NULL DEFAULT 1,
|
|
888
|
+
created_at REAL NOT NULL,
|
|
889
|
+
created_by_id INTEGER,
|
|
890
|
+
created_by_login TEXT,
|
|
891
|
+
updated_at REAL NOT NULL,
|
|
892
|
+
submitted_at REAL,
|
|
893
|
+
submitted_by_id INTEGER,
|
|
894
|
+
submitted_by_login TEXT,
|
|
895
|
+
published_version INTEGER,
|
|
896
|
+
published_at REAL,
|
|
897
|
+
published_by_login TEXT,
|
|
898
|
+
emergency INTEGER NOT NULL DEFAULT 0,
|
|
899
|
+
rollout_id TEXT
|
|
900
|
+
);
|
|
901
|
+
CREATE INDEX policy_drafts_account ON policy_drafts (account_id, state, updated_at);
|
|
902
|
+
CREATE TABLE policy_approvals (
|
|
903
|
+
approval_id TEXT PRIMARY KEY,
|
|
904
|
+
draft_id TEXT NOT NULL REFERENCES policy_drafts (draft_id),
|
|
905
|
+
account_id INTEGER NOT NULL,
|
|
906
|
+
fingerprint TEXT NOT NULL,
|
|
907
|
+
requested_by_id INTEGER,
|
|
908
|
+
requested_by_login TEXT,
|
|
909
|
+
requested_at REAL NOT NULL,
|
|
910
|
+
status TEXT NOT NULL CHECK (status IN ('pending', 'approved', 'rejected', 'cancelled')),
|
|
911
|
+
decided_by_id INTEGER,
|
|
912
|
+
decided_by_login TEXT,
|
|
913
|
+
decided_at REAL,
|
|
914
|
+
reason TEXT
|
|
915
|
+
);
|
|
916
|
+
CREATE UNIQUE INDEX policy_approvals_pending
|
|
917
|
+
ON policy_approvals (draft_id) WHERE status = 'pending';
|
|
918
|
+
CREATE INDEX policy_approvals_account ON policy_approvals (account_id, status, requested_at);
|
|
919
|
+
|
|
920
|
+
CREATE TABLE policy_exceptions (
|
|
921
|
+
exception_id TEXT PRIMARY KEY,
|
|
922
|
+
account_id INTEGER NOT NULL,
|
|
923
|
+
rule_id TEXT NOT NULL,
|
|
924
|
+
scope_type TEXT NOT NULL CHECK (scope_type IN ('organization', 'group', 'repository')),
|
|
925
|
+
scope_id TEXT NOT NULL,
|
|
926
|
+
action TEXT NOT NULL CHECK (action IN ('warn', 'allow')),
|
|
927
|
+
severity TEXT NOT NULL,
|
|
928
|
+
reason TEXT NOT NULL,
|
|
929
|
+
status TEXT NOT NULL
|
|
930
|
+
CHECK (status IN ('requested', 'active', 'rejected', 'cancelled', 'revoked', 'expired')),
|
|
931
|
+
requires_approval INTEGER NOT NULL,
|
|
932
|
+
permanent INTEGER NOT NULL DEFAULT 0,
|
|
933
|
+
expires_at REAL,
|
|
934
|
+
requested_at REAL NOT NULL,
|
|
935
|
+
requested_by_id INTEGER,
|
|
936
|
+
requested_by_login TEXT,
|
|
937
|
+
decided_at REAL,
|
|
938
|
+
decided_by_id INTEGER,
|
|
939
|
+
decided_by_login TEXT,
|
|
940
|
+
decision_note TEXT,
|
|
941
|
+
activated_at REAL,
|
|
942
|
+
revoked_at REAL,
|
|
943
|
+
revoked_by_login TEXT,
|
|
944
|
+
revoke_reason TEXT,
|
|
945
|
+
expired_at REAL,
|
|
946
|
+
warnings_sent TEXT NOT NULL DEFAULT '[]',
|
|
947
|
+
updated_at REAL NOT NULL,
|
|
948
|
+
CHECK (permanent = 1 OR expires_at IS NOT NULL)
|
|
949
|
+
);
|
|
950
|
+
CREATE INDEX policy_exceptions_status ON policy_exceptions (account_id, status, expires_at);
|
|
951
|
+
CREATE INDEX policy_exceptions_due ON policy_exceptions (status, expires_at);
|
|
952
|
+
CREATE UNIQUE INDEX policy_exceptions_open
|
|
953
|
+
ON policy_exceptions (account_id, rule_id, scope_type, scope_id)
|
|
954
|
+
WHERE status IN ('requested', 'active');
|
|
955
|
+
CREATE TRIGGER policy_exceptions_permanent
|
|
956
|
+
BEFORE DELETE ON policy_exceptions
|
|
957
|
+
BEGIN SELECT RAISE(ABORT, 'policy exceptions are kept as history'); END;
|
|
958
|
+
|
|
959
|
+
CREATE TABLE policy_rollouts (
|
|
960
|
+
rollout_id TEXT PRIMARY KEY,
|
|
961
|
+
account_id INTEGER NOT NULL,
|
|
962
|
+
target_type TEXT NOT NULL CHECK (target_type IN ('organization', 'group')),
|
|
963
|
+
target_id TEXT NOT NULL,
|
|
964
|
+
from_version INTEGER NOT NULL,
|
|
965
|
+
to_version INTEGER NOT NULL,
|
|
966
|
+
state TEXT NOT NULL CHECK (state IN ('pilot', 'rollout', 'paused', 'active', 'rolled_back')),
|
|
967
|
+
stages TEXT NOT NULL,
|
|
968
|
+
current_stage INTEGER NOT NULL,
|
|
969
|
+
thresholds TEXT NOT NULL,
|
|
970
|
+
auto_pause INTEGER NOT NULL,
|
|
971
|
+
auto_rollback INTEGER NOT NULL,
|
|
972
|
+
created_at REAL NOT NULL,
|
|
973
|
+
created_by_id INTEGER,
|
|
974
|
+
created_by_login TEXT,
|
|
975
|
+
updated_at REAL NOT NULL,
|
|
976
|
+
stage_started_at REAL NOT NULL,
|
|
977
|
+
paused_at REAL,
|
|
978
|
+
paused_reason TEXT,
|
|
979
|
+
paused_from TEXT,
|
|
980
|
+
completed_at REAL,
|
|
981
|
+
rolled_back_at REAL,
|
|
982
|
+
rollback_version INTEGER
|
|
983
|
+
);
|
|
984
|
+
CREATE UNIQUE INDEX policy_rollouts_in_progress
|
|
985
|
+
ON policy_rollouts (account_id, target_type, target_id)
|
|
986
|
+
WHERE state IN ('pilot', 'rollout', 'paused');
|
|
987
|
+
CREATE UNIQUE INDEX policy_rollouts_version
|
|
988
|
+
ON policy_rollouts (account_id, target_type, target_id, to_version);
|
|
989
|
+
CREATE TABLE policy_rollout_repositories (
|
|
990
|
+
rollout_id TEXT NOT NULL REFERENCES policy_rollouts (rollout_id),
|
|
991
|
+
repository_id INTEGER NOT NULL,
|
|
992
|
+
stage INTEGER NOT NULL,
|
|
993
|
+
enrolled_at REAL NOT NULL,
|
|
994
|
+
PRIMARY KEY (rollout_id, repository_id)
|
|
995
|
+
);
|
|
996
|
+
|
|
997
|
+
CREATE TABLE policy_simulations (
|
|
998
|
+
simulation_id TEXT PRIMARY KEY,
|
|
999
|
+
account_id INTEGER NOT NULL,
|
|
1000
|
+
draft_id TEXT,
|
|
1001
|
+
target_type TEXT NOT NULL,
|
|
1002
|
+
target_id TEXT NOT NULL,
|
|
1003
|
+
current_version INTEGER NOT NULL,
|
|
1004
|
+
document TEXT NOT NULL,
|
|
1005
|
+
parameters TEXT NOT NULL,
|
|
1006
|
+
state TEXT NOT NULL CHECK (state IN ('queued', 'running', 'completed', 'failed')),
|
|
1007
|
+
requested_by_id INTEGER,
|
|
1008
|
+
requested_by_login TEXT,
|
|
1009
|
+
requested_at REAL NOT NULL,
|
|
1010
|
+
started_at REAL,
|
|
1011
|
+
completed_at REAL,
|
|
1012
|
+
lease_expires_at REAL,
|
|
1013
|
+
attempts INTEGER NOT NULL DEFAULT 0,
|
|
1014
|
+
result TEXT,
|
|
1015
|
+
error TEXT
|
|
1016
|
+
);
|
|
1017
|
+
CREATE INDEX policy_simulations_account ON policy_simulations (account_id, requested_at);
|
|
1018
|
+
CREATE INDEX policy_simulations_state ON policy_simulations (state, requested_at);
|
|
1019
|
+
|
|
1020
|
+
CREATE TABLE bulk_operations (
|
|
1021
|
+
operation_id TEXT PRIMARY KEY,
|
|
1022
|
+
account_id INTEGER NOT NULL,
|
|
1023
|
+
type TEXT NOT NULL,
|
|
1024
|
+
parameters TEXT NOT NULL,
|
|
1025
|
+
idempotency_key TEXT NOT NULL,
|
|
1026
|
+
requested_by_id INTEGER,
|
|
1027
|
+
requested_by_login TEXT,
|
|
1028
|
+
created_at REAL NOT NULL,
|
|
1029
|
+
status TEXT NOT NULL CHECK (status IN
|
|
1030
|
+
('queued', 'running', 'completed', 'partial', 'failed', 'cancelled')),
|
|
1031
|
+
total INTEGER NOT NULL,
|
|
1032
|
+
completed INTEGER NOT NULL DEFAULT 0,
|
|
1033
|
+
failed INTEGER NOT NULL DEFAULT 0,
|
|
1034
|
+
started_at REAL,
|
|
1035
|
+
completed_at REAL,
|
|
1036
|
+
updated_at REAL NOT NULL,
|
|
1037
|
+
lease_expires_at REAL,
|
|
1038
|
+
cancelled_by_login TEXT
|
|
1039
|
+
);
|
|
1040
|
+
CREATE UNIQUE INDEX bulk_operations_idempotency ON bulk_operations (account_id, idempotency_key);
|
|
1041
|
+
CREATE INDEX bulk_operations_status ON bulk_operations (status, created_at);
|
|
1042
|
+
CREATE INDEX bulk_operations_account ON bulk_operations (account_id, created_at);
|
|
1043
|
+
CREATE TABLE bulk_operation_items (
|
|
1044
|
+
operation_id TEXT NOT NULL REFERENCES bulk_operations (operation_id),
|
|
1045
|
+
repository_id INTEGER NOT NULL,
|
|
1046
|
+
status TEXT NOT NULL CHECK (status IN
|
|
1047
|
+
('pending', 'completed', 'failed', 'skipped', 'cancelled')),
|
|
1048
|
+
attempts INTEGER NOT NULL DEFAULT 0,
|
|
1049
|
+
detail TEXT,
|
|
1050
|
+
updated_at REAL NOT NULL,
|
|
1051
|
+
PRIMARY KEY (operation_id, repository_id)
|
|
1052
|
+
);
|
|
1053
|
+
CREATE INDEX bulk_operation_items_status ON bulk_operation_items (operation_id, status);
|
|
1054
|
+
|
|
1055
|
+
CREATE TABLE scan_schedules (
|
|
1056
|
+
schedule_id TEXT PRIMARY KEY,
|
|
1057
|
+
account_id INTEGER NOT NULL,
|
|
1058
|
+
target_type TEXT NOT NULL CHECK (target_type IN ('organization', 'group', 'repository')),
|
|
1059
|
+
target_id TEXT NOT NULL,
|
|
1060
|
+
name TEXT NOT NULL,
|
|
1061
|
+
cadence TEXT NOT NULL CHECK (cadence IN ('daily', 'weekly')),
|
|
1062
|
+
hour INTEGER NOT NULL,
|
|
1063
|
+
minute INTEGER NOT NULL,
|
|
1064
|
+
weekday INTEGER,
|
|
1065
|
+
timezone TEXT NOT NULL,
|
|
1066
|
+
enabled INTEGER NOT NULL,
|
|
1067
|
+
next_run_at REAL,
|
|
1068
|
+
revision INTEGER NOT NULL DEFAULT 1,
|
|
1069
|
+
created_at REAL NOT NULL,
|
|
1070
|
+
created_by_login TEXT,
|
|
1071
|
+
updated_at REAL NOT NULL,
|
|
1072
|
+
updated_by_login TEXT
|
|
1073
|
+
);
|
|
1074
|
+
CREATE INDEX scan_schedules_due ON scan_schedules (enabled, next_run_at);
|
|
1075
|
+
CREATE INDEX scan_schedules_account ON scan_schedules (account_id, created_at);
|
|
1076
|
+
CREATE TABLE scan_schedule_runs (
|
|
1077
|
+
run_id TEXT PRIMARY KEY,
|
|
1078
|
+
schedule_id TEXT NOT NULL REFERENCES scan_schedules (schedule_id),
|
|
1079
|
+
account_id INTEGER NOT NULL,
|
|
1080
|
+
slot REAL NOT NULL,
|
|
1081
|
+
state TEXT NOT NULL CHECK (state IN ('running', 'completed', 'partial', 'failed')),
|
|
1082
|
+
started_at REAL NOT NULL,
|
|
1083
|
+
completed_at REAL,
|
|
1084
|
+
repositories INTEGER NOT NULL DEFAULT 0,
|
|
1085
|
+
queued INTEGER NOT NULL DEFAULT 0,
|
|
1086
|
+
skipped INTEGER NOT NULL DEFAULT 0,
|
|
1087
|
+
failed INTEGER NOT NULL DEFAULT 0,
|
|
1088
|
+
cursor INTEGER NOT NULL DEFAULT 0,
|
|
1089
|
+
detail TEXT,
|
|
1090
|
+
UNIQUE (schedule_id, slot)
|
|
1091
|
+
);
|
|
1092
|
+
CREATE INDEX scan_schedule_runs_state ON scan_schedule_runs (state, started_at);
|
|
1093
|
+
|
|
1094
|
+
CREATE TABLE repository_effective_policies (
|
|
1095
|
+
account_id INTEGER NOT NULL,
|
|
1096
|
+
repository_id INTEGER NOT NULL,
|
|
1097
|
+
state TEXT NOT NULL CHECK (state IN ('up_to_date', 'stale', 'syncing', 'error')),
|
|
1098
|
+
fingerprint TEXT,
|
|
1099
|
+
document TEXT,
|
|
1100
|
+
computed_at REAL,
|
|
1101
|
+
invalidated_at REAL,
|
|
1102
|
+
valid_until REAL,
|
|
1103
|
+
attempts INTEGER NOT NULL DEFAULT 0,
|
|
1104
|
+
error TEXT,
|
|
1105
|
+
PRIMARY KEY (account_id, repository_id)
|
|
1106
|
+
);
|
|
1107
|
+
CREATE INDEX repository_effective_policies_state
|
|
1108
|
+
ON repository_effective_policies (state, invalidated_at);
|
|
1109
|
+
|
|
1110
|
+
CREATE TABLE organization_rule_versions (
|
|
1111
|
+
account_id INTEGER NOT NULL,
|
|
1112
|
+
version INTEGER NOT NULL,
|
|
1113
|
+
document TEXT NOT NULL,
|
|
1114
|
+
fingerprint TEXT NOT NULL,
|
|
1115
|
+
created_at REAL NOT NULL,
|
|
1116
|
+
created_by_id INTEGER,
|
|
1117
|
+
created_by_login TEXT,
|
|
1118
|
+
reason TEXT,
|
|
1119
|
+
PRIMARY KEY (account_id, version)
|
|
1120
|
+
);
|
|
1121
|
+
CREATE TRIGGER organization_rule_versions_immutable
|
|
1122
|
+
BEFORE UPDATE ON organization_rule_versions
|
|
1123
|
+
BEGIN SELECT RAISE(ABORT, 'rule versions are immutable'); END;
|
|
1124
|
+
CREATE TRIGGER organization_rule_versions_permanent
|
|
1125
|
+
BEFORE DELETE ON organization_rule_versions
|
|
1126
|
+
BEGIN SELECT RAISE(ABORT, 'rule versions are immutable'); END;
|
|
1127
|
+
|
|
1128
|
+
CREATE TABLE installation_sync_status (
|
|
1129
|
+
installation_id INTEGER PRIMARY KEY,
|
|
1130
|
+
account_id INTEGER NOT NULL,
|
|
1131
|
+
state TEXT NOT NULL CHECK (state IN ('healthy', 'syncing', 'degraded', 'failed')),
|
|
1132
|
+
started_at REAL,
|
|
1133
|
+
completed_at REAL,
|
|
1134
|
+
last_success_at REAL,
|
|
1135
|
+
repositories INTEGER,
|
|
1136
|
+
added INTEGER,
|
|
1137
|
+
removed INTEGER,
|
|
1138
|
+
error TEXT,
|
|
1139
|
+
updated_at REAL NOT NULL
|
|
1140
|
+
);
|
|
1141
|
+
|
|
1142
|
+
CREATE TABLE security_metric_snapshots (
|
|
1143
|
+
account_id INTEGER NOT NULL,
|
|
1144
|
+
day TEXT NOT NULL,
|
|
1145
|
+
computed_at REAL NOT NULL,
|
|
1146
|
+
document TEXT NOT NULL,
|
|
1147
|
+
PRIMARY KEY (account_id, day)
|
|
1148
|
+
);
|
|
1149
|
+
|
|
1150
|
+
CREATE TABLE notification_acknowledgements (
|
|
1151
|
+
event_id TEXT PRIMARY KEY REFERENCES notification_events (event_id) ON DELETE CASCADE,
|
|
1152
|
+
account_id INTEGER NOT NULL,
|
|
1153
|
+
acknowledged_by_id INTEGER NOT NULL,
|
|
1154
|
+
acknowledged_by_login TEXT NOT NULL,
|
|
1155
|
+
acknowledged_at REAL NOT NULL,
|
|
1156
|
+
note TEXT
|
|
1157
|
+
);
|
|
1158
|
+
"""
|
|
1159
|
+
|
|
1160
|
+
_MIGRATIONS: tuple[str, ...] = (_SCHEMA_V1, _SCHEMA_V2, _SCHEMA_V3, _SCHEMA_V4)
|
|
1161
|
+
|
|
1162
|
+
|
|
1163
|
+
_ORPHAN_DELETES = (
|
|
1164
|
+
"DELETE FROM violation_exposures WHERE installation_id NOT IN "
|
|
1165
|
+
"(SELECT installation_id FROM installations)",
|
|
1166
|
+
"DELETE FROM violations WHERE installation_id NOT IN "
|
|
1167
|
+
"(SELECT installation_id FROM installations)",
|
|
1168
|
+
"DELETE FROM findings WHERE installation_id NOT IN (SELECT installation_id FROM installations)",
|
|
1169
|
+
"DELETE FROM repository_settings WHERE installation_id NOT IN "
|
|
1170
|
+
"(SELECT installation_id FROM installations)",
|
|
1171
|
+
"DELETE FROM enforcement_status WHERE installation_id NOT IN "
|
|
1172
|
+
"(SELECT installation_id FROM installations)",
|
|
1173
|
+
# The repository inventory of an installation that is gone. `known_repositories`
|
|
1174
|
+
# is otherwise purged only once a repository has been *removed* from a live
|
|
1175
|
+
# installation, so rows for repositories still granted at uninstall time would
|
|
1176
|
+
# keep an organisation's repository names forever.
|
|
1177
|
+
"DELETE FROM installation_repositories WHERE installation_id NOT IN "
|
|
1178
|
+
"(SELECT installation_id FROM installations)",
|
|
1179
|
+
"DELETE FROM known_repositories WHERE installation_id NOT IN "
|
|
1180
|
+
"(SELECT installation_id FROM installations)",
|
|
1181
|
+
"DELETE FROM installation_sync_status WHERE installation_id NOT IN "
|
|
1182
|
+
"(SELECT installation_id FROM installations)",
|
|
1183
|
+
)
|
|
1184
|
+
|
|
1185
|
+
|
|
1186
|
+
def _statements(script: str) -> list[str]:
|
|
1187
|
+
"""Split a migration into statements.
|
|
1188
|
+
|
|
1189
|
+
A statement ends with ``;`` at the end of a line, so a trigger body whose
|
|
1190
|
+
statements end mid-line (``BEGIN SELECT ...; END;``) stays one statement.
|
|
1191
|
+
Migrations contain no string literals with a line-final ``;``.
|
|
1192
|
+
"""
|
|
1193
|
+
return [part.strip() for part in re.split(r";[ \t]*(?:\n|\Z)", script) if part.strip()]
|
|
1194
|
+
|
|
1195
|
+
|
|
1196
|
+
# Static statements only: no SQL is ever assembled from field names at run time.
|
|
1197
|
+
_JOB_UPDATES = {
|
|
1198
|
+
"state": "UPDATE scan_jobs SET state = ?, updated_at = ? WHERE job_id = ?",
|
|
1199
|
+
"lease_expires_at": (
|
|
1200
|
+
"UPDATE scan_jobs SET lease_expires_at = ?, updated_at = ? WHERE job_id = ?"
|
|
1201
|
+
),
|
|
1202
|
+
"scan_id": "UPDATE scan_jobs SET scan_id = ?, updated_at = ? WHERE job_id = ?",
|
|
1203
|
+
"check_run_id": "UPDATE scan_jobs SET check_run_id = ?, updated_at = ? WHERE job_id = ?",
|
|
1204
|
+
"result_action": "UPDATE scan_jobs SET result_action = ?, updated_at = ? WHERE job_id = ?",
|
|
1205
|
+
"conclusion": "UPDATE scan_jobs SET conclusion = ?, updated_at = ? WHERE job_id = ?",
|
|
1206
|
+
"failure_kind": "UPDATE scan_jobs SET failure_kind = ?, updated_at = ? WHERE job_id = ?",
|
|
1207
|
+
"message": "UPDATE scan_jobs SET message = ?, updated_at = ? WHERE job_id = ?",
|
|
1208
|
+
"commits_scanned": "UPDATE scan_jobs SET commits_scanned = ?, updated_at = ? WHERE job_id = ?",
|
|
1209
|
+
"violations": "UPDATE scan_jobs SET violations = ?, updated_at = ? WHERE job_id = ?",
|
|
1210
|
+
"warnings": "UPDATE scan_jobs SET warnings = ?, updated_at = ? WHERE job_id = ?",
|
|
1211
|
+
"base_sha": "UPDATE scan_jobs SET base_sha = ?, updated_at = ? WHERE job_id = ?",
|
|
1212
|
+
"completed_at": "UPDATE scan_jobs SET completed_at = ?, updated_at = ? WHERE job_id = ?",
|
|
1213
|
+
"tool_version": "UPDATE scan_jobs SET tool_version = ?, updated_at = ? WHERE job_id = ?",
|
|
1214
|
+
"rules_version": "UPDATE scan_jobs SET rules_version = ?, updated_at = ? WHERE job_id = ?",
|
|
1215
|
+
"policy_version": "UPDATE scan_jobs SET policy_version = ?, updated_at = ? WHERE job_id = ?",
|
|
1216
|
+
"policy_source": "UPDATE scan_jobs SET policy_source = ?, updated_at = ? WHERE job_id = ?",
|
|
1217
|
+
"organization_policy_version": (
|
|
1218
|
+
"UPDATE scan_jobs SET organization_policy_version = ?, updated_at = ? WHERE job_id = ?"
|
|
1219
|
+
),
|
|
1220
|
+
"effective_policies": (
|
|
1221
|
+
"UPDATE scan_jobs SET effective_policies = ?, updated_at = ? WHERE job_id = ?"
|
|
1222
|
+
),
|
|
1223
|
+
"findings_count": "UPDATE scan_jobs SET findings_count = ?, updated_at = ? WHERE job_id = ?",
|
|
1224
|
+
"detector_failures": (
|
|
1225
|
+
"UPDATE scan_jobs SET detector_failures = ?, updated_at = ? WHERE job_id = ?"
|
|
1226
|
+
),
|
|
1227
|
+
"notices": "UPDATE scan_jobs SET notices = ?, updated_at = ? WHERE job_id = ?",
|
|
1228
|
+
}
|
|
1229
|
+
|
|
1230
|
+
|
|
1231
|
+
class SqliteStateStore(AuditStorage):
|
|
1232
|
+
"""All GitHub App state in one SQLite database."""
|
|
1233
|
+
|
|
1234
|
+
def __init__(self, path: Path | str) -> None:
|
|
1235
|
+
self.path = Path(path) if str(path) != ":memory:" else None
|
|
1236
|
+
if self.path is not None:
|
|
1237
|
+
self.path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
|
|
1238
|
+
if not self.path.exists():
|
|
1239
|
+
fd = os.open(self.path, os.O_CREAT | os.O_WRONLY, 0o600)
|
|
1240
|
+
os.close(fd)
|
|
1241
|
+
self._lock = threading.RLock()
|
|
1242
|
+
try:
|
|
1243
|
+
self._db = sqlite3.connect(
|
|
1244
|
+
str(self.path) if self.path else ":memory:",
|
|
1245
|
+
check_same_thread=False,
|
|
1246
|
+
isolation_level=None,
|
|
1247
|
+
timeout=10.0,
|
|
1248
|
+
)
|
|
1249
|
+
self._db.row_factory = sqlite3.Row
|
|
1250
|
+
self._db.execute("PRAGMA busy_timeout = 10000")
|
|
1251
|
+
if self.path is not None:
|
|
1252
|
+
self._db.execute("PRAGMA journal_mode = WAL")
|
|
1253
|
+
self._db.execute("PRAGMA foreign_keys = ON")
|
|
1254
|
+
self._migrate()
|
|
1255
|
+
except sqlite3.Error as exc:
|
|
1256
|
+
raise InfrastructureError(f"state store unavailable ({type(exc).__name__})") from None
|
|
1257
|
+
|
|
1258
|
+
def _migrate(self) -> None:
|
|
1259
|
+
"""Apply pending migrations in one transaction; refuse unknown newer schemas."""
|
|
1260
|
+
db = self._db
|
|
1261
|
+
db.execute("BEGIN IMMEDIATE")
|
|
1262
|
+
try:
|
|
1263
|
+
db.execute(
|
|
1264
|
+
"CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT NOT NULL)"
|
|
1265
|
+
)
|
|
1266
|
+
row = db.execute("SELECT value FROM meta WHERE key = 'schema_version'").fetchone()
|
|
1267
|
+
current = int(row["value"]) if row is not None and row["value"].isdigit() else 0
|
|
1268
|
+
if row is not None and not row["value"].isdigit():
|
|
1269
|
+
raise InfrastructureError("state store has an unsupported schema version")
|
|
1270
|
+
if current > SCHEMA_VERSION:
|
|
1271
|
+
raise InfrastructureError("state store has an unsupported schema version")
|
|
1272
|
+
for version in range(current + 1, SCHEMA_VERSION + 1):
|
|
1273
|
+
for statement in _statements(_MIGRATIONS[version - 1]):
|
|
1274
|
+
db.execute(statement)
|
|
1275
|
+
db.execute(
|
|
1276
|
+
"INSERT INTO meta (key, value) VALUES ('schema_version', ?) "
|
|
1277
|
+
"ON CONFLICT (key) DO UPDATE SET value = excluded.value",
|
|
1278
|
+
(str(SCHEMA_VERSION),),
|
|
1279
|
+
)
|
|
1280
|
+
except BaseException:
|
|
1281
|
+
db.execute("ROLLBACK")
|
|
1282
|
+
raise
|
|
1283
|
+
db.execute("COMMIT")
|
|
1284
|
+
|
|
1285
|
+
def close(self) -> None:
|
|
1286
|
+
with self._lock:
|
|
1287
|
+
self._db.close()
|
|
1288
|
+
|
|
1289
|
+
@contextmanager
|
|
1290
|
+
def transaction(self) -> Iterator[sqlite3.Connection]:
|
|
1291
|
+
"""A write transaction (``BEGIN IMMEDIATE``) for multi-statement changes."""
|
|
1292
|
+
with self._transaction() as db:
|
|
1293
|
+
yield db
|
|
1294
|
+
|
|
1295
|
+
def query(self, sql: str, params: Sequence[Any] = ()) -> list[sqlite3.Row]:
|
|
1296
|
+
"""Run a read-only, parameterised statement."""
|
|
1297
|
+
return self._query(sql, params)
|
|
1298
|
+
|
|
1299
|
+
@contextmanager
|
|
1300
|
+
def _transaction(self) -> Iterator[sqlite3.Connection]:
|
|
1301
|
+
with self._lock:
|
|
1302
|
+
try:
|
|
1303
|
+
self._db.execute("BEGIN IMMEDIATE")
|
|
1304
|
+
try:
|
|
1305
|
+
yield self._db
|
|
1306
|
+
except BaseException:
|
|
1307
|
+
self._db.execute("ROLLBACK")
|
|
1308
|
+
raise
|
|
1309
|
+
self._db.execute("COMMIT")
|
|
1310
|
+
except sqlite3.Error as exc:
|
|
1311
|
+
raise InfrastructureError(f"state store error ({type(exc).__name__})") from None
|
|
1312
|
+
|
|
1313
|
+
def _query(self, sql: str, params: Sequence[Any] = ()) -> list[sqlite3.Row]:
|
|
1314
|
+
with self._lock:
|
|
1315
|
+
try:
|
|
1316
|
+
return self._db.execute(sql, params).fetchall()
|
|
1317
|
+
except sqlite3.Error as exc:
|
|
1318
|
+
raise InfrastructureError(f"state store error ({type(exc).__name__})") from None
|
|
1319
|
+
|
|
1320
|
+
def ping(self) -> bool:
|
|
1321
|
+
try:
|
|
1322
|
+
self._query("SELECT 1")
|
|
1323
|
+
except InfrastructureError:
|
|
1324
|
+
return False
|
|
1325
|
+
return True
|
|
1326
|
+
|
|
1327
|
+
# -- deliveries ----------------------------------------------------- #
|
|
1328
|
+
def record_delivery(
|
|
1329
|
+
self,
|
|
1330
|
+
delivery_id: str,
|
|
1331
|
+
event: str,
|
|
1332
|
+
body_sha256: str,
|
|
1333
|
+
received_at: datetime,
|
|
1334
|
+
*,
|
|
1335
|
+
action: str | None = None,
|
|
1336
|
+
) -> DeliveryStatus:
|
|
1337
|
+
"""Record a verified delivery (the event record) before it is processed.
|
|
1338
|
+
|
|
1339
|
+
The delivery ID is GitHub's unique event identifier; GitHub reuses it when a
|
|
1340
|
+
delivery is redelivered. A known ID with the same payload is a duplicate,
|
|
1341
|
+
unless its earlier processing failed or was abandoned - then it is processed
|
|
1342
|
+
again (``RETRY``). A known ID with a different payload is never processed.
|
|
1343
|
+
"""
|
|
1344
|
+
now = _ts(received_at)
|
|
1345
|
+
with self._transaction() as db:
|
|
1346
|
+
cursor = db.execute(
|
|
1347
|
+
"INSERT OR IGNORE INTO deliveries (delivery_id, event, body_sha256, received_at, "
|
|
1348
|
+
"provider, action, status, attempts, updated_at) "
|
|
1349
|
+
"VALUES (?, ?, ?, ?, 'github', ?, 'processing', 1, ?)",
|
|
1350
|
+
(delivery_id, event, body_sha256, now, action, now),
|
|
1351
|
+
)
|
|
1352
|
+
if cursor.rowcount == 1:
|
|
1353
|
+
return DeliveryStatus.NEW
|
|
1354
|
+
row = db.execute(
|
|
1355
|
+
"SELECT event, body_sha256, status, updated_at FROM deliveries "
|
|
1356
|
+
"WHERE delivery_id = ?",
|
|
1357
|
+
(delivery_id,),
|
|
1358
|
+
).fetchone()
|
|
1359
|
+
if row is None or row["event"] != event or row["body_sha256"] != body_sha256:
|
|
1360
|
+
return DeliveryStatus.CONFLICT
|
|
1361
|
+
stuck = (
|
|
1362
|
+
row["status"] == EventProcessingStatus.PROCESSING.value
|
|
1363
|
+
and (row["updated_at"] or 0) < now - STUCK_DELIVERY_SECONDS
|
|
1364
|
+
)
|
|
1365
|
+
if row["status"] == EventProcessingStatus.FAILED.value or stuck:
|
|
1366
|
+
db.execute(
|
|
1367
|
+
"UPDATE deliveries SET status = 'processing', attempts = attempts + 1, "
|
|
1368
|
+
"updated_at = ?, detail = NULL WHERE delivery_id = ?",
|
|
1369
|
+
(now, delivery_id),
|
|
1370
|
+
)
|
|
1371
|
+
return DeliveryStatus.RETRY
|
|
1372
|
+
return DeliveryStatus.DUPLICATE
|
|
1373
|
+
|
|
1374
|
+
def finish_delivery(
|
|
1375
|
+
self,
|
|
1376
|
+
delivery_id: str,
|
|
1377
|
+
status: EventProcessingStatus,
|
|
1378
|
+
now: datetime,
|
|
1379
|
+
*,
|
|
1380
|
+
installation_id: int | None = None,
|
|
1381
|
+
repository_id: int | None = None,
|
|
1382
|
+
detail: str | None = None,
|
|
1383
|
+
) -> None:
|
|
1384
|
+
with self._transaction() as db:
|
|
1385
|
+
db.execute(
|
|
1386
|
+
"UPDATE deliveries SET status = ?, updated_at = ?, "
|
|
1387
|
+
"installation_id = COALESCE(?, installation_id), "
|
|
1388
|
+
"repository_id = COALESCE(?, repository_id), detail = ? WHERE delivery_id = ?",
|
|
1389
|
+
(
|
|
1390
|
+
status.value,
|
|
1391
|
+
_ts(now),
|
|
1392
|
+
installation_id,
|
|
1393
|
+
repository_id,
|
|
1394
|
+
(detail or "")[:500] or None,
|
|
1395
|
+
delivery_id,
|
|
1396
|
+
),
|
|
1397
|
+
)
|
|
1398
|
+
|
|
1399
|
+
def delivery_status(self, delivery_id: str) -> EventProcessingStatus | None:
|
|
1400
|
+
rows = self._query("SELECT status FROM deliveries WHERE delivery_id = ?", (delivery_id,))
|
|
1401
|
+
return EventProcessingStatus(rows[0]["status"]) if rows else None
|
|
1402
|
+
|
|
1403
|
+
def fail_stuck_deliveries(self, now: datetime) -> int:
|
|
1404
|
+
"""Mark deliveries abandoned mid-processing as failed, so a redelivery is processed."""
|
|
1405
|
+
with self._transaction() as db:
|
|
1406
|
+
return db.execute(
|
|
1407
|
+
"UPDATE deliveries SET status = 'failed', detail = 'processing abandoned', "
|
|
1408
|
+
"updated_at = ? WHERE status = 'processing' AND updated_at < ?",
|
|
1409
|
+
(_ts(now), _ts(now) - STUCK_DELIVERY_SECONDS),
|
|
1410
|
+
).rowcount
|
|
1411
|
+
|
|
1412
|
+
# -- installations -------------------------------------------------- #
|
|
1413
|
+
def upsert_installation(self, record: InstallationRecord) -> None:
|
|
1414
|
+
with self._transaction() as db:
|
|
1415
|
+
self.upsert_installation_in(db, record)
|
|
1416
|
+
|
|
1417
|
+
@staticmethod
|
|
1418
|
+
def upsert_installation_in(db: sqlite3.Connection, record: InstallationRecord) -> None:
|
|
1419
|
+
db.execute(
|
|
1420
|
+
"INSERT INTO installations (installation_id, account_id, account_login, "
|
|
1421
|
+
"account_type, repository_selection, state, permissions, created_at, updated_at) "
|
|
1422
|
+
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) "
|
|
1423
|
+
"ON CONFLICT (installation_id) DO UPDATE SET account_id = excluded.account_id, "
|
|
1424
|
+
"account_login = excluded.account_login, account_type = excluded.account_type, "
|
|
1425
|
+
"repository_selection = excluded.repository_selection, state = excluded.state, "
|
|
1426
|
+
"permissions = excluded.permissions, updated_at = excluded.updated_at",
|
|
1427
|
+
(
|
|
1428
|
+
record.installation_id,
|
|
1429
|
+
record.account_id,
|
|
1430
|
+
record.account_login,
|
|
1431
|
+
record.account_type.value,
|
|
1432
|
+
record.repository_selection,
|
|
1433
|
+
record.state.value,
|
|
1434
|
+
json.dumps(record.permissions, sort_keys=True),
|
|
1435
|
+
_ts(record.created_at),
|
|
1436
|
+
_ts(record.updated_at),
|
|
1437
|
+
),
|
|
1438
|
+
)
|
|
1439
|
+
|
|
1440
|
+
def get_installation(self, installation_id: int) -> InstallationRecord | None:
|
|
1441
|
+
rows = self._query(
|
|
1442
|
+
"SELECT * FROM installations WHERE installation_id = ?", (int(installation_id),)
|
|
1443
|
+
)
|
|
1444
|
+
if not rows:
|
|
1445
|
+
return None
|
|
1446
|
+
row = rows[0]
|
|
1447
|
+
return InstallationRecord(
|
|
1448
|
+
installation_id=row["installation_id"],
|
|
1449
|
+
account_id=row["account_id"],
|
|
1450
|
+
account_login=row["account_login"],
|
|
1451
|
+
account_type=AccountType(row["account_type"]),
|
|
1452
|
+
repository_selection=row["repository_selection"],
|
|
1453
|
+
state=InstallationState(row["state"]),
|
|
1454
|
+
permissions=json.loads(row["permissions"]),
|
|
1455
|
+
created_at=_dt(row["created_at"]),
|
|
1456
|
+
updated_at=_dt(row["updated_at"]),
|
|
1457
|
+
)
|
|
1458
|
+
|
|
1459
|
+
def set_installation_state(
|
|
1460
|
+
self, installation_id: int, state: InstallationState, now: datetime
|
|
1461
|
+
) -> None:
|
|
1462
|
+
with self._transaction() as db:
|
|
1463
|
+
self.set_installation_state_in(db, installation_id, state, now)
|
|
1464
|
+
|
|
1465
|
+
@staticmethod
|
|
1466
|
+
def set_installation_state_in(
|
|
1467
|
+
db: sqlite3.Connection, installation_id: int, state: InstallationState, now: datetime
|
|
1468
|
+
) -> None:
|
|
1469
|
+
db.execute(
|
|
1470
|
+
"UPDATE installations SET state = ?, updated_at = ? WHERE installation_id = ?",
|
|
1471
|
+
(state.value, _ts(now), int(installation_id)),
|
|
1472
|
+
)
|
|
1473
|
+
if state is InstallationState.DELETED:
|
|
1474
|
+
db.execute(
|
|
1475
|
+
"DELETE FROM installation_repositories WHERE installation_id = ?",
|
|
1476
|
+
(int(installation_id),),
|
|
1477
|
+
)
|
|
1478
|
+
db.execute(
|
|
1479
|
+
"UPDATE known_repositories SET removed_at = ? "
|
|
1480
|
+
"WHERE installation_id = ? AND removed_at IS NULL",
|
|
1481
|
+
(_ts(now), int(installation_id)),
|
|
1482
|
+
)
|
|
1483
|
+
db.execute(
|
|
1484
|
+
"UPDATE scan_jobs SET state = 'cancelled', message = 'installation removed', "
|
|
1485
|
+
"updated_at = ?, completed_at = ? WHERE installation_id = ? "
|
|
1486
|
+
"AND state IN ('queued', 'running')",
|
|
1487
|
+
(_ts(now), _ts(now), int(installation_id)),
|
|
1488
|
+
)
|
|
1489
|
+
|
|
1490
|
+
def replace_repositories(
|
|
1491
|
+
self, installation_id: int, repositories: Sequence[RepositoryRef], now: datetime
|
|
1492
|
+
) -> None:
|
|
1493
|
+
with self._transaction() as db:
|
|
1494
|
+
db.execute(
|
|
1495
|
+
"DELETE FROM installation_repositories WHERE installation_id = ?",
|
|
1496
|
+
(int(installation_id),),
|
|
1497
|
+
)
|
|
1498
|
+
db.execute(
|
|
1499
|
+
"UPDATE known_repositories SET removed_at = ? "
|
|
1500
|
+
"WHERE installation_id = ? AND removed_at IS NULL",
|
|
1501
|
+
(_ts(now), int(installation_id)),
|
|
1502
|
+
)
|
|
1503
|
+
self._insert_repositories(db, installation_id, repositories, now)
|
|
1504
|
+
|
|
1505
|
+
def add_repositories(
|
|
1506
|
+
self, installation_id: int, repositories: Sequence[RepositoryRef], now: datetime
|
|
1507
|
+
) -> None:
|
|
1508
|
+
with self._transaction() as db:
|
|
1509
|
+
self._insert_repositories(db, installation_id, repositories, now)
|
|
1510
|
+
|
|
1511
|
+
@staticmethod
|
|
1512
|
+
def _insert_repositories(
|
|
1513
|
+
db: sqlite3.Connection,
|
|
1514
|
+
installation_id: int,
|
|
1515
|
+
repositories: Sequence[RepositoryRef],
|
|
1516
|
+
now: datetime,
|
|
1517
|
+
) -> None:
|
|
1518
|
+
db.executemany(
|
|
1519
|
+
"INSERT INTO installation_repositories (installation_id, repository_id, owner, name, "
|
|
1520
|
+
"added_at) VALUES (?, ?, ?, ?, ?) ON CONFLICT (installation_id, repository_id) "
|
|
1521
|
+
"DO UPDATE SET owner = excluded.owner, name = excluded.name",
|
|
1522
|
+
[(int(installation_id), r.id, r.owner, r.name, _ts(now)) for r in repositories],
|
|
1523
|
+
)
|
|
1524
|
+
db.executemany(
|
|
1525
|
+
"INSERT INTO known_repositories (installation_id, repository_id, owner, name, "
|
|
1526
|
+
"first_seen_at, last_seen_at) VALUES (?, ?, ?, ?, ?, ?) "
|
|
1527
|
+
"ON CONFLICT (installation_id, repository_id) DO UPDATE SET owner = excluded.owner, "
|
|
1528
|
+
"name = excluded.name, last_seen_at = excluded.last_seen_at, removed_at = NULL",
|
|
1529
|
+
[
|
|
1530
|
+
(int(installation_id), r.id, r.owner, r.name, _ts(now), _ts(now))
|
|
1531
|
+
for r in repositories
|
|
1532
|
+
],
|
|
1533
|
+
)
|
|
1534
|
+
|
|
1535
|
+
def remove_repositories(
|
|
1536
|
+
self, installation_id: int, repository_ids: Sequence[int], now: datetime | None = None
|
|
1537
|
+
) -> None:
|
|
1538
|
+
removed_at = _ts(now or datetime.now(UTC))
|
|
1539
|
+
with self._transaction() as db:
|
|
1540
|
+
db.executemany(
|
|
1541
|
+
"UPDATE known_repositories SET removed_at = ? WHERE installation_id = ? "
|
|
1542
|
+
"AND repository_id = ?",
|
|
1543
|
+
[(removed_at, int(installation_id), int(r)) for r in repository_ids],
|
|
1544
|
+
)
|
|
1545
|
+
db.executemany(
|
|
1546
|
+
"DELETE FROM installation_repositories WHERE installation_id = ? "
|
|
1547
|
+
"AND repository_id = ?",
|
|
1548
|
+
[(int(installation_id), int(r)) for r in repository_ids],
|
|
1549
|
+
)
|
|
1550
|
+
db.executemany(
|
|
1551
|
+
"UPDATE scan_jobs SET state = 'cancelled', message = 'repository removed' "
|
|
1552
|
+
"WHERE installation_id = ? AND repository_id = ? "
|
|
1553
|
+
"AND state IN ('queued', 'running')",
|
|
1554
|
+
[(int(installation_id), int(r)) for r in repository_ids],
|
|
1555
|
+
)
|
|
1556
|
+
|
|
1557
|
+
def set_default_branch(
|
|
1558
|
+
self, installation_id: int, repository_id: int, default_branch: str | None
|
|
1559
|
+
) -> None:
|
|
1560
|
+
with self._transaction() as db:
|
|
1561
|
+
db.execute(
|
|
1562
|
+
"UPDATE known_repositories SET default_branch = ? WHERE installation_id = ? "
|
|
1563
|
+
"AND repository_id = ?",
|
|
1564
|
+
(default_branch, int(installation_id), int(repository_id)),
|
|
1565
|
+
)
|
|
1566
|
+
|
|
1567
|
+
def set_repository_details(
|
|
1568
|
+
self,
|
|
1569
|
+
installation_id: int,
|
|
1570
|
+
repository_id: int,
|
|
1571
|
+
*,
|
|
1572
|
+
default_branch: str | None,
|
|
1573
|
+
private: bool,
|
|
1574
|
+
archived: bool,
|
|
1575
|
+
) -> None:
|
|
1576
|
+
"""Record GitHub's authoritative repository details (from the API, never a client)."""
|
|
1577
|
+
with self._transaction() as db:
|
|
1578
|
+
db.execute(
|
|
1579
|
+
"UPDATE known_repositories SET default_branch = ?, private = ?, archived = ? "
|
|
1580
|
+
"WHERE installation_id = ? AND repository_id = ?",
|
|
1581
|
+
(
|
|
1582
|
+
default_branch,
|
|
1583
|
+
1 if private else 0,
|
|
1584
|
+
1 if archived else 0,
|
|
1585
|
+
int(installation_id),
|
|
1586
|
+
int(repository_id),
|
|
1587
|
+
),
|
|
1588
|
+
)
|
|
1589
|
+
|
|
1590
|
+
def monitoring_enabled(self, installation_id: int, repository_id: int) -> bool:
|
|
1591
|
+
"""False when an administrator paused CommitGuard for this repository."""
|
|
1592
|
+
rows = self._query(
|
|
1593
|
+
"SELECT monitoring_enabled FROM repository_settings WHERE installation_id = ? "
|
|
1594
|
+
"AND repository_id = ?",
|
|
1595
|
+
(int(installation_id), int(repository_id)),
|
|
1596
|
+
)
|
|
1597
|
+
return not rows or bool(rows[0]["monitoring_enabled"])
|
|
1598
|
+
|
|
1599
|
+
def repository_listed(self, installation_id: int, repository_id: int) -> bool:
|
|
1600
|
+
return bool(
|
|
1601
|
+
self._query(
|
|
1602
|
+
"SELECT 1 FROM installation_repositories WHERE installation_id = ? "
|
|
1603
|
+
"AND repository_id = ?",
|
|
1604
|
+
(int(installation_id), int(repository_id)),
|
|
1605
|
+
)
|
|
1606
|
+
)
|
|
1607
|
+
|
|
1608
|
+
def list_repositories(self, installation_id: int) -> list[RepositoryRef]:
|
|
1609
|
+
rows = self._query(
|
|
1610
|
+
"SELECT repository_id, owner, name FROM installation_repositories "
|
|
1611
|
+
"WHERE installation_id = ? ORDER BY repository_id",
|
|
1612
|
+
(int(installation_id),),
|
|
1613
|
+
)
|
|
1614
|
+
return [
|
|
1615
|
+
RepositoryRef(id=r["repository_id"], owner=r["owner"], name=r["name"]) for r in rows
|
|
1616
|
+
]
|
|
1617
|
+
|
|
1618
|
+
# -- scan jobs ------------------------------------------------------ #
|
|
1619
|
+
def _job(self, row: sqlite3.Row) -> ScanJob:
|
|
1620
|
+
return ScanJob(
|
|
1621
|
+
job_id=row["job_id"],
|
|
1622
|
+
sequence=row["sequence"],
|
|
1623
|
+
job_key=row["job_key"],
|
|
1624
|
+
installation_id=row["installation_id"],
|
|
1625
|
+
repository=RepositoryRef(id=row["repository_id"], owner=row["owner"], name=row["name"]),
|
|
1626
|
+
delivery_id=row["delivery_id"],
|
|
1627
|
+
event=row["event"],
|
|
1628
|
+
group_key=row["group_key"],
|
|
1629
|
+
head_sha=row["head_sha"],
|
|
1630
|
+
check_name=row["check_name"],
|
|
1631
|
+
pull_request_number=row["pull_request_number"],
|
|
1632
|
+
context=CIContext.model_validate_json(row["context"]),
|
|
1633
|
+
state=JobState(row["state"]),
|
|
1634
|
+
attempts=row["attempts"],
|
|
1635
|
+
created_at=_dt(row["created_at"]),
|
|
1636
|
+
updated_at=_dt(row["updated_at"]),
|
|
1637
|
+
lease_expires_at=_opt_dt(row["lease_expires_at"]),
|
|
1638
|
+
scan_id=row["scan_id"],
|
|
1639
|
+
check_run_id=row["check_run_id"],
|
|
1640
|
+
result_action=row["result_action"],
|
|
1641
|
+
conclusion=row["conclusion"],
|
|
1642
|
+
failure_kind=row["failure_kind"],
|
|
1643
|
+
message=row["message"],
|
|
1644
|
+
commits_scanned=row["commits_scanned"],
|
|
1645
|
+
violations=row["violations"],
|
|
1646
|
+
warnings=row["warnings"],
|
|
1647
|
+
base_sha=row["base_sha"],
|
|
1648
|
+
ref=row["ref"],
|
|
1649
|
+
started_at=_opt_dt(row["started_at"]),
|
|
1650
|
+
completed_at=_opt_dt(row["completed_at"]),
|
|
1651
|
+
tool_version=row["tool_version"],
|
|
1652
|
+
rules_version=row["rules_version"],
|
|
1653
|
+
policy_version=row["policy_version"],
|
|
1654
|
+
policy_source=row["policy_source"],
|
|
1655
|
+
organization_policy_version=row["organization_policy_version"],
|
|
1656
|
+
effective_policies=tuple(json.loads(row["effective_policies"] or "[]")),
|
|
1657
|
+
findings_count=row["findings_count"],
|
|
1658
|
+
detector_failures=row["detector_failures"],
|
|
1659
|
+
notices=tuple(json.loads(row["notices"] or "[]")),
|
|
1660
|
+
requested_by=row["requested_by"],
|
|
1661
|
+
scan_key=row["scan_key"] or row["job_key"],
|
|
1662
|
+
execution=row["execution"] or 1,
|
|
1663
|
+
trigger=ScanTrigger(row["trigger_kind"] or row["event"]),
|
|
1664
|
+
previous_job_id=row["previous_job_id"],
|
|
1665
|
+
)
|
|
1666
|
+
|
|
1667
|
+
def create_job(self, job: NewScanJob, now: datetime) -> tuple[ScanJob, bool]:
|
|
1668
|
+
"""Create a job unless an equivalent one is queued, running or completed.
|
|
1669
|
+
|
|
1670
|
+
An equivalent job that ended in ``error`` or ``cancelled`` does not block a
|
|
1671
|
+
new attempt (e.g. a pull request reopened after GitHub was unavailable): the
|
|
1672
|
+
new job is the next *execution* of the same logical scan.
|
|
1673
|
+
"""
|
|
1674
|
+
with self._transaction() as db:
|
|
1675
|
+
existing = db.execute(
|
|
1676
|
+
"SELECT * FROM scan_jobs WHERE installation_id = ? AND repository_id = ? "
|
|
1677
|
+
"AND job_key = ? ORDER BY sequence DESC LIMIT 1",
|
|
1678
|
+
(job.installation_id, job.repository.id, job.job_key),
|
|
1679
|
+
).fetchone()
|
|
1680
|
+
if existing is not None and existing["state"] not in ("error", "cancelled"):
|
|
1681
|
+
return self._job(existing), False
|
|
1682
|
+
row = self._insert_job(
|
|
1683
|
+
db,
|
|
1684
|
+
job,
|
|
1685
|
+
now,
|
|
1686
|
+
scan_key=job.job_key,
|
|
1687
|
+
trigger=job.trigger,
|
|
1688
|
+
previous_job_id=existing["job_id"] if existing is not None else None,
|
|
1689
|
+
)
|
|
1690
|
+
return self._job(row), True
|
|
1691
|
+
|
|
1692
|
+
def create_execution(
|
|
1693
|
+
self,
|
|
1694
|
+
previous: ScanJob,
|
|
1695
|
+
*,
|
|
1696
|
+
trigger: ScanTrigger,
|
|
1697
|
+
now: datetime,
|
|
1698
|
+
delivery_id: str | None = None,
|
|
1699
|
+
requested_by: str | None = None,
|
|
1700
|
+
) -> tuple[ScanJob, bool]:
|
|
1701
|
+
"""A new execution of the logical scan ``previous`` belongs to (re-run, manual, retry).
|
|
1702
|
+
|
|
1703
|
+
The repository, commits, event and check name are copied from the stored
|
|
1704
|
+
execution - never taken from a request. If an execution of the same scan is
|
|
1705
|
+
already queued or running, that execution is returned instead of starting
|
|
1706
|
+
another one, so repeated requests cannot pile up duplicate scans.
|
|
1707
|
+
"""
|
|
1708
|
+
with self._transaction() as db:
|
|
1709
|
+
active = db.execute(
|
|
1710
|
+
"SELECT * FROM scan_jobs WHERE installation_id = ? AND repository_id = ? "
|
|
1711
|
+
"AND scan_key = ? AND state IN ('queued', 'running') ORDER BY sequence DESC "
|
|
1712
|
+
"LIMIT 1",
|
|
1713
|
+
(previous.installation_id, previous.repository.id, previous.scan_key),
|
|
1714
|
+
).fetchone()
|
|
1715
|
+
if active is not None:
|
|
1716
|
+
return self._job(active), False
|
|
1717
|
+
if delivery_id is not None:
|
|
1718
|
+
same_delivery = db.execute(
|
|
1719
|
+
"SELECT * FROM scan_jobs WHERE installation_id = ? AND repository_id = ? "
|
|
1720
|
+
"AND scan_key = ? AND delivery_id = ?",
|
|
1721
|
+
(
|
|
1722
|
+
previous.installation_id,
|
|
1723
|
+
previous.repository.id,
|
|
1724
|
+
previous.scan_key,
|
|
1725
|
+
delivery_id,
|
|
1726
|
+
),
|
|
1727
|
+
).fetchone()
|
|
1728
|
+
if same_delivery is not None:
|
|
1729
|
+
return self._job(same_delivery), False
|
|
1730
|
+
new_job = NewScanJob(
|
|
1731
|
+
job_key=f"{trigger.value}:{uuid.uuid4().hex}",
|
|
1732
|
+
installation_id=previous.installation_id,
|
|
1733
|
+
repository=previous.repository,
|
|
1734
|
+
delivery_id=delivery_id,
|
|
1735
|
+
event=previous.event,
|
|
1736
|
+
group_key=previous.group_key,
|
|
1737
|
+
head_sha=previous.head_sha,
|
|
1738
|
+
check_name=previous.check_name,
|
|
1739
|
+
pull_request_number=previous.pull_request_number,
|
|
1740
|
+
context=previous.context,
|
|
1741
|
+
requested_by=requested_by,
|
|
1742
|
+
)
|
|
1743
|
+
row = self._insert_job(
|
|
1744
|
+
db,
|
|
1745
|
+
new_job,
|
|
1746
|
+
now,
|
|
1747
|
+
scan_key=previous.scan_key,
|
|
1748
|
+
trigger=trigger,
|
|
1749
|
+
previous_job_id=previous.job_id,
|
|
1750
|
+
)
|
|
1751
|
+
return self._job(row), True
|
|
1752
|
+
|
|
1753
|
+
def _insert_job(
|
|
1754
|
+
self,
|
|
1755
|
+
db: sqlite3.Connection,
|
|
1756
|
+
job: NewScanJob,
|
|
1757
|
+
now: datetime,
|
|
1758
|
+
*,
|
|
1759
|
+
scan_key: str,
|
|
1760
|
+
trigger: ScanTrigger,
|
|
1761
|
+
previous_job_id: str | None,
|
|
1762
|
+
) -> sqlite3.Row:
|
|
1763
|
+
db.execute(
|
|
1764
|
+
"INSERT INTO known_repositories (installation_id, repository_id, owner, name, "
|
|
1765
|
+
"first_seen_at, last_seen_at, removed_at) VALUES (?, ?, ?, ?, ?, ?, ?) "
|
|
1766
|
+
"ON CONFLICT (installation_id, repository_id) DO UPDATE SET "
|
|
1767
|
+
"last_seen_at = excluded.last_seen_at",
|
|
1768
|
+
(
|
|
1769
|
+
job.installation_id,
|
|
1770
|
+
job.repository.id,
|
|
1771
|
+
job.repository.owner,
|
|
1772
|
+
job.repository.name,
|
|
1773
|
+
_ts(now),
|
|
1774
|
+
_ts(now),
|
|
1775
|
+
None,
|
|
1776
|
+
),
|
|
1777
|
+
)
|
|
1778
|
+
execution = db.execute(
|
|
1779
|
+
"SELECT COALESCE(MAX(execution), 0) + 1 AS next FROM scan_jobs "
|
|
1780
|
+
"WHERE installation_id = ? AND repository_id = ? AND scan_key = ?",
|
|
1781
|
+
(job.installation_id, job.repository.id, scan_key),
|
|
1782
|
+
).fetchone()["next"]
|
|
1783
|
+
job_id = uuid.uuid4().hex
|
|
1784
|
+
db.execute(
|
|
1785
|
+
"INSERT INTO scan_jobs (job_id, job_key, installation_id, repository_id, owner, "
|
|
1786
|
+
"name, delivery_id, event, group_key, head_sha, check_name, pull_request_number, "
|
|
1787
|
+
"context, state, attempts, created_at, updated_at, base_sha, ref, requested_by, "
|
|
1788
|
+
"scan_key, execution, trigger_kind, previous_job_id) "
|
|
1789
|
+
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'queued', 0, "
|
|
1790
|
+
"?, ?, ?, ?, ?, ?, ?, ?, ?)",
|
|
1791
|
+
(
|
|
1792
|
+
job_id,
|
|
1793
|
+
job.job_key,
|
|
1794
|
+
job.installation_id,
|
|
1795
|
+
job.repository.id,
|
|
1796
|
+
job.repository.owner,
|
|
1797
|
+
job.repository.name,
|
|
1798
|
+
job.delivery_id,
|
|
1799
|
+
job.event,
|
|
1800
|
+
job.group_key,
|
|
1801
|
+
job.head_sha,
|
|
1802
|
+
job.check_name,
|
|
1803
|
+
job.pull_request_number,
|
|
1804
|
+
job.context.model_dump_json(),
|
|
1805
|
+
_ts(now),
|
|
1806
|
+
_ts(now),
|
|
1807
|
+
job.context.base_sha or job.context.before_sha,
|
|
1808
|
+
job.context.ref,
|
|
1809
|
+
job.requested_by,
|
|
1810
|
+
scan_key,
|
|
1811
|
+
int(execution),
|
|
1812
|
+
trigger.value,
|
|
1813
|
+
previous_job_id,
|
|
1814
|
+
),
|
|
1815
|
+
)
|
|
1816
|
+
row: sqlite3.Row | None = db.execute(
|
|
1817
|
+
"SELECT * FROM scan_jobs WHERE job_id = ?", (job_id,)
|
|
1818
|
+
).fetchone()
|
|
1819
|
+
assert row is not None # noqa: S101 - inserted above
|
|
1820
|
+
return row
|
|
1821
|
+
|
|
1822
|
+
def list_executions(
|
|
1823
|
+
self, installation_id: int, repository_id: int, scan_key: str, *, limit: int = 100
|
|
1824
|
+
) -> list[ScanJob]:
|
|
1825
|
+
rows = self._query(
|
|
1826
|
+
"SELECT * FROM scan_jobs WHERE installation_id = ? AND repository_id = ? "
|
|
1827
|
+
"AND scan_key = ? ORDER BY sequence DESC LIMIT ?",
|
|
1828
|
+
(int(installation_id), int(repository_id), scan_key, int(limit)),
|
|
1829
|
+
)
|
|
1830
|
+
return [self._job(r) for r in rows]
|
|
1831
|
+
|
|
1832
|
+
def latest_group_job(
|
|
1833
|
+
self, installation_id: int, repository_id: int, group_key: str
|
|
1834
|
+
) -> ScanJob | None:
|
|
1835
|
+
rows = self._query(
|
|
1836
|
+
"SELECT * FROM scan_jobs WHERE installation_id = ? AND repository_id = ? "
|
|
1837
|
+
"AND group_key = ? ORDER BY sequence DESC LIMIT 1",
|
|
1838
|
+
(int(installation_id), int(repository_id), group_key),
|
|
1839
|
+
)
|
|
1840
|
+
return self._job(rows[0]) if rows else None
|
|
1841
|
+
|
|
1842
|
+
def latest_jobs_for_commit(
|
|
1843
|
+
self, installation_id: int, repository_id: int, head_sha: str
|
|
1844
|
+
) -> list[ScanJob]:
|
|
1845
|
+
"""The newest execution per check name for one commit."""
|
|
1846
|
+
rows = self._query(
|
|
1847
|
+
"SELECT * FROM scan_jobs j WHERE installation_id = ? AND repository_id = ? "
|
|
1848
|
+
"AND head_sha = ? AND sequence = (SELECT MAX(sequence) FROM scan_jobs o WHERE "
|
|
1849
|
+
"o.installation_id = j.installation_id AND o.repository_id = j.repository_id "
|
|
1850
|
+
"AND o.head_sha = j.head_sha AND o.check_name = j.check_name) ORDER BY check_name",
|
|
1851
|
+
(int(installation_id), int(repository_id), head_sha),
|
|
1852
|
+
)
|
|
1853
|
+
return [self._job(r) for r in rows]
|
|
1854
|
+
|
|
1855
|
+
def get_job(self, job_id: str) -> ScanJob | None:
|
|
1856
|
+
rows = self._query("SELECT * FROM scan_jobs WHERE job_id = ?", (job_id,))
|
|
1857
|
+
return self._job(rows[0]) if rows else None
|
|
1858
|
+
|
|
1859
|
+
def list_jobs(
|
|
1860
|
+
self, *, installation_id: int, repository_id: int | None = None, limit: int = 100
|
|
1861
|
+
) -> list[ScanJob]:
|
|
1862
|
+
if repository_id is None:
|
|
1863
|
+
rows = self._query(
|
|
1864
|
+
"SELECT * FROM scan_jobs WHERE installation_id = ? ORDER BY sequence DESC LIMIT ?",
|
|
1865
|
+
(int(installation_id), int(limit)),
|
|
1866
|
+
)
|
|
1867
|
+
else:
|
|
1868
|
+
rows = self._query(
|
|
1869
|
+
"SELECT * FROM scan_jobs WHERE installation_id = ? AND repository_id = ? "
|
|
1870
|
+
"ORDER BY sequence DESC LIMIT ?",
|
|
1871
|
+
(int(installation_id), int(repository_id), int(limit)),
|
|
1872
|
+
)
|
|
1873
|
+
return [self._job(r) for r in rows]
|
|
1874
|
+
|
|
1875
|
+
def claim_job(
|
|
1876
|
+
self, job_id: str, now: datetime, lease_seconds: float, max_attempts: int
|
|
1877
|
+
) -> ScanJob | None:
|
|
1878
|
+
"""Atomically move a queued (or abandoned running) job to running."""
|
|
1879
|
+
return self.claim_job_outcome(job_id, now, lease_seconds, max_attempts).job
|
|
1880
|
+
|
|
1881
|
+
def claim_job_outcome(
|
|
1882
|
+
self, job_id: str, now: datetime, lease_seconds: float, max_attempts: int
|
|
1883
|
+
) -> ClaimOutcome:
|
|
1884
|
+
"""Claim a job; a job that used up its attempts is ended as ``error`` and returned."""
|
|
1885
|
+
with self._transaction() as db:
|
|
1886
|
+
row = db.execute("SELECT * FROM scan_jobs WHERE job_id = ?", (job_id,)).fetchone()
|
|
1887
|
+
if row is None:
|
|
1888
|
+
return ClaimOutcome()
|
|
1889
|
+
state = row["state"]
|
|
1890
|
+
abandoned = state == "running" and (row["lease_expires_at"] or 0) < _ts(now)
|
|
1891
|
+
if state != "queued" and not abandoned:
|
|
1892
|
+
return ClaimOutcome()
|
|
1893
|
+
if row["attempts"] >= max_attempts:
|
|
1894
|
+
db.execute(
|
|
1895
|
+
"UPDATE scan_jobs SET state = 'error', failure_kind = 'internal', "
|
|
1896
|
+
"message = 'scan abandoned after repeated attempts', updated_at = ?, "
|
|
1897
|
+
"lease_expires_at = NULL, completed_at = ? WHERE job_id = ?",
|
|
1898
|
+
(_ts(now), _ts(now), job_id),
|
|
1899
|
+
)
|
|
1900
|
+
row = db.execute("SELECT * FROM scan_jobs WHERE job_id = ?", (job_id,)).fetchone()
|
|
1901
|
+
return ClaimOutcome(exhausted=self._job(row))
|
|
1902
|
+
db.execute(
|
|
1903
|
+
"UPDATE scan_jobs SET state = 'running', attempts = attempts + 1, "
|
|
1904
|
+
"lease_expires_at = ?, updated_at = ?, started_at = ? WHERE job_id = ?",
|
|
1905
|
+
(_ts(now) + lease_seconds, _ts(now), _ts(now), job_id),
|
|
1906
|
+
)
|
|
1907
|
+
row = db.execute("SELECT * FROM scan_jobs WHERE job_id = ?", (job_id,)).fetchone()
|
|
1908
|
+
return ClaimOutcome(job=self._job(row))
|
|
1909
|
+
|
|
1910
|
+
def update_job(self, job_id: str, now: datetime, **fields: Any) -> None:
|
|
1911
|
+
unknown = set(fields) - set(_JOB_UPDATES)
|
|
1912
|
+
if unknown:
|
|
1913
|
+
raise ValueError(f"cannot update job fields: {', '.join(sorted(unknown))}")
|
|
1914
|
+
with self._transaction() as db:
|
|
1915
|
+
self.update_job_in(db, job_id, now, **fields)
|
|
1916
|
+
|
|
1917
|
+
@staticmethod
|
|
1918
|
+
def update_job_in(db: sqlite3.Connection, job_id: str, now: datetime, **fields: Any) -> None:
|
|
1919
|
+
""":meth:`update_job` inside the caller's transaction."""
|
|
1920
|
+
unknown = set(fields) - set(_JOB_UPDATES)
|
|
1921
|
+
if unknown:
|
|
1922
|
+
raise ValueError(f"cannot update job fields: {', '.join(sorted(unknown))}")
|
|
1923
|
+
for name, value in fields.items():
|
|
1924
|
+
if isinstance(value, StrEnum):
|
|
1925
|
+
value = value.value
|
|
1926
|
+
db.execute(_JOB_UPDATES[name], (value, _ts(now), job_id))
|
|
1927
|
+
|
|
1928
|
+
def recoverable_jobs(
|
|
1929
|
+
self, now: datetime, *, queued_before: datetime, limit: int = 100
|
|
1930
|
+
) -> list[str]:
|
|
1931
|
+
"""Jobs waiting since before ``queued_before``, and running jobs whose lease expired."""
|
|
1932
|
+
rows = self._query(
|
|
1933
|
+
"SELECT job_id FROM scan_jobs WHERE (state = 'queued' AND updated_at <= ?) "
|
|
1934
|
+
"OR (state = 'running' AND lease_expires_at < ?) ORDER BY sequence LIMIT ?",
|
|
1935
|
+
(_ts(queued_before), _ts(now), int(limit)),
|
|
1936
|
+
)
|
|
1937
|
+
return [r["job_id"] for r in rows]
|
|
1938
|
+
|
|
1939
|
+
def latest_group_sequence(
|
|
1940
|
+
self, installation_id: int, repository_id: int, group_key: str
|
|
1941
|
+
) -> int:
|
|
1942
|
+
rows = self._query(
|
|
1943
|
+
"SELECT MAX(sequence) AS latest FROM scan_jobs WHERE installation_id = ? "
|
|
1944
|
+
"AND repository_id = ? AND group_key = ?",
|
|
1945
|
+
(int(installation_id), int(repository_id), group_key),
|
|
1946
|
+
)
|
|
1947
|
+
return int(rows[0]["latest"] or 0)
|
|
1948
|
+
|
|
1949
|
+
def cancel_queued_group(
|
|
1950
|
+
self, installation_id: int, repository_id: int, group_key: str, now: datetime
|
|
1951
|
+
) -> int:
|
|
1952
|
+
with self._transaction() as db:
|
|
1953
|
+
cursor = db.execute(
|
|
1954
|
+
"UPDATE scan_jobs SET state = 'cancelled', message = 'no longer relevant', "
|
|
1955
|
+
"updated_at = ? WHERE installation_id = ? AND repository_id = ? "
|
|
1956
|
+
"AND group_key = ? AND state = 'queued'",
|
|
1957
|
+
(_ts(now), int(installation_id), int(repository_id), group_key),
|
|
1958
|
+
)
|
|
1959
|
+
return cursor.rowcount
|
|
1960
|
+
|
|
1961
|
+
# -- check run ownership --------------------------------------------- #
|
|
1962
|
+
def claim_check(
|
|
1963
|
+
self,
|
|
1964
|
+
installation_id: int,
|
|
1965
|
+
repository_id: int,
|
|
1966
|
+
head_sha: str,
|
|
1967
|
+
check_name: str,
|
|
1968
|
+
sequence: int,
|
|
1969
|
+
now: datetime,
|
|
1970
|
+
) -> CheckClaim:
|
|
1971
|
+
"""Newer jobs take over a (repository, SHA, check name) slot; older ones lose it."""
|
|
1972
|
+
key = (int(installation_id), int(repository_id), head_sha, check_name)
|
|
1973
|
+
with self._transaction() as db:
|
|
1974
|
+
row = db.execute(
|
|
1975
|
+
"SELECT owner_sequence, check_run_id FROM check_runs WHERE installation_id = ? "
|
|
1976
|
+
"AND repository_id = ? AND head_sha = ? AND check_name = ?",
|
|
1977
|
+
key,
|
|
1978
|
+
).fetchone()
|
|
1979
|
+
if row is None:
|
|
1980
|
+
db.execute(
|
|
1981
|
+
"INSERT INTO check_runs (installation_id, repository_id, head_sha, "
|
|
1982
|
+
"check_name, owner_sequence, check_run_id, updated_at) "
|
|
1983
|
+
"VALUES (?, ?, ?, ?, ?, NULL, ?)",
|
|
1984
|
+
(*key, sequence, _ts(now)),
|
|
1985
|
+
)
|
|
1986
|
+
return CheckClaim(owned=True, owner_sequence=sequence, check_run_id=None)
|
|
1987
|
+
if row["owner_sequence"] > sequence:
|
|
1988
|
+
return CheckClaim(
|
|
1989
|
+
owned=False,
|
|
1990
|
+
owner_sequence=row["owner_sequence"],
|
|
1991
|
+
check_run_id=row["check_run_id"],
|
|
1992
|
+
)
|
|
1993
|
+
db.execute(
|
|
1994
|
+
"UPDATE check_runs SET owner_sequence = ?, updated_at = ? "
|
|
1995
|
+
"WHERE installation_id = ? AND repository_id = ? AND head_sha = ? "
|
|
1996
|
+
"AND check_name = ?",
|
|
1997
|
+
(sequence, _ts(now), *key),
|
|
1998
|
+
)
|
|
1999
|
+
return CheckClaim(owned=True, owner_sequence=sequence, check_run_id=row["check_run_id"])
|
|
2000
|
+
|
|
2001
|
+
def set_check_run_id(
|
|
2002
|
+
self,
|
|
2003
|
+
installation_id: int,
|
|
2004
|
+
repository_id: int,
|
|
2005
|
+
head_sha: str,
|
|
2006
|
+
check_name: str,
|
|
2007
|
+
sequence: int,
|
|
2008
|
+
check_run_id: int,
|
|
2009
|
+
) -> bool:
|
|
2010
|
+
with self._transaction() as db:
|
|
2011
|
+
cursor = db.execute(
|
|
2012
|
+
"UPDATE check_runs SET check_run_id = ? WHERE installation_id = ? "
|
|
2013
|
+
"AND repository_id = ? AND head_sha = ? AND check_name = ? AND owner_sequence = ?",
|
|
2014
|
+
(
|
|
2015
|
+
int(check_run_id),
|
|
2016
|
+
int(installation_id),
|
|
2017
|
+
int(repository_id),
|
|
2018
|
+
head_sha,
|
|
2019
|
+
check_name,
|
|
2020
|
+
sequence,
|
|
2021
|
+
),
|
|
2022
|
+
)
|
|
2023
|
+
return cursor.rowcount == 1
|
|
2024
|
+
|
|
2025
|
+
def check_owner(
|
|
2026
|
+
self, installation_id: int, repository_id: int, head_sha: str, check_name: str
|
|
2027
|
+
) -> int | None:
|
|
2028
|
+
rows = self._query(
|
|
2029
|
+
"SELECT owner_sequence FROM check_runs WHERE installation_id = ? AND repository_id = ? "
|
|
2030
|
+
"AND head_sha = ? AND check_name = ?",
|
|
2031
|
+
(int(installation_id), int(repository_id), head_sha, check_name),
|
|
2032
|
+
)
|
|
2033
|
+
return int(rows[0]["owner_sequence"]) if rows else None
|
|
2034
|
+
|
|
2035
|
+
# -- merge queue ----------------------------------------------------- #
|
|
2036
|
+
@staticmethod
|
|
2037
|
+
def _merge_group(row: sqlite3.Row) -> MergeGroupRecord:
|
|
2038
|
+
return MergeGroupRecord(
|
|
2039
|
+
installation_id=row["installation_id"],
|
|
2040
|
+
repository_id=row["repository_id"],
|
|
2041
|
+
head_sha=row["head_sha"],
|
|
2042
|
+
head_ref=row["head_ref"],
|
|
2043
|
+
base_sha=row["base_sha"],
|
|
2044
|
+
base_ref=row["base_ref"],
|
|
2045
|
+
pull_requests=tuple(json.loads(row["pull_requests"] or "[]")),
|
|
2046
|
+
state=MergeGroupState(row["state"]),
|
|
2047
|
+
destroyed_reason=row["destroyed_reason"],
|
|
2048
|
+
job_id=row["job_id"],
|
|
2049
|
+
created_at=_dt(row["created_at"]),
|
|
2050
|
+
updated_at=_dt(row["updated_at"]),
|
|
2051
|
+
destroyed_at=_opt_dt(row["destroyed_at"]),
|
|
2052
|
+
)
|
|
2053
|
+
|
|
2054
|
+
def record_merge_group(
|
|
2055
|
+
self,
|
|
2056
|
+
*,
|
|
2057
|
+
installation_id: int,
|
|
2058
|
+
repository_id: int,
|
|
2059
|
+
head_sha: str,
|
|
2060
|
+
head_ref: str,
|
|
2061
|
+
base_sha: str,
|
|
2062
|
+
base_ref: str,
|
|
2063
|
+
pull_requests: Sequence[int],
|
|
2064
|
+
delivery_id: str | None,
|
|
2065
|
+
now: datetime,
|
|
2066
|
+
) -> tuple[MergeGroupRecord, bool]:
|
|
2067
|
+
"""Store a merge group GitHub requested checks for.
|
|
2068
|
+
|
|
2069
|
+
Returns ``(record, requested)``; ``requested`` is False when the group is
|
|
2070
|
+
already known - in particular when its ``destroyed`` event arrived first, so
|
|
2071
|
+
an out-of-order request can never revive a destroyed merge group.
|
|
2072
|
+
"""
|
|
2073
|
+
with self._transaction() as db:
|
|
2074
|
+
cursor = db.execute(
|
|
2075
|
+
"INSERT OR IGNORE INTO merge_groups (installation_id, repository_id, head_sha, "
|
|
2076
|
+
"head_ref, base_sha, base_ref, pull_requests, state, delivery_id, created_at, "
|
|
2077
|
+
"updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, 'checks_requested', ?, ?, ?)",
|
|
2078
|
+
(
|
|
2079
|
+
int(installation_id),
|
|
2080
|
+
int(repository_id),
|
|
2081
|
+
head_sha,
|
|
2082
|
+
head_ref,
|
|
2083
|
+
base_sha,
|
|
2084
|
+
base_ref,
|
|
2085
|
+
json.dumps(sorted({int(n) for n in pull_requests})),
|
|
2086
|
+
delivery_id,
|
|
2087
|
+
_ts(now),
|
|
2088
|
+
_ts(now),
|
|
2089
|
+
),
|
|
2090
|
+
)
|
|
2091
|
+
row = db.execute(
|
|
2092
|
+
"SELECT * FROM merge_groups WHERE installation_id = ? AND repository_id = ? "
|
|
2093
|
+
"AND head_sha = ?",
|
|
2094
|
+
(int(installation_id), int(repository_id), head_sha),
|
|
2095
|
+
).fetchone()
|
|
2096
|
+
return self._merge_group(row), cursor.rowcount == 1
|
|
2097
|
+
|
|
2098
|
+
def destroy_merge_group(
|
|
2099
|
+
self,
|
|
2100
|
+
*,
|
|
2101
|
+
installation_id: int,
|
|
2102
|
+
repository_id: int,
|
|
2103
|
+
head_sha: str,
|
|
2104
|
+
head_ref: str,
|
|
2105
|
+
base_sha: str,
|
|
2106
|
+
base_ref: str,
|
|
2107
|
+
pull_requests: Sequence[int],
|
|
2108
|
+
reason: str | None,
|
|
2109
|
+
now: datetime,
|
|
2110
|
+
) -> tuple[MergeGroupRecord, bool]:
|
|
2111
|
+
"""Mark a merge group destroyed (inserting it if its request was never seen).
|
|
2112
|
+
|
|
2113
|
+
Returns ``(record, changed)``; repeated ``destroyed`` events change nothing.
|
|
2114
|
+
"""
|
|
2115
|
+
with self._transaction() as db:
|
|
2116
|
+
db.execute(
|
|
2117
|
+
"INSERT OR IGNORE INTO merge_groups (installation_id, repository_id, head_sha, "
|
|
2118
|
+
"head_ref, base_sha, base_ref, pull_requests, state, created_at, updated_at) "
|
|
2119
|
+
"VALUES (?, ?, ?, ?, ?, ?, ?, 'checks_requested', ?, ?)",
|
|
2120
|
+
(
|
|
2121
|
+
int(installation_id),
|
|
2122
|
+
int(repository_id),
|
|
2123
|
+
head_sha,
|
|
2124
|
+
head_ref,
|
|
2125
|
+
base_sha,
|
|
2126
|
+
base_ref,
|
|
2127
|
+
json.dumps(sorted({int(n) for n in pull_requests})),
|
|
2128
|
+
_ts(now),
|
|
2129
|
+
_ts(now),
|
|
2130
|
+
),
|
|
2131
|
+
)
|
|
2132
|
+
cursor = db.execute(
|
|
2133
|
+
"UPDATE merge_groups SET state = 'destroyed', destroyed_reason = ?, "
|
|
2134
|
+
"destroyed_at = ?, updated_at = ? WHERE installation_id = ? "
|
|
2135
|
+
"AND repository_id = ? AND head_sha = ? AND state != 'destroyed'",
|
|
2136
|
+
(reason, _ts(now), _ts(now), int(installation_id), int(repository_id), head_sha),
|
|
2137
|
+
)
|
|
2138
|
+
row = db.execute(
|
|
2139
|
+
"SELECT * FROM merge_groups WHERE installation_id = ? AND repository_id = ? "
|
|
2140
|
+
"AND head_sha = ?",
|
|
2141
|
+
(int(installation_id), int(repository_id), head_sha),
|
|
2142
|
+
).fetchone()
|
|
2143
|
+
return self._merge_group(row), cursor.rowcount == 1
|
|
2144
|
+
|
|
2145
|
+
def set_merge_group_job(
|
|
2146
|
+
self, installation_id: int, repository_id: int, head_sha: str, job_id: str, now: datetime
|
|
2147
|
+
) -> None:
|
|
2148
|
+
with self._transaction() as db:
|
|
2149
|
+
db.execute(
|
|
2150
|
+
"UPDATE merge_groups SET job_id = ?, updated_at = ? WHERE installation_id = ? "
|
|
2151
|
+
"AND repository_id = ? AND head_sha = ?",
|
|
2152
|
+
(job_id, _ts(now), int(installation_id), int(repository_id), head_sha),
|
|
2153
|
+
)
|
|
2154
|
+
|
|
2155
|
+
def get_merge_group(
|
|
2156
|
+
self, installation_id: int, repository_id: int, head_sha: str
|
|
2157
|
+
) -> MergeGroupRecord | None:
|
|
2158
|
+
rows = self._query(
|
|
2159
|
+
"SELECT * FROM merge_groups WHERE installation_id = ? AND repository_id = ? "
|
|
2160
|
+
"AND head_sha = ?",
|
|
2161
|
+
(int(installation_id), int(repository_id), head_sha),
|
|
2162
|
+
)
|
|
2163
|
+
return self._merge_group(rows[0]) if rows else None
|
|
2164
|
+
|
|
2165
|
+
def list_merge_groups(
|
|
2166
|
+
self, installation_id: int, repository_id: int, *, limit: int = 10
|
|
2167
|
+
) -> list[MergeGroupRecord]:
|
|
2168
|
+
rows = self._query(
|
|
2169
|
+
"SELECT * FROM merge_groups WHERE installation_id = ? AND repository_id = ? "
|
|
2170
|
+
"ORDER BY created_at DESC, head_sha LIMIT ?",
|
|
2171
|
+
(int(installation_id), int(repository_id), int(limit)),
|
|
2172
|
+
)
|
|
2173
|
+
return [self._merge_group(r) for r in rows]
|
|
2174
|
+
|
|
2175
|
+
# -- audit ----------------------------------------------------------- #
|
|
2176
|
+
def append_audit_event(self, event: AuditEvent) -> None:
|
|
2177
|
+
with self._transaction() as db:
|
|
2178
|
+
self.insert_audit_event(db, event)
|
|
2179
|
+
|
|
2180
|
+
@staticmethod
|
|
2181
|
+
def insert_audit_event(db: sqlite3.Connection, event: AuditEvent) -> AuditEvent:
|
|
2182
|
+
"""Store ``event`` inside the caller's transaction.
|
|
2183
|
+
|
|
2184
|
+
The tenant (``account_id``) is taken from the installation when the event
|
|
2185
|
+
does not name one, so every event about an installation is visible to -
|
|
2186
|
+
and only to - that account.
|
|
2187
|
+
"""
|
|
2188
|
+
if event.account_id is None and event.installation_id is not None:
|
|
2189
|
+
row = db.execute(
|
|
2190
|
+
"SELECT account_id FROM installations WHERE installation_id = ?",
|
|
2191
|
+
(event.installation_id,),
|
|
2192
|
+
).fetchone()
|
|
2193
|
+
if row is not None:
|
|
2194
|
+
event = event.model_copy(update={"account_id": row["account_id"]})
|
|
2195
|
+
db.execute(
|
|
2196
|
+
"INSERT INTO audit_events (event_id, occurred_at, type, installation_id, "
|
|
2197
|
+
"repository_id, account_id, actor_login, document) VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
|
|
2198
|
+
(
|
|
2199
|
+
event.event_id,
|
|
2200
|
+
_ts(event.occurred_at),
|
|
2201
|
+
event.type.value,
|
|
2202
|
+
event.installation_id,
|
|
2203
|
+
event.repository_id,
|
|
2204
|
+
event.account_id,
|
|
2205
|
+
event.actor_login,
|
|
2206
|
+
event.model_dump_json(),
|
|
2207
|
+
),
|
|
2208
|
+
)
|
|
2209
|
+
return event
|
|
2210
|
+
|
|
2211
|
+
def list_audit_events(
|
|
2212
|
+
self, *, installation_id: int, repository_id: int | None = None, limit: int = 100
|
|
2213
|
+
) -> list[AuditEvent]:
|
|
2214
|
+
if repository_id is None:
|
|
2215
|
+
rows = self._query(
|
|
2216
|
+
"SELECT document FROM audit_events WHERE installation_id = ? "
|
|
2217
|
+
"ORDER BY occurred_at DESC LIMIT ?",
|
|
2218
|
+
(int(installation_id), int(limit)),
|
|
2219
|
+
)
|
|
2220
|
+
else:
|
|
2221
|
+
rows = self._query(
|
|
2222
|
+
"SELECT document FROM audit_events WHERE installation_id = ? AND repository_id = ? "
|
|
2223
|
+
"ORDER BY occurred_at DESC LIMIT ?",
|
|
2224
|
+
(int(installation_id), int(repository_id), int(limit)),
|
|
2225
|
+
)
|
|
2226
|
+
return [AuditEvent.model_validate_json(r["document"]) for r in rows]
|
|
2227
|
+
|
|
2228
|
+
def purge_audit_events(self, before: datetime) -> int:
|
|
2229
|
+
with self._transaction() as db:
|
|
2230
|
+
return db.execute(
|
|
2231
|
+
"DELETE FROM audit_events WHERE occurred_at < ?", (_ts(before),)
|
|
2232
|
+
).rowcount
|
|
2233
|
+
|
|
2234
|
+
# -- retention ------------------------------------------------------- #
|
|
2235
|
+
def purge_expired(self, before: datetime) -> dict[str, int]:
|
|
2236
|
+
"""Delete deliveries, finished jobs, check slots, audit events and removed
|
|
2237
|
+
installations older than ``before``."""
|
|
2238
|
+
cutoff = _ts(before)
|
|
2239
|
+
with self._transaction() as db:
|
|
2240
|
+
# Control plane: history older than the cutoff, except anything an open
|
|
2241
|
+
# violation still depends on.
|
|
2242
|
+
findings = db.execute(
|
|
2243
|
+
"DELETE FROM findings WHERE created_at < ? AND (violation_id IS NULL OR "
|
|
2244
|
+
"violation_id NOT IN (SELECT violation_id FROM violations WHERE status = 'open'))",
|
|
2245
|
+
(cutoff,),
|
|
2246
|
+
).rowcount
|
|
2247
|
+
db.execute(
|
|
2248
|
+
"DELETE FROM violation_exposures WHERE violation_id IN (SELECT violation_id FROM "
|
|
2249
|
+
"violations WHERE status = 'resolved' AND updated_at < ?)",
|
|
2250
|
+
(cutoff,),
|
|
2251
|
+
)
|
|
2252
|
+
violations = db.execute(
|
|
2253
|
+
"DELETE FROM violations WHERE status = 'resolved' AND updated_at < ?", (cutoff,)
|
|
2254
|
+
).rowcount
|
|
2255
|
+
counts = {
|
|
2256
|
+
"merge_groups": db.execute(
|
|
2257
|
+
"DELETE FROM merge_groups WHERE updated_at < ? AND state = 'destroyed'",
|
|
2258
|
+
(cutoff,),
|
|
2259
|
+
).rowcount,
|
|
2260
|
+
"findings": findings,
|
|
2261
|
+
"violations": violations,
|
|
2262
|
+
"deliveries": db.execute(
|
|
2263
|
+
"DELETE FROM deliveries WHERE received_at < ?", (cutoff,)
|
|
2264
|
+
).rowcount,
|
|
2265
|
+
"scan_jobs": db.execute(
|
|
2266
|
+
"DELETE FROM scan_jobs WHERE updated_at < ? AND state NOT IN "
|
|
2267
|
+
"('queued', 'running')",
|
|
2268
|
+
(cutoff,),
|
|
2269
|
+
).rowcount,
|
|
2270
|
+
"check_runs": db.execute(
|
|
2271
|
+
"DELETE FROM check_runs WHERE updated_at < ?", (cutoff,)
|
|
2272
|
+
).rowcount,
|
|
2273
|
+
"audit_events": db.execute(
|
|
2274
|
+
"DELETE FROM audit_events WHERE occurred_at < ?", (cutoff,)
|
|
2275
|
+
).rowcount,
|
|
2276
|
+
"installations": db.execute(
|
|
2277
|
+
"DELETE FROM installations WHERE state = 'deleted' AND updated_at < ?",
|
|
2278
|
+
(cutoff,),
|
|
2279
|
+
).rowcount,
|
|
2280
|
+
}
|
|
2281
|
+
# Organization governance: finished background work. Policy versions, drafts,
|
|
2282
|
+
# approvals, exceptions, rollouts and rule versions are history and are kept.
|
|
2283
|
+
counts["simulations"] = db.execute(
|
|
2284
|
+
"DELETE FROM policy_simulations WHERE state IN ('completed', 'failed') "
|
|
2285
|
+
"AND completed_at < ?",
|
|
2286
|
+
(cutoff,),
|
|
2287
|
+
).rowcount
|
|
2288
|
+
db.execute(
|
|
2289
|
+
"DELETE FROM bulk_operation_items WHERE operation_id IN (SELECT operation_id "
|
|
2290
|
+
"FROM bulk_operations WHERE status IN ('completed', 'partial', 'failed', "
|
|
2291
|
+
"'cancelled') AND updated_at < ?)",
|
|
2292
|
+
(cutoff,),
|
|
2293
|
+
)
|
|
2294
|
+
counts["bulk_operations"] = db.execute(
|
|
2295
|
+
"DELETE FROM bulk_operations WHERE status IN ('completed', 'partial', 'failed', "
|
|
2296
|
+
"'cancelled') AND updated_at < ?",
|
|
2297
|
+
(cutoff,),
|
|
2298
|
+
).rowcount
|
|
2299
|
+
# A slot is never due again once its schedule moved on, so old runs can go.
|
|
2300
|
+
counts["schedule_runs"] = db.execute(
|
|
2301
|
+
"DELETE FROM scan_schedule_runs WHERE state != 'running' AND started_at < ?",
|
|
2302
|
+
(cutoff,),
|
|
2303
|
+
).rowcount
|
|
2304
|
+
# Data whose installation record is gone can no longer be attributed to a
|
|
2305
|
+
# tenant: remove it rather than keep it unreachable.
|
|
2306
|
+
for statement in _ORPHAN_DELETES:
|
|
2307
|
+
db.execute(statement)
|
|
2308
|
+
counts["repositories"] = db.execute(
|
|
2309
|
+
"DELETE FROM known_repositories WHERE removed_at IS NOT NULL AND removed_at < ? "
|
|
2310
|
+
"AND NOT EXISTS (SELECT 1 FROM scan_jobs j WHERE j.installation_id = "
|
|
2311
|
+
"known_repositories.installation_id AND j.repository_id = "
|
|
2312
|
+
"known_repositories.repository_id)",
|
|
2313
|
+
(cutoff,),
|
|
2314
|
+
).rowcount
|
|
2315
|
+
return counts
|