keble-scraper-api 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- keble_scraper_api/__init__.py +3 -0
- keble_scraper_api/api/dependencies.py +100 -0
- keble_scraper_api/api/errors.py +72 -0
- keble_scraper_api/api/router.py +473 -0
- keble_scraper_api/api/schemas.py +26 -0
- keble_scraper_api/application/admin.py +285 -0
- keble_scraper_api/application/artifacts.py +393 -0
- keble_scraper_api/application/auth.py +324 -0
- keble_scraper_api/application/failures.py +427 -0
- keble_scraper_api/application/fetching.py +72 -0
- keble_scraper_api/application/jobs.py +891 -0
- keble_scraper_api/application/proxy_health.py +126 -0
- keble_scraper_api/application/scheduling.py +24 -0
- keble_scraper_api/artifact_namespace.py +76 -0
- keble_scraper_api/container.py +361 -0
- keble_scraper_api/errors.py +25 -0
- keble_scraper_api/infrastructure/artifact_store.py +585 -0
- keble_scraper_api/infrastructure/callbacks.py +285 -0
- keble_scraper_api/infrastructure/celery_factory.py +63 -0
- keble_scraper_api/infrastructure/cloudflare.py +259 -0
- keble_scraper_api/infrastructure/credentials.py +75 -0
- keble_scraper_api/infrastructure/dispatcher.py +57 -0
- keble_scraper_api/infrastructure/http_fetcher.py +351 -0
- keble_scraper_api/infrastructure/mongo.py +2345 -0
- keble_scraper_api/infrastructure/proxy_router.py +460 -0
- keble_scraper_api/infrastructure/request_profiles.py +220 -0
- keble_scraper_api/infrastructure/url_policy.py +341 -0
- keble_scraper_api/main.py +162 -0
- keble_scraper_api/maintenance/__init__.py +1 -0
- keble_scraper_api/maintenance/reset_pre_release_state.py +743 -0
- keble_scraper_api/models.py +473 -0
- keble_scraper_api/protocols.py +947 -0
- keble_scraper_api/settings.py +484 -0
- keble_scraper_api/telemetry.py +109 -0
- keble_scraper_api/workers/celery_app.py +48 -0
- keble_scraper_api/workers/cli.py +106 -0
- keble_scraper_api/workers/runtime.py +78 -0
- keble_scraper_api/workers/tasks.py +97 -0
- keble_scraper_api-0.2.0.dist-info/METADATA +160 -0
- keble_scraper_api-0.2.0.dist-info/RECORD +42 -0
- keble_scraper_api-0.2.0.dist-info/WHEEL +4 -0
- keble_scraper_api-0.2.0.dist-info/entry_points.txt +4 -0
|
@@ -0,0 +1,324 @@
|
|
|
1
|
+
"""Truth-backed Google sign-in, encrypted sessions, and static service keys."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
import hmac
|
|
7
|
+
from datetime import UTC, datetime, timedelta
|
|
8
|
+
|
|
9
|
+
import httpx
|
|
10
|
+
from cryptography.fernet import Fernet
|
|
11
|
+
from keble_helpers.typings import PydanticModelConfig
|
|
12
|
+
from pydantic import AliasChoices, BaseModel, Field, SecretStr
|
|
13
|
+
|
|
14
|
+
from keble_scraper_contract import OperatorRole, OperatorSession
|
|
15
|
+
|
|
16
|
+
from keble_scraper_api.errors import ScraperDomainError, ScraperErrorCode
|
|
17
|
+
from keble_scraper_api.settings import RuntimeEnvironment, ScraperSettings
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class TruthUser(BaseModel):
|
|
21
|
+
"""Validated Truth identity returned by the Google workspace exchange.
|
|
22
|
+
|
|
23
|
+
Side effects if changes:
|
|
24
|
+
- active-role/domain gates and encrypted browser sessions.
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
model_config = PydanticModelConfig.default(frozen=True, extra="ignore")
|
|
28
|
+
|
|
29
|
+
id: str | int
|
|
30
|
+
email: str
|
|
31
|
+
role: str
|
|
32
|
+
status: str
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class TruthExchange(BaseModel):
|
|
36
|
+
"""Server-only Truth exchange result including the private bearer.
|
|
37
|
+
|
|
38
|
+
Side effects if changes:
|
|
39
|
+
- encrypted cookie contents; the bearer must never enter browser JSON.
|
|
40
|
+
"""
|
|
41
|
+
|
|
42
|
+
model_config = PydanticModelConfig.default(frozen=True, extra="ignore")
|
|
43
|
+
|
|
44
|
+
access_token: str = Field(
|
|
45
|
+
min_length=16,
|
|
46
|
+
validation_alias=AliasChoices("accessToken", "access_token"),
|
|
47
|
+
)
|
|
48
|
+
expires_at: datetime | None = Field(
|
|
49
|
+
default=None,
|
|
50
|
+
validation_alias=AliasChoices("expiresAt", "expires_at"),
|
|
51
|
+
)
|
|
52
|
+
user: TruthUser
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
class EncryptedSessionPayload(BaseModel):
|
|
56
|
+
"""Backend-only encrypted cookie payload with non-exported Truth bearer.
|
|
57
|
+
|
|
58
|
+
Side effects if changes:
|
|
59
|
+
- session rotation and browser authentication continuity.
|
|
60
|
+
"""
|
|
61
|
+
|
|
62
|
+
model_config = PydanticModelConfig.default(frozen=True, extra="forbid")
|
|
63
|
+
|
|
64
|
+
session: OperatorSession
|
|
65
|
+
truth_bearer: SecretStr | None = None
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
class SessionCipher:
|
|
69
|
+
"""Encrypt and validate HttpOnly session payloads.
|
|
70
|
+
|
|
71
|
+
Steps:
|
|
72
|
+
1. own one stable or development-only ephemeral Fernet key;
|
|
73
|
+
2. encrypt private Truth-backed session payloads;
|
|
74
|
+
3. decrypt and validate them before authorization.
|
|
75
|
+
|
|
76
|
+
Side effects if changes:
|
|
77
|
+
- all console authentication and stored Truth bearer secrecy.
|
|
78
|
+
"""
|
|
79
|
+
|
|
80
|
+
def __init__(self, *, key: SecretStr | None) -> None:
|
|
81
|
+
"""Use a configured stable key or an ephemeral local-development key.
|
|
82
|
+
|
|
83
|
+
Steps:
|
|
84
|
+
1. unwrap configured write-only material when present;
|
|
85
|
+
2. otherwise generate an ephemeral development key;
|
|
86
|
+
3. construct the authenticated Fernet cipher exactly once.
|
|
87
|
+
|
|
88
|
+
Side effects if changes:
|
|
89
|
+
- sessions survive deployment restarts only with configured secret material.
|
|
90
|
+
"""
|
|
91
|
+
|
|
92
|
+
material = (
|
|
93
|
+
key.get_secret_value().encode()
|
|
94
|
+
if key is not None
|
|
95
|
+
else Fernet.generate_key()
|
|
96
|
+
)
|
|
97
|
+
self._fernet = Fernet(material)
|
|
98
|
+
|
|
99
|
+
def encrypt(self, *, payload: EncryptedSessionPayload) -> str:
|
|
100
|
+
"""Serialize and encrypt one session cookie value.
|
|
101
|
+
|
|
102
|
+
Steps:
|
|
103
|
+
1. serialize the strict private payload using wire aliases;
|
|
104
|
+
2. authenticate and encrypt its exact bytes;
|
|
105
|
+
3. return URL-safe cookie text without exposing the bearer.
|
|
106
|
+
|
|
107
|
+
Side effects if changes:
|
|
108
|
+
- browser cookie bytes and future decryption compatibility.
|
|
109
|
+
"""
|
|
110
|
+
|
|
111
|
+
return self._fernet.encrypt(
|
|
112
|
+
payload.model_dump_json(by_alias=True).encode()
|
|
113
|
+
).decode()
|
|
114
|
+
|
|
115
|
+
async def decrypt(self, *, token: str) -> EncryptedSessionPayload:
|
|
116
|
+
"""Decrypt, validate, and reject expired session material.
|
|
117
|
+
|
|
118
|
+
Steps:
|
|
119
|
+
1. perform CPU-bound authenticated decryption off the event loop;
|
|
120
|
+
2. validate the private payload against its strict Pydantic schema;
|
|
121
|
+
3. reject a structurally valid but expired operator session.
|
|
122
|
+
|
|
123
|
+
Side effects if changes:
|
|
124
|
+
- every protected admin API authorization decision;
|
|
125
|
+
- invalid Fernet tokens propagate to the dedicated HTTP auth handler.
|
|
126
|
+
"""
|
|
127
|
+
|
|
128
|
+
raw = await asyncio.to_thread(self._fernet.decrypt, token.encode())
|
|
129
|
+
payload = EncryptedSessionPayload.model_validate_json(raw)
|
|
130
|
+
if payload.session.expires_at <= datetime.now(UTC):
|
|
131
|
+
raise ScraperDomainError(
|
|
132
|
+
code=ScraperErrorCode.AUTH_INVALID,
|
|
133
|
+
message="operator session has expired",
|
|
134
|
+
)
|
|
135
|
+
return payload
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
class OperatorAuthService:
|
|
139
|
+
"""Exchange Google credentials through Truth and issue safe sessions.
|
|
140
|
+
|
|
141
|
+
Steps:
|
|
142
|
+
1. exchange Google credentials only through the configured Truth service;
|
|
143
|
+
2. enforce active Keble role/domain policy;
|
|
144
|
+
3. mint encrypted production or explicit local-development sessions.
|
|
145
|
+
|
|
146
|
+
Side effects if changes:
|
|
147
|
+
- production identity, role authorization, and development bypass.
|
|
148
|
+
"""
|
|
149
|
+
|
|
150
|
+
def __init__(self, *, settings: ScraperSettings, cipher: SessionCipher) -> None:
|
|
151
|
+
"""Bind environment policy and encrypted session ownership.
|
|
152
|
+
|
|
153
|
+
Side effects if changes:
|
|
154
|
+
- all auth routes and admin dependencies.
|
|
155
|
+
"""
|
|
156
|
+
|
|
157
|
+
self._settings = settings
|
|
158
|
+
self._cipher = cipher
|
|
159
|
+
|
|
160
|
+
async def google_login(self, *, credential: str) -> tuple[OperatorSession, str]:
|
|
161
|
+
"""Post one Google ID token to Truth and mint an HttpOnly session.
|
|
162
|
+
|
|
163
|
+
Steps:
|
|
164
|
+
1. never store or return the submitted Google credential;
|
|
165
|
+
2. accept only active Keble roles and `@keble.ai` identities;
|
|
166
|
+
3. encrypt the Truth bearer server-side inside the cookie payload.
|
|
167
|
+
|
|
168
|
+
Side effects if changes:
|
|
169
|
+
- external Truth auth call and browser session creation.
|
|
170
|
+
"""
|
|
171
|
+
|
|
172
|
+
if self._settings.truth_base_url is None:
|
|
173
|
+
raise ScraperDomainError(
|
|
174
|
+
code=ScraperErrorCode.CONFIGURATION_INVALID,
|
|
175
|
+
message="Truth authentication is not configured",
|
|
176
|
+
)
|
|
177
|
+
endpoint = (
|
|
178
|
+
str(self._settings.truth_base_url).rstrip("/")
|
|
179
|
+
+ self._settings.truth_auth_path
|
|
180
|
+
)
|
|
181
|
+
async with httpx.AsyncClient(
|
|
182
|
+
timeout=self._settings.truth_timeout_seconds,
|
|
183
|
+
trust_env=False,
|
|
184
|
+
) as client:
|
|
185
|
+
raw = await client.post(endpoint, json={"id_token": credential})
|
|
186
|
+
if not raw.is_success:
|
|
187
|
+
raise ScraperDomainError(
|
|
188
|
+
code=ScraperErrorCode.AUTH_INVALID,
|
|
189
|
+
message="Google workspace identity exchange failed",
|
|
190
|
+
)
|
|
191
|
+
exchange = await asyncio.to_thread(
|
|
192
|
+
TruthExchange.model_validate_json,
|
|
193
|
+
raw.content,
|
|
194
|
+
)
|
|
195
|
+
if exchange.user.status != "ACTIVE" or exchange.user.role not in {
|
|
196
|
+
role.value for role in OperatorRole
|
|
197
|
+
}:
|
|
198
|
+
raise ScraperDomainError(
|
|
199
|
+
code=ScraperErrorCode.AUTH_FORBIDDEN,
|
|
200
|
+
message="Truth identity is not an active Keble operator",
|
|
201
|
+
)
|
|
202
|
+
email = exchange.user.email.strip().lower()
|
|
203
|
+
if not email.endswith("@keble.ai"):
|
|
204
|
+
raise ScraperDomainError(
|
|
205
|
+
code=ScraperErrorCode.AUTH_FORBIDDEN,
|
|
206
|
+
message="Google identity is outside the Keble workspace",
|
|
207
|
+
)
|
|
208
|
+
configured_expiry = datetime.now(UTC) + timedelta(
|
|
209
|
+
hours=self._settings.session_hours
|
|
210
|
+
)
|
|
211
|
+
expires_at = min(
|
|
212
|
+
configured_expiry,
|
|
213
|
+
exchange.expires_at or configured_expiry,
|
|
214
|
+
)
|
|
215
|
+
session = OperatorSession(
|
|
216
|
+
user_id=str(exchange.user.id),
|
|
217
|
+
email=email,
|
|
218
|
+
role=OperatorRole(exchange.user.role),
|
|
219
|
+
expires_at=expires_at,
|
|
220
|
+
is_local=False,
|
|
221
|
+
)
|
|
222
|
+
cookie = self._cipher.encrypt(
|
|
223
|
+
payload=EncryptedSessionPayload(
|
|
224
|
+
session=session,
|
|
225
|
+
truth_bearer=SecretStr(exchange.access_token),
|
|
226
|
+
)
|
|
227
|
+
)
|
|
228
|
+
return session, cookie
|
|
229
|
+
|
|
230
|
+
def dev_login(self) -> tuple[OperatorSession, str]:
|
|
231
|
+
"""Mint a visibly local principal only under the explicit development gate.
|
|
232
|
+
|
|
233
|
+
Steps:
|
|
234
|
+
1. require both development environment and explicit bypass enablement;
|
|
235
|
+
2. build a bounded visibly local operator session;
|
|
236
|
+
3. encrypt it without a Truth bearer.
|
|
237
|
+
|
|
238
|
+
Side effects if changes:
|
|
239
|
+
- local bypass access; production settings validation must remain aligned.
|
|
240
|
+
"""
|
|
241
|
+
|
|
242
|
+
if (
|
|
243
|
+
self._settings.environment is not RuntimeEnvironment.DEVELOPMENT
|
|
244
|
+
or not self._settings.dev_auth_enabled
|
|
245
|
+
):
|
|
246
|
+
raise ScraperDomainError(
|
|
247
|
+
code=ScraperErrorCode.AUTH_FORBIDDEN,
|
|
248
|
+
message="development authentication bypass is disabled",
|
|
249
|
+
)
|
|
250
|
+
session = OperatorSession(
|
|
251
|
+
user_id="local-scraper-operator",
|
|
252
|
+
email=self._settings.dev_auth_email,
|
|
253
|
+
role=self._settings.dev_auth_role,
|
|
254
|
+
expires_at=datetime.now(UTC)
|
|
255
|
+
+ timedelta(hours=self._settings.session_hours),
|
|
256
|
+
is_local=True,
|
|
257
|
+
)
|
|
258
|
+
cookie = self._cipher.encrypt(
|
|
259
|
+
payload=EncryptedSessionPayload(session=session, truth_bearer=None)
|
|
260
|
+
)
|
|
261
|
+
return session, cookie
|
|
262
|
+
|
|
263
|
+
async def session_from_cookie(self, *, token: str | None) -> OperatorSession:
|
|
264
|
+
"""Resolve a safe current session or fail closed.
|
|
265
|
+
|
|
266
|
+
Side effects if changes:
|
|
267
|
+
- every console read/mutation authorization dependency.
|
|
268
|
+
"""
|
|
269
|
+
|
|
270
|
+
if token is None:
|
|
271
|
+
raise ScraperDomainError(
|
|
272
|
+
code=ScraperErrorCode.AUTH_INVALID,
|
|
273
|
+
message="operator session is required",
|
|
274
|
+
)
|
|
275
|
+
return (await self._cipher.decrypt(token=token)).session
|
|
276
|
+
|
|
277
|
+
|
|
278
|
+
class StaticServiceKeyAuthorizer:
|
|
279
|
+
"""Constant-time current/next service key rotation boundary.
|
|
280
|
+
|
|
281
|
+
Steps:
|
|
282
|
+
1. retain independent current and optional next keys;
|
|
283
|
+
2. compare supplied credentials in constant time;
|
|
284
|
+
3. reject absence and mismatch with one finite auth code.
|
|
285
|
+
|
|
286
|
+
Side effects if changes:
|
|
287
|
+
- Shopify submission/read access to scraper APIs.
|
|
288
|
+
"""
|
|
289
|
+
|
|
290
|
+
def __init__(self, *, current: SecretStr, next_key: SecretStr | None) -> None:
|
|
291
|
+
"""Bind one audience's rotation pair.
|
|
292
|
+
|
|
293
|
+
Side effects if changes:
|
|
294
|
+
- independent service credential rollout.
|
|
295
|
+
"""
|
|
296
|
+
|
|
297
|
+
self._current = current
|
|
298
|
+
self._next = next_key
|
|
299
|
+
|
|
300
|
+
def authorize(self, *, supplied: str | None) -> None:
|
|
301
|
+
"""Accept the current or next key and reject every absent/mismatched value.
|
|
302
|
+
|
|
303
|
+
Steps:
|
|
304
|
+
1. normalize an absent header to a non-matching value;
|
|
305
|
+
2. compare current and optional next keys independently;
|
|
306
|
+
3. reject unless at least one rotation key matches.
|
|
307
|
+
|
|
308
|
+
Side effects if changes:
|
|
309
|
+
- service API authentication.
|
|
310
|
+
"""
|
|
311
|
+
|
|
312
|
+
candidate = supplied or ""
|
|
313
|
+
current_matches = hmac.compare_digest(
|
|
314
|
+
candidate,
|
|
315
|
+
self._current.get_secret_value(),
|
|
316
|
+
)
|
|
317
|
+
next_matches = self._next is not None and hmac.compare_digest(
|
|
318
|
+
candidate, self._next.get_secret_value()
|
|
319
|
+
)
|
|
320
|
+
if not current_matches and not next_matches:
|
|
321
|
+
raise ScraperDomainError(
|
|
322
|
+
code=ScraperErrorCode.AUTH_INVALID,
|
|
323
|
+
message="service credential is invalid",
|
|
324
|
+
)
|