keble-scraper-api 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- keble_scraper_api/__init__.py +3 -0
- keble_scraper_api/api/dependencies.py +100 -0
- keble_scraper_api/api/errors.py +72 -0
- keble_scraper_api/api/router.py +473 -0
- keble_scraper_api/api/schemas.py +26 -0
- keble_scraper_api/application/admin.py +285 -0
- keble_scraper_api/application/artifacts.py +393 -0
- keble_scraper_api/application/auth.py +324 -0
- keble_scraper_api/application/failures.py +427 -0
- keble_scraper_api/application/fetching.py +72 -0
- keble_scraper_api/application/jobs.py +891 -0
- keble_scraper_api/application/proxy_health.py +126 -0
- keble_scraper_api/application/scheduling.py +24 -0
- keble_scraper_api/artifact_namespace.py +76 -0
- keble_scraper_api/container.py +361 -0
- keble_scraper_api/errors.py +25 -0
- keble_scraper_api/infrastructure/artifact_store.py +585 -0
- keble_scraper_api/infrastructure/callbacks.py +285 -0
- keble_scraper_api/infrastructure/celery_factory.py +63 -0
- keble_scraper_api/infrastructure/cloudflare.py +259 -0
- keble_scraper_api/infrastructure/credentials.py +75 -0
- keble_scraper_api/infrastructure/dispatcher.py +57 -0
- keble_scraper_api/infrastructure/http_fetcher.py +351 -0
- keble_scraper_api/infrastructure/mongo.py +2345 -0
- keble_scraper_api/infrastructure/proxy_router.py +460 -0
- keble_scraper_api/infrastructure/request_profiles.py +220 -0
- keble_scraper_api/infrastructure/url_policy.py +341 -0
- keble_scraper_api/main.py +162 -0
- keble_scraper_api/maintenance/__init__.py +1 -0
- keble_scraper_api/maintenance/reset_pre_release_state.py +743 -0
- keble_scraper_api/models.py +473 -0
- keble_scraper_api/protocols.py +947 -0
- keble_scraper_api/settings.py +484 -0
- keble_scraper_api/telemetry.py +109 -0
- keble_scraper_api/workers/celery_app.py +48 -0
- keble_scraper_api/workers/cli.py +106 -0
- keble_scraper_api/workers/runtime.py +78 -0
- keble_scraper_api/workers/tasks.py +97 -0
- keble_scraper_api-0.2.0.dist-info/METADATA +160 -0
- keble_scraper_api-0.2.0.dist-info/RECORD +42 -0
- keble_scraper_api-0.2.0.dist-info/WHEEL +4 -0
- keble_scraper_api-0.2.0.dist-info/entry_points.txt +4 -0
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
"""Explicit worker/beat command-line entrypoints."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import sys
|
|
7
|
+
from enum import StrEnum
|
|
8
|
+
|
|
9
|
+
from keble_scraper_api.settings import ScraperSettings
|
|
10
|
+
from keble_scraper_api.telemetry import TelemetryRuntime, configure_sentry
|
|
11
|
+
from keble_scraper_api.workers.celery_app import celery_app, settings
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class WorkerKind(StrEnum):
|
|
15
|
+
"""Finite scraper worker roles with isolated queue ownership.
|
|
16
|
+
|
|
17
|
+
Side effects if changes:
|
|
18
|
+
- worker process topology and deployment command contracts.
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
FETCH = "fetch"
|
|
22
|
+
RENDER = "render"
|
|
23
|
+
PROXY_HEALTH = "proxy-health"
|
|
24
|
+
MAINTENANCE = "maintenance"
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def worker_arguments(*, kind: WorkerKind, settings: ScraperSettings) -> list[str]:
|
|
28
|
+
"""Build one queue- and concurrency-bounded Celery worker command.
|
|
29
|
+
|
|
30
|
+
Side effects if changes:
|
|
31
|
+
- queue isolation, process count, and deployment capacity requirements.
|
|
32
|
+
"""
|
|
33
|
+
|
|
34
|
+
queue_by_kind = {
|
|
35
|
+
WorkerKind.FETCH: settings.fetch_queue,
|
|
36
|
+
WorkerKind.RENDER: settings.render_queue,
|
|
37
|
+
WorkerKind.PROXY_HEALTH: settings.proxy_health_queue,
|
|
38
|
+
WorkerKind.MAINTENANCE: settings.maintenance_queue,
|
|
39
|
+
}
|
|
40
|
+
concurrency_by_kind = {
|
|
41
|
+
WorkerKind.FETCH: settings.fetch_worker_concurrency,
|
|
42
|
+
WorkerKind.RENDER: settings.render_worker_concurrency,
|
|
43
|
+
WorkerKind.PROXY_HEALTH: settings.proxy_health_worker_concurrency,
|
|
44
|
+
WorkerKind.MAINTENANCE: settings.maintenance_worker_concurrency,
|
|
45
|
+
}
|
|
46
|
+
return [
|
|
47
|
+
"worker",
|
|
48
|
+
"--loglevel=INFO",
|
|
49
|
+
"--queues",
|
|
50
|
+
queue_by_kind[kind],
|
|
51
|
+
"--concurrency",
|
|
52
|
+
str(concurrency_by_kind[kind]),
|
|
53
|
+
]
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def beat_arguments() -> list[str]:
|
|
57
|
+
"""Build arguments for Celery's already-entered command parser.
|
|
58
|
+
|
|
59
|
+
Side effects if changes:
|
|
60
|
+
- periodic maintenance, callback, and proxy-health publication startup.
|
|
61
|
+
"""
|
|
62
|
+
|
|
63
|
+
return ["beat", "--loglevel=INFO"]
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def run_worker() -> None:
|
|
67
|
+
"""Run one queue-specific Celery worker process.
|
|
68
|
+
|
|
69
|
+
Steps:
|
|
70
|
+
1. configure parent-process telemetry before argument handling;
|
|
71
|
+
2. validate the finite worker kind from the command line;
|
|
72
|
+
3. enter Celery with settings-owned queue and concurrency arguments.
|
|
73
|
+
|
|
74
|
+
Side effects if changes:
|
|
75
|
+
- consumes RabbitMQ tasks and starts child process resources.
|
|
76
|
+
"""
|
|
77
|
+
|
|
78
|
+
configure_sentry(settings=settings, runtime=TelemetryRuntime.CELERY_PARENT)
|
|
79
|
+
parser = argparse.ArgumentParser(prog="keble-scraper-worker")
|
|
80
|
+
parser.add_argument(
|
|
81
|
+
"kind",
|
|
82
|
+
choices=tuple(kind.value for kind in WorkerKind),
|
|
83
|
+
)
|
|
84
|
+
parsed = parser.parse_args(sys.argv[1:])
|
|
85
|
+
celery_app.worker_main(
|
|
86
|
+
worker_arguments(
|
|
87
|
+
kind=WorkerKind(parsed.kind),
|
|
88
|
+
settings=settings,
|
|
89
|
+
)
|
|
90
|
+
)
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def run_beat() -> None:
|
|
94
|
+
"""Run Celery beat for maintenance, callback, and health schedules.
|
|
95
|
+
|
|
96
|
+
Steps:
|
|
97
|
+
1. rebind telemetry to the dedicated scheduler process role;
|
|
98
|
+
2. build the finite beat command arguments;
|
|
99
|
+
3. enter Celery's periodic publisher loop.
|
|
100
|
+
|
|
101
|
+
Side effects if changes:
|
|
102
|
+
- publishes periodic RabbitMQ tasks.
|
|
103
|
+
"""
|
|
104
|
+
|
|
105
|
+
configure_sentry(settings=settings, runtime=TelemetryRuntime.BEAT)
|
|
106
|
+
celery_app.start(beat_arguments())
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
"""Persistent async runtime owned by each Celery child process."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
from collections.abc import Coroutine
|
|
7
|
+
from typing import Any, TypeVar
|
|
8
|
+
|
|
9
|
+
from keble_scraper_api.container import ScraperContainer
|
|
10
|
+
from keble_scraper_api.settings import ScraperSettings
|
|
11
|
+
|
|
12
|
+
ResultT = TypeVar("ResultT")
|
|
13
|
+
_worker_loop: asyncio.AbstractEventLoop | None = None
|
|
14
|
+
_worker_container: ScraperContainer | None = None
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def initialize_worker_runtime(*, settings: ScraperSettings) -> None:
|
|
18
|
+
"""Start one reusable async graph for the current Celery child.
|
|
19
|
+
|
|
20
|
+
Steps:
|
|
21
|
+
1. reject duplicate process initialization;
|
|
22
|
+
2. build one event loop and typed dependency graph;
|
|
23
|
+
3. await startup directly so initialization faults reach Celery/Sentry;
|
|
24
|
+
4. publish globals only after readiness succeeds.
|
|
25
|
+
|
|
26
|
+
Side effects if changes:
|
|
27
|
+
- worker event loop, indexes, Mongo/Redis connection pools.
|
|
28
|
+
"""
|
|
29
|
+
|
|
30
|
+
global _worker_loop, _worker_container
|
|
31
|
+
if _worker_loop is not None or _worker_container is not None:
|
|
32
|
+
raise RuntimeError("worker runtime is already initialized")
|
|
33
|
+
loop = asyncio.new_event_loop()
|
|
34
|
+
asyncio.set_event_loop(loop)
|
|
35
|
+
container = ScraperContainer.build(settings=settings)
|
|
36
|
+
loop.run_until_complete(container.start())
|
|
37
|
+
_worker_loop = loop
|
|
38
|
+
_worker_container = container
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def shutdown_worker_runtime() -> None:
|
|
42
|
+
"""Close the reusable graph and event loop exactly once.
|
|
43
|
+
|
|
44
|
+
Side effects if changes:
|
|
45
|
+
- worker process resource cleanup.
|
|
46
|
+
"""
|
|
47
|
+
|
|
48
|
+
global _worker_loop, _worker_container
|
|
49
|
+
if _worker_loop is None or _worker_container is None:
|
|
50
|
+
return
|
|
51
|
+
_worker_loop.run_until_complete(_worker_container.close())
|
|
52
|
+
_worker_loop.close()
|
|
53
|
+
_worker_loop = None
|
|
54
|
+
_worker_container = None
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def require_worker_container() -> ScraperContainer:
|
|
58
|
+
"""Return the initialized child-process graph or fail closed.
|
|
59
|
+
|
|
60
|
+
Side effects if changes:
|
|
61
|
+
- every Celery task dependency resolution.
|
|
62
|
+
"""
|
|
63
|
+
|
|
64
|
+
if _worker_container is None:
|
|
65
|
+
raise RuntimeError("worker runtime is not initialized")
|
|
66
|
+
return _worker_container
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def run_async(operation: Coroutine[Any, Any, ResultT]) -> ResultT:
|
|
70
|
+
"""Execute one task coroutine on the persistent child event loop.
|
|
71
|
+
|
|
72
|
+
Side effects if changes:
|
|
73
|
+
- all worker async network/database execution.
|
|
74
|
+
"""
|
|
75
|
+
|
|
76
|
+
if _worker_loop is None:
|
|
77
|
+
raise RuntimeError("worker runtime is not initialized")
|
|
78
|
+
return _worker_loop.run_until_complete(operation)
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
"""Opaque-identity Celery task entrypoints."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
import socket
|
|
7
|
+
from uuid import UUID
|
|
8
|
+
|
|
9
|
+
import sentry_sdk
|
|
10
|
+
|
|
11
|
+
from keble_scraper_api.infrastructure.callbacks import CallbackDeliveryOutcome
|
|
12
|
+
from keble_scraper_api.workers.celery_app import celery_app
|
|
13
|
+
from keble_scraper_api.workers.runtime import require_worker_container, run_async
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def _worker_id() -> str:
|
|
17
|
+
"""Return one stable process lease owner identity.
|
|
18
|
+
|
|
19
|
+
Side effects if changes:
|
|
20
|
+
- Mongo lease ownership and late acknowledgement checks.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
return f"{socket.gethostname()}:{os.getpid()}"
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@celery_app.task(name="keble_scraper.execute_job")
|
|
27
|
+
def execute_job(job_id: str) -> None:
|
|
28
|
+
"""Execute one Mongo-owned job without serializing request/secret payloads.
|
|
29
|
+
|
|
30
|
+
Side effects if changes:
|
|
31
|
+
- external fetch/vendor calls, attempts, artifacts, and outbox state.
|
|
32
|
+
"""
|
|
33
|
+
|
|
34
|
+
container = require_worker_container()
|
|
35
|
+
with sentry_sdk.start_span(op="scraper.job", name="execute") as span:
|
|
36
|
+
result = run_async(
|
|
37
|
+
container.execution.process(
|
|
38
|
+
job_id=UUID(job_id),
|
|
39
|
+
worker_id=_worker_id(),
|
|
40
|
+
)
|
|
41
|
+
)
|
|
42
|
+
span.set_data("claimed", result is not None)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
@celery_app.task(name="keble_scraper.run_maintenance")
|
|
46
|
+
def run_maintenance() -> None:
|
|
47
|
+
"""Run bounded recovery/retention/redispatch maintenance.
|
|
48
|
+
|
|
49
|
+
Side effects if changes:
|
|
50
|
+
- Mongo/OSS cleanup and RabbitMQ task publication.
|
|
51
|
+
"""
|
|
52
|
+
|
|
53
|
+
container = require_worker_container()
|
|
54
|
+
with sentry_sdk.start_span(op="scraper.maintenance", name="run") as span:
|
|
55
|
+
result = run_async(container.maintenance.run())
|
|
56
|
+
for key, value in result.items():
|
|
57
|
+
span.set_data(key, value)
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
@celery_app.task(name="keble_scraper.deliver_callbacks")
|
|
61
|
+
def deliver_callbacks() -> None:
|
|
62
|
+
"""Deliver a bounded batch of signed callback outbox events.
|
|
63
|
+
|
|
64
|
+
Side effects if changes:
|
|
65
|
+
- external Shopify callback calls and outbox transitions.
|
|
66
|
+
"""
|
|
67
|
+
|
|
68
|
+
with sentry_sdk.start_span(op="scraper.callbacks", name="deliver") as span:
|
|
69
|
+
container = require_worker_container()
|
|
70
|
+
delivered = 0
|
|
71
|
+
rescheduled = 0
|
|
72
|
+
for _index in range(100):
|
|
73
|
+
outcome = run_async(
|
|
74
|
+
container.callbacks.deliver_one(worker_id=_worker_id())
|
|
75
|
+
)
|
|
76
|
+
if outcome is CallbackDeliveryOutcome.EMPTY:
|
|
77
|
+
break
|
|
78
|
+
if outcome is CallbackDeliveryOutcome.DELIVERED:
|
|
79
|
+
delivered += 1
|
|
80
|
+
else:
|
|
81
|
+
rescheduled += 1
|
|
82
|
+
span.set_data("delivered_count", delivered)
|
|
83
|
+
span.set_data("rescheduled_count", rescheduled)
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
@celery_app.task(name="keble_scraper.check_proxies")
|
|
87
|
+
def check_proxies() -> None:
|
|
88
|
+
"""Check and decay/promote a bounded set of quarantined public proxies.
|
|
89
|
+
|
|
90
|
+
Side effects if changes:
|
|
91
|
+
- experimental proxy traffic and health Mongo state.
|
|
92
|
+
"""
|
|
93
|
+
|
|
94
|
+
with sentry_sdk.start_span(op="scraper.proxy_health", name="check") as span:
|
|
95
|
+
container = require_worker_container()
|
|
96
|
+
checked = run_async(container.proxy_health.run_due())
|
|
97
|
+
span.set_data("checked_count", checked)
|
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: keble-scraper-api
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: Asynchronous policy-bounded scraping API, workers, proxy router, and artifact service.
|
|
5
|
+
Author-email: zma <bob0103779@gmail.com>
|
|
6
|
+
Requires-Python: <3.14,>=3.13
|
|
7
|
+
Requires-Dist: aiohttp<4,>=3.11
|
|
8
|
+
Requires-Dist: celery<6,>=5.6
|
|
9
|
+
Requires-Dist: cryptography<47,>=45
|
|
10
|
+
Requires-Dist: fastapi<1,>=0.116
|
|
11
|
+
Requires-Dist: google-auth<3,>=2.40
|
|
12
|
+
Requires-Dist: httpx<1,>=0.28
|
|
13
|
+
Requires-Dist: keble-data-infra-contract<1,>=0.9.0
|
|
14
|
+
Requires-Dist: keble-helpers<2,>=1.54.0
|
|
15
|
+
Requires-Dist: keble-scraper-contract==0.3.0
|
|
16
|
+
Requires-Dist: nh3<1,>=0.3
|
|
17
|
+
Requires-Dist: oss2<3,>=2.19
|
|
18
|
+
Requires-Dist: pydantic-settings<3,>=2.10
|
|
19
|
+
Requires-Dist: pymongo<5,>=4.12
|
|
20
|
+
Requires-Dist: redis<7,>=5
|
|
21
|
+
Requires-Dist: sentry-sdk[celery,fastapi]<3,>=2.32
|
|
22
|
+
Requires-Dist: uvicorn[standard]<1,>=0.35
|
|
23
|
+
Description-Content-Type: text/markdown
|
|
24
|
+
|
|
25
|
+
# Keble Scraper API
|
|
26
|
+
|
|
27
|
+
The API package owns FastAPI composition, Mongo lifecycle/index persistence,
|
|
28
|
+
Redis-backed egress budgets and sessions, Celery workers/beat, secure HTTP and
|
|
29
|
+
Cloudflare fetching, private OSS artifacts, HMAC callbacks, authentication,
|
|
30
|
+
auditing, and Sentry instrumentation. It imports `keble-scraper-contract`; the
|
|
31
|
+
contract never imports this package.
|
|
32
|
+
|
|
33
|
+
Version `0.2.0` retains the undiscounted Bright pay-as-you-go baseline for
|
|
34
|
+
pre-dispatch cost reservation so temporary coupon pricing cannot understate
|
|
35
|
+
production spend and releases typed provider recovery, crash-safe callback and
|
|
36
|
+
job budgets, bounded history, artifact-namespace ownership, ordered retention,
|
|
37
|
+
and the guarded first-release reset.
|
|
38
|
+
|
|
39
|
+
The workspace consumes `keble-scraper-contract 0.3.0` as the foundation for
|
|
40
|
+
the API `0.2.0` recovery release. Attempt/job/callback
|
|
41
|
+
persistence stores discriminated typed failure snapshots instead of terminal
|
|
42
|
+
strings, and retry classification uses their finite dispositions. Public HTTP
|
|
43
|
+
errors use the same envelope; mandatory retention revisions prevent delayed pin
|
|
44
|
+
delivery from reversing a newer paid-storage policy.
|
|
45
|
+
|
|
46
|
+
Each job and callback row owns its immutable attempt budget;
|
|
47
|
+
jobs also own a fixed deadline. Lease acquisition and recovery enforce those
|
|
48
|
+
stored values, temporary capacity retains its future horizon, and unexpected
|
|
49
|
+
runtime faults remain leased for expiry recovery rather than becoming a false
|
|
50
|
+
network failure. Cache pointers cannot outlive referenced artifacts, missing
|
|
51
|
+
cache metadata invalidates the pointer before refetch, and expired callback
|
|
52
|
+
leases are recovered or dead-lettered by maintenance.
|
|
53
|
+
|
|
54
|
+
The WP8 clean-break command lives at `scripts/reset_pre_release_state.py` and
|
|
55
|
+
delegates to the package-owned typed maintenance module. DRY_RUN is the default.
|
|
56
|
+
Local/test-only APPLY deletes referenced binaries before their Mongo metadata,
|
|
57
|
+
then the fixed Scraper collection allowlist and exact Redis namespace. It
|
|
58
|
+
requires explicit writer-stop, database, Redis, artifact-owner, snapshot,
|
|
59
|
+
artifact-inventory, campaign, and immutable-audit evidence; failures propagate
|
|
60
|
+
with metadata or RUNNING evidence retained for replay. See `../docs/RUNBOOK.md`.
|
|
61
|
+
|
|
62
|
+
HTTP, Cloudflare, and registered callback transports own reusable connection
|
|
63
|
+
pools closed by `ScraperContainer`. The shared Data Infra Retry-After parser is
|
|
64
|
+
used at every response boundary, with durable due times rounded upward to Mongo
|
|
65
|
+
millisecond precision. Maintenance projects due job ID plus transport policy in
|
|
66
|
+
one indexed read and publishes with configured bounded concurrency.
|
|
67
|
+
|
|
68
|
+
Maintenance also owns bounded historical retirement. Terminal Jobs without a
|
|
69
|
+
callback receive an absolute retention clock immediately; callback Jobs receive
|
|
70
|
+
one only in the same transaction that marks their outbox event delivered.
|
|
71
|
+
Pending, leased, and dead-letter outbox rows therefore cannot be purged. One
|
|
72
|
+
Mongo singleton lease serializes each minute's bounded purge, and the majority
|
|
73
|
+
transaction deletes each expired Job with all immutable Attempt children before
|
|
74
|
+
reporting success. Delivered callbacks and audit facts use independent absolute
|
|
75
|
+
clocks. Defaults are 90 days for Job/Attempt evidence, 30 days for delivered
|
|
76
|
+
callback replay, 400 days for security audit, and 1,000 rows per collection per
|
|
77
|
+
minute; every value is a typed `SCRAPER_*` environment setting.
|
|
78
|
+
|
|
79
|
+
Sentry is initialized explicitly for the API, Celery parent, every worker child,
|
|
80
|
+
and beat. Runtime tags distinguish those processes while credentials, URLs,
|
|
81
|
+
request bodies, and provider payloads stay out of telemetry configuration.
|
|
82
|
+
|
|
83
|
+
Side effects if changes:
|
|
84
|
+
- Shopify must repin and exhaustively map local versus upstream failures;
|
|
85
|
+
- frontend error presentation and OpenAPI generation ship with the API release;
|
|
86
|
+
- pre-production `0.1.x` scraper job rows require the guarded reset/repair step.
|
|
87
|
+
|
|
88
|
+
Runtime commands:
|
|
89
|
+
|
|
90
|
+
```bash
|
|
91
|
+
uv run --package keble-scraper-api keble-scraper-api
|
|
92
|
+
uv run --package keble-scraper-api keble-scraper-worker fetch
|
|
93
|
+
uv run --package keble-scraper-api keble-scraper-worker render
|
|
94
|
+
uv run --package keble-scraper-api keble-scraper-worker proxy-health
|
|
95
|
+
uv run --package keble-scraper-api keble-scraper-worker maintenance
|
|
96
|
+
uv run --package keble-scraper-api keble-scraper-beat
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
Service-authenticated consumers may download a verified artifact through the
|
|
100
|
+
audited signed-URL endpoint and synchronize a downstream retention pin with a
|
|
101
|
+
required positive `retentionRevision`. The Mongo transaction increments the
|
|
102
|
+
artifact reference only on first owner-pin creation, applies only newer policy
|
|
103
|
+
revisions, ignores stale deliveries, and rejects a divergent replay at the same
|
|
104
|
+
revision. Both finite archive expiry and indefinite latest retention are valid
|
|
105
|
+
ordered policies.
|
|
106
|
+
The operator and service paths share the same public download route but retain
|
|
107
|
+
separate authorization and audit identities.
|
|
108
|
+
|
|
109
|
+
The download service reserves a server UUID, places it in the immutable audit
|
|
110
|
+
event and the signed URL's `x-keble-audit-id` query, and returns the URL only
|
|
111
|
+
after the audit append succeeds. OSS `GetObject` logs can therefore reconcile
|
|
112
|
+
reads exactly. New OSS objects also carry their canonical SHA-256 as private
|
|
113
|
+
user metadata for bounded integrity checks.
|
|
114
|
+
|
|
115
|
+
For a multi-process local canary without OSS, configure both
|
|
116
|
+
`SCRAPER_LOCAL_ARTIFACT_DIRECTORY` and `SCRAPER_LOCAL_ARTIFACT_BASE_URL`, then
|
|
117
|
+
serve only that isolated directory on localhost. The development adapter keeps
|
|
118
|
+
content-addressed bytes shared between API and Celery workers; staging and
|
|
119
|
+
production reject it and continue to require private OSS.
|
|
120
|
+
|
|
121
|
+
Empty or whitespace-only values for those two optional local-artifact settings
|
|
122
|
+
are treated as absent. This supports explicit Compose overrides when private OSS
|
|
123
|
+
replaces a legacy local-artifact `env_file` configuration; all backend-completeness
|
|
124
|
+
and environment restrictions still apply.
|
|
125
|
+
|
|
126
|
+
The optional OSS endpoint, bucket, access-key ID, and access-key secret are also
|
|
127
|
+
normalized from blank to absent as one typed tuple. Partial or empty production
|
|
128
|
+
configuration fails startup before any external request or artifact write.
|
|
129
|
+
|
|
130
|
+
Startup now applies two independent artifact fences before readiness. Mongo
|
|
131
|
+
persists and compares the complete non-secret backend identity: backend kind,
|
|
132
|
+
deterministic prefix, OSS endpoint/bucket or resolved local directory, and the
|
|
133
|
+
configured owner. The binary store also contains one deterministic owner canary
|
|
134
|
+
inside that prefix. Existing rows without a complete marker, backend/location
|
|
135
|
+
changes, and a second owner all fail closed before workers accept jobs.
|
|
136
|
+
|
|
137
|
+
Set `SCRAPER_ARTIFACT_NAMESPACE_OWNER` in staging and production to a stable,
|
|
138
|
+
non-secret value unique to the Mongo deployment/database that owns the prefix,
|
|
139
|
+
for example `cn-hangzhou/mongo-primary/keble_scraper`. Do not reuse it for a
|
|
140
|
+
second database. Development/test may derive `environment/database` locally.
|
|
141
|
+
The pre-production prefix-only marker is intentionally incompatible and must be
|
|
142
|
+
cleared or explicitly migrated before this release line starts.
|
|
143
|
+
|
|
144
|
+
When a local VPN returns fake-IP DNS answers, configure both
|
|
145
|
+
`SCRAPER_DEVELOPMENT_HTTP_PROXY_URL` and
|
|
146
|
+
`SCRAPER_DEVELOPMENT_DNS_OVER_HTTPS_URL`. The worker validates real DoH A/AAAA
|
|
147
|
+
answers through the explicit proxy, preserves IP pinning, and applies the proxy
|
|
148
|
+
only to the verified `DIRECT` route. The pair is rejected outside
|
|
149
|
+
development/test.
|
|
150
|
+
|
|
151
|
+
Every route admission is atomically assigned a future Redis timestamp derived
|
|
152
|
+
from its requests-per-second policy. A fetch worker asynchronously waits for
|
|
153
|
+
that timestamp, preserving the requested route and job attempt; internal burst
|
|
154
|
+
pressure therefore does not become terminal `ROUTE_UNAVAILABLE` evidence or a
|
|
155
|
+
paid fallback.
|
|
156
|
+
|
|
157
|
+
The repository release workflow builds this package's runtime image only from
|
|
158
|
+
an annotated `keble-scraper-api-vX.Y.Z` tag whose version matches this package.
|
|
159
|
+
GHCR version/source tags are discovery aids; production consumes the digest
|
|
160
|
+
evidence artifact and pins `ghcr.io/keble-ai/keble-scraper-api@sha256:...`.
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
keble_scraper_api/__init__.py,sha256=noNqhxeGjGXaW416uANBLazuUsVq2_iRQju895gaxHk,98
|
|
2
|
+
keble_scraper_api/artifact_namespace.py,sha256=umwA7FnZSItZafkSs4bd00R2wQ57Sw3rQ1L3SfCSGxo,3119
|
|
3
|
+
keble_scraper_api/container.py,sha256=0NWvpeFhqylR7uS0d_RfrvmtnjfH9Ny_YKRkMcPnPNQ,13266
|
|
4
|
+
keble_scraper_api/errors.py,sha256=vgGhmc3cid71dkZXTo5xY0dnpMBhsDlIsNU6JHUKeew,734
|
|
5
|
+
keble_scraper_api/main.py,sha256=Mmv_QXh_UjK5dlsxHGHuBmsv1EodJpukpGAnKRxKp7s,5066
|
|
6
|
+
keble_scraper_api/models.py,sha256=fskufDh3Wn9Ab3eQPEtZBorJtykikx-XvtMzbFDj8aM,14940
|
|
7
|
+
keble_scraper_api/protocols.py,sha256=DUUh8yDIEw1MVOALYnmH0y5K8OtkEdJIsKmF90UM6i0,25598
|
|
8
|
+
keble_scraper_api/settings.py,sha256=LhGuWd860Mxj-7vg7LrUS8_hZTxxmygaZAY3xTm-xbE,20402
|
|
9
|
+
keble_scraper_api/telemetry.py,sha256=8wAcJRTUkkeJ07os3AxvGgkTwZzT_QvkFe7-DQWyp-c,3348
|
|
10
|
+
keble_scraper_api/api/dependencies.py,sha256=8sSWU9OOeU0LjgUiQbEPYpf-qUPHYy-vF-cefKf5d_U,2867
|
|
11
|
+
keble_scraper_api/api/errors.py,sha256=Y4pyn_2fuQtWw3vpc29L2ZCO292ABBi7W0qlkxNFasw,2423
|
|
12
|
+
keble_scraper_api/api/router.py,sha256=gGVjHHCO19Bs85XX5QzDwQ3LTS9D4iKNK-XeQCJY68g,15213
|
|
13
|
+
keble_scraper_api/api/schemas.py,sha256=UzBg8SFrySwMuZbQMRpUYVF0eLSNGcg3Yd3nlW4cDTk,630
|
|
14
|
+
keble_scraper_api/application/admin.py,sha256=uiAv1O7KGjAyOu_SvXQ9fwHlXuLBDUIdGJOlbonuT2E,8744
|
|
15
|
+
keble_scraper_api/application/artifacts.py,sha256=bvflOw7jBG79ttjdFxd7iH3EDQvJTGQjr5dkcMx907Y,13200
|
|
16
|
+
keble_scraper_api/application/auth.py,sha256=YzqjCf5-L9iXIu4N03FuOFBE-71MlFOztKzUpPTTMXw,11244
|
|
17
|
+
keble_scraper_api/application/failures.py,sha256=VOLWf0GF79NGQTZgFQjdzb6cBXZPfT8mSmTx1dCmdS4,15940
|
|
18
|
+
keble_scraper_api/application/fetching.py,sha256=Oae3e-DbRV_JVUSdQOL5G7SiA2MFw6TuNp6-4jCkXl0,2294
|
|
19
|
+
keble_scraper_api/application/jobs.py,sha256=Bi2Ip0-gA2KIpuGbJ6duNuu2ZAPte6lOVlw7EAZcUMc,31432
|
|
20
|
+
keble_scraper_api/application/proxy_health.py,sha256=Poc9lJT2ys_BjxOqwu6dsTmqipGMxcecvg2TWILzsxQ,4404
|
|
21
|
+
keble_scraper_api/application/scheduling.py,sha256=FP22dgXUFIA6_LFIZ2Hjl6lcZwhBZHIkZU0NAlSwXLg,878
|
|
22
|
+
keble_scraper_api/infrastructure/artifact_store.py,sha256=7lp3bTOurkjXgxOeb2cMxwcyIfQENEnsxEzFh3rAUIQ,20405
|
|
23
|
+
keble_scraper_api/infrastructure/callbacks.py,sha256=nNc6fWqfpvbcZKH9G2gwv-D8oGx8amYNqyyV2h9e5E4,9807
|
|
24
|
+
keble_scraper_api/infrastructure/celery_factory.py,sha256=UtnjlNK9fqiFvTSmykFScb71IKOsxvUfCeDzfa_aurg,2128
|
|
25
|
+
keble_scraper_api/infrastructure/cloudflare.py,sha256=HnRmBqfP_ew3BsrvFGPiVHUn0pJqOdguWLWILxNC_q4,9305
|
|
26
|
+
keble_scraper_api/infrastructure/credentials.py,sha256=pDioRKbK6jH4gAfCHkClONT043tAxkGygYaEaI3cCUM,2674
|
|
27
|
+
keble_scraper_api/infrastructure/dispatcher.py,sha256=_wBgRqHuBRBGpRFML8dYyrqJQiVS4JOwK71g5qmFBUU,1550
|
|
28
|
+
keble_scraper_api/infrastructure/http_fetcher.py,sha256=GK1nn97Wo7ms_23vnkvR09ZU9RrESF-5wiiVyhQ8xLM,12749
|
|
29
|
+
keble_scraper_api/infrastructure/mongo.py,sha256=uEee0jree5N2atfnZhdtQEB_yqDV9NSWk-weGi-9-wE,86666
|
|
30
|
+
keble_scraper_api/infrastructure/proxy_router.py,sha256=abbn4a5AN_ste6Ziyy7M7qGw44irbT8WThDwNqaA6KU,16295
|
|
31
|
+
keble_scraper_api/infrastructure/request_profiles.py,sha256=opVWjcY4hmjImI76lsirHKcu_3JwZEFIDm7qnAVDS04,6982
|
|
32
|
+
keble_scraper_api/infrastructure/url_policy.py,sha256=lix2Ss46seby19cPU3Oh7i7BOS3kiCf4ltqpxVmsq18,11871
|
|
33
|
+
keble_scraper_api/maintenance/__init__.py,sha256=68yBjQEPprl2DRTC89TEj4d_cJNsapmk3p38c0YLdf4,73
|
|
34
|
+
keble_scraper_api/maintenance/reset_pre_release_state.py,sha256=5OhYgvcy5a_Dk5Y6pByThzmgLW7uSyu7E72EiX7NRmg,24539
|
|
35
|
+
keble_scraper_api/workers/celery_app.py,sha256=EW87tI9Dp9ZnjnKEbHDzQUHu2xR2COH9lYvGnj265r8,1636
|
|
36
|
+
keble_scraper_api/workers/cli.py,sha256=ET2kVPh1c5mN25bxRmfnILvtDaezMtXcd5qhqqolDYc,3211
|
|
37
|
+
keble_scraper_api/workers/runtime.py,sha256=mTAUAH6ZptMURIu56gFGnOMuT1B1gNDd4gflCXz2PLM,2479
|
|
38
|
+
keble_scraper_api/workers/tasks.py,sha256=EB-UrTEwOl8P2pvWfdYi6qRikB0Pg5N1rdvdVJGZ7bg,3137
|
|
39
|
+
keble_scraper_api-0.2.0.dist-info/METADATA,sha256=9-7TYLcoB68WC5B6HR-GdSY7bhigDlmvyack7z1I9PE,8836
|
|
40
|
+
keble_scraper_api-0.2.0.dist-info/WHEEL,sha256=lCkmxWfQsSc9CfIClYeavTdQeEX2toPqufh9gI35EQA,87
|
|
41
|
+
keble_scraper_api-0.2.0.dist-info/entry_points.txt,sha256=4AovGM7UVcUssi-kWFDf8ijnHXB-uSb9Hk7SJnCLPEo,189
|
|
42
|
+
keble_scraper_api-0.2.0.dist-info/RECORD,,
|