keble-data-infra-api 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- keble_data_infra_api-0.1.0/.gitignore +42 -0
- keble_data_infra_api-0.1.0/Dockerfile +45 -0
- keble_data_infra_api-0.1.0/PKG-INFO +152 -0
- keble_data_infra_api-0.1.0/README.md +130 -0
- keble_data_infra_api-0.1.0/pyproject.toml +67 -0
- keble_data_infra_api-0.1.0/scripts/__init__.py +1 -0
- keble_data_infra_api-0.1.0/scripts/build_local_docker_env.py +78 -0
- keble_data_infra_api-0.1.0/scripts/build_local_env.py +134 -0
- keble_data_infra_api-0.1.0/scripts/export_openapi.py +58 -0
- keble_data_infra_api-0.1.0/src/keble_data_infra_api/__init__.py +2 -0
- keble_data_infra_api-0.1.0/src/keble_data_infra_api/api/__init__.py +2 -0
- keble_data_infra_api-0.1.0/src/keble_data_infra_api/api/error_handlers.py +126 -0
- keble_data_infra_api-0.1.0/src/keble_data_infra_api/api/router.py +177 -0
- keble_data_infra_api-0.1.0/src/keble_data_infra_api/api/video_enrichment_router.py +169 -0
- keble_data_infra_api-0.1.0/src/keble_data_infra_api/integrations/__init__.py +2 -0
- keble_data_infra_api-0.1.0/src/keble_data_infra_api/integrations/video_enrichment/__init__.py +2 -0
- keble_data_infra_api-0.1.0/src/keble_data_infra_api/integrations/video_enrichment/service.py +452 -0
- keble_data_infra_api-0.1.0/src/keble_data_infra_api/main.py +66 -0
- keble_data_infra_api-0.1.0/src/keble_data_infra_api/runtime/__init__.py +1 -0
- keble_data_infra_api-0.1.0/src/keble_data_infra_api/runtime/bootstrap.py +347 -0
- keble_data_infra_api-0.1.0/src/keble_data_infra_api/runtime/observability.py +24 -0
- keble_data_infra_api-0.1.0/src/keble_data_infra_api/runtime/provider_registry.py +78 -0
- keble_data_infra_api-0.1.0/src/keble_data_infra_api/runtime/settings.py +216 -0
- keble_data_infra_api-0.1.0/src/keble_data_infra_api/runtime/video_enrichment_client.py +275 -0
- keble_data_infra_api-0.1.0/src/keble_data_infra_api/runtime/video_enrichment_redis.py +202 -0
- keble_data_infra_api-0.1.0/src/keble_data_infra_api/runtime/video_enrichment_store.py +507 -0
- keble_data_infra_api-0.1.0/tests/__init__.py +1 -0
- keble_data_infra_api-0.1.0/tests/conftest.py +23 -0
- keble_data_infra_api-0.1.0/tests/integration/rest/test_api_app.py +153 -0
- keble_data_infra_api-0.1.0/tests/integration/test_provider_runtime.py +94 -0
- keble_data_infra_api-0.1.0/tests/integration/test_video_enrichment_store.py +213 -0
- keble_data_infra_api-0.1.0/tests/live/test_provider_routes_live.py +123 -0
- keble_data_infra_api-0.1.0/tests/live/test_video_enrichment_result_read_live.py +42 -0
- keble_data_infra_api-0.1.0/tests/unit/integrations/video_enrichment/test_ingestion_service.py +345 -0
- keble_data_infra_api-0.1.0/tests/unit/test_bootstrap.py +64 -0
- keble_data_infra_api-0.1.0/tests/unit/test_dependency_direction.py +178 -0
- keble_data_infra_api-0.1.0/tests/unit/test_local_env_scripts.py +64 -0
- keble_data_infra_api-0.1.0/tests/unit/test_observability.py +49 -0
- keble_data_infra_api-0.1.0/tests/unit/test_package_imports.py +21 -0
- keble_data_infra_api-0.1.0/tests/unit/test_persistence_inheritance.py +40 -0
- keble_data_infra_api-0.1.0/tests/unit/test_provider_registry.py +82 -0
- keble_data_infra_api-0.1.0/tests/unit/test_settings.py +101 -0
- keble_data_infra_api-0.1.0/tests/unit/test_video_enrichment_client.py +289 -0
- keble_data_infra_api-0.1.0/tests/unit/test_video_enrichment_redis.py +176 -0
- keble_data_infra_api-0.1.0/tests/unit/test_video_enrichment_router.py +203 -0
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
# Generated by Cargo
|
|
2
|
+
# will have compiled files and executables
|
|
3
|
+
debug/
|
|
4
|
+
target/
|
|
5
|
+
cert/
|
|
6
|
+
|
|
7
|
+
# Remove Cargo.lock from gitignore if creating an executable, leave it for libraries
|
|
8
|
+
# More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html
|
|
9
|
+
Cargo.lock
|
|
10
|
+
|
|
11
|
+
# These are backup files generated by rustfmt
|
|
12
|
+
**/*.rs.bk
|
|
13
|
+
|
|
14
|
+
# MSVC Windows builds of rustc generate these, which store debugging information
|
|
15
|
+
*.pdb
|
|
16
|
+
*__ignored__*
|
|
17
|
+
|
|
18
|
+
*.env
|
|
19
|
+
**/.env.docker
|
|
20
|
+
*.DS_Store
|
|
21
|
+
**/.DS_Store
|
|
22
|
+
|
|
23
|
+
.vscode
|
|
24
|
+
.idea
|
|
25
|
+
|
|
26
|
+
*.crt
|
|
27
|
+
|
|
28
|
+
uploads/
|
|
29
|
+
!strapi-v5/public/uploads/
|
|
30
|
+
!strapi-v5/public/uploads/.gitkeep
|
|
31
|
+
test/
|
|
32
|
+
resources/
|
|
33
|
+
Brewfile
|
|
34
|
+
qdrant/
|
|
35
|
+
*.pyc
|
|
36
|
+
**.pyc.*
|
|
37
|
+
*/traefik/certs
|
|
38
|
+
*/traefik/certs/*
|
|
39
|
+
gcp.json
|
|
40
|
+
.history
|
|
41
|
+
|
|
42
|
+
prod.env.*
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
FROM ghcr.io/astral-sh/uv:python3.13-bookworm-slim AS builder
|
|
2
|
+
|
|
3
|
+
RUN apt-get update
|
|
4
|
+
RUN apt-get install --yes --no-install-recommends git build-essential
|
|
5
|
+
RUN rm -rf /var/lib/apt/lists/*
|
|
6
|
+
|
|
7
|
+
ARG KEBLE_AMZ_REF=b6ce544995cd14c6930bdfdfa809cd5ba10e7833
|
|
8
|
+
ARG KEBLE_TIKTOK_REF=f14a9c670d270a201f2359e608f6b1e2b0493825
|
|
9
|
+
ARG KEBLE_PY_REF=9db27c8c4cac25fcf4993620168c9f77dd95a0f5
|
|
10
|
+
ARG KEBLE_KEEPA_REF=5907fe43d3368fec6e64da4c30e33eddb7b1d885
|
|
11
|
+
|
|
12
|
+
WORKDIR /workspace
|
|
13
|
+
RUN --mount=type=secret,id=gh_token \
|
|
14
|
+
sh -eu -c 'TOKEN="$(cat /run/secrets/gh_token)"; BASIC="$(printf "x-access-token:%s" "${TOKEN}" | base64 -w0)"; git -c http.extraHeader="Authorization: Basic ${BASIC}" clone https://github.com/keble-ai/keble-amz.git data-infra/keble-amz'
|
|
15
|
+
RUN git -C data-infra/keble-amz checkout ${KEBLE_AMZ_REF}
|
|
16
|
+
RUN --mount=type=secret,id=gh_token \
|
|
17
|
+
sh -eu -c 'TOKEN="$(cat /run/secrets/gh_token)"; BASIC="$(printf "x-access-token:%s" "${TOKEN}" | base64 -w0)"; git -c http.extraHeader="Authorization: Basic ${BASIC}" clone https://github.com/keble-ai/keble-tiktok.git data-infra/keble-tiktok'
|
|
18
|
+
RUN git -C data-infra/keble-tiktok checkout ${KEBLE_TIKTOK_REF}
|
|
19
|
+
RUN --mount=type=secret,id=gh_token \
|
|
20
|
+
sh -eu -c 'TOKEN="$(cat /run/secrets/gh_token)"; BASIC="$(printf "x-access-token:%s" "${TOKEN}" | base64 -w0)"; git -c http.extraHeader="Authorization: Basic ${BASIC}" clone https://github.com/keble-ai/keble-py.git deps/keble-py'
|
|
21
|
+
RUN git -C deps/keble-py checkout ${KEBLE_PY_REF}
|
|
22
|
+
RUN --mount=type=secret,id=gh_token \
|
|
23
|
+
sh -eu -c 'TOKEN="$(cat /run/secrets/gh_token)"; BASIC="$(printf "x-access-token:%s" "${TOKEN}" | base64 -w0)"; git -c http.extraHeader="Authorization: Basic ${BASIC}" clone https://github.com/keble-ai/keble-keepa.git deps/keble-keepa'
|
|
24
|
+
RUN git -C deps/keble-keepa checkout ${KEBLE_KEEPA_REF}
|
|
25
|
+
|
|
26
|
+
COPY . /workspace/data-infra/keble-data-infra
|
|
27
|
+
WORKDIR /workspace/data-infra/keble-data-infra
|
|
28
|
+
RUN uv sync --no-editable --no-default-groups --package keble-data-infra-api
|
|
29
|
+
|
|
30
|
+
FROM python:3.13-slim-bookworm AS runtime
|
|
31
|
+
|
|
32
|
+
ENV PATH="/workspace/data-infra/keble-data-infra/.venv/bin:${PATH}" \
|
|
33
|
+
PYTHONDONTWRITEBYTECODE=1 \
|
|
34
|
+
PYTHONUNBUFFERED=1
|
|
35
|
+
|
|
36
|
+
RUN useradd --create-home --uid 10001 keble
|
|
37
|
+
WORKDIR /workspace/data-infra/keble-data-infra
|
|
38
|
+
COPY --from=builder --chown=keble:keble /workspace/data-infra/keble-data-infra/.venv /workspace/data-infra/keble-data-infra/.venv
|
|
39
|
+
|
|
40
|
+
USER keble
|
|
41
|
+
EXPOSE 8000
|
|
42
|
+
HEALTHCHECK --interval=15s --timeout=5s --start-period=15s --retries=4 \
|
|
43
|
+
CMD ["python", "-c", "from urllib.request import urlopen; urlopen('http://127.0.0.1:8000/ready', timeout=3).read()"]
|
|
44
|
+
|
|
45
|
+
CMD ["uvicorn", "keble_data_infra_api.main:app", "--host", "0.0.0.0", "--port", "8000"]
|
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: keble-data-infra-api
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Sole FastAPI composition root for Keble raw provider data.
|
|
5
|
+
Author-email: zhenhao-ma <bob0103779@gmail.com>
|
|
6
|
+
Requires-Python: <3.14,>=3.13
|
|
7
|
+
Requires-Dist: aiohttp<4,>=3
|
|
8
|
+
Requires-Dist: fastapi<1,>=0.110
|
|
9
|
+
Requires-Dist: httpx<1,>=0.28
|
|
10
|
+
Requires-Dist: keble-amz<3,>=2.0.0
|
|
11
|
+
Requires-Dist: keble-data-infra-contract<1,>=0.1.0
|
|
12
|
+
Requires-Dist: keble-db<2,>=1.9.0
|
|
13
|
+
Requires-Dist: keble-helpers<2,>=1.12.2
|
|
14
|
+
Requires-Dist: keble-keepa<2,>=1.3.0
|
|
15
|
+
Requires-Dist: keble-tiktok<1,>=0.5.0
|
|
16
|
+
Requires-Dist: motor<4,>=3
|
|
17
|
+
Requires-Dist: pymongo<5,>=4.10
|
|
18
|
+
Requires-Dist: redis<7,>=5
|
|
19
|
+
Requires-Dist: sentry-sdk<3,>=2
|
|
20
|
+
Requires-Dist: uvicorn<1,>=0.29
|
|
21
|
+
Description-Content-Type: text/markdown
|
|
22
|
+
|
|
23
|
+
# keble-data-infra-api
|
|
24
|
+
|
|
25
|
+
Install the composed raw-data API with
|
|
26
|
+
`pip install "keble-data-infra-api>=0.1.0,<1"`. The published distribution
|
|
27
|
+
resolves the released Amazon and TikTok providers; editable local sources in
|
|
28
|
+
the uv workspace are development-only.
|
|
29
|
+
|
|
30
|
+
The sole FastAPI composition root for raw Amazon and TikTok provider data.
|
|
31
|
+
Provider packages export framework-free services and manifests; this package
|
|
32
|
+
owns route generation, exception mapping, runtime resources, and ingestion.
|
|
33
|
+
|
|
34
|
+
The Stand Out platform consumes this service over HTTP and does not import its
|
|
35
|
+
contracts or provider packages.
|
|
36
|
+
|
|
37
|
+
## Local provider environment
|
|
38
|
+
|
|
39
|
+
Build an ignored API-local `.env` from the existing Keble developer env files.
|
|
40
|
+
The helper copies only the required named settings and prints key names, never
|
|
41
|
+
credential values:
|
|
42
|
+
|
|
43
|
+
```bash
|
|
44
|
+
python scripts/build_local_env.py
|
|
45
|
+
uv run --env-file .env pytest -q tests/live -m live
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
The live suite uses independent `RUN_KEEPA_LIVE`, `RUN_FASTMOSS_LIVE`, and
|
|
49
|
+
`RUN_LIVE_ECHOTIK` gates. A gate is written only when its complete credential
|
|
50
|
+
set exists. Each configured provider runs as a separate canary, so one missing
|
|
51
|
+
credential cannot hide another provider's result. Provider failures are test
|
|
52
|
+
failures; missing opt-in credentials are the only valid reason to skip. Live
|
|
53
|
+
providers honor `PROVIDER_HTTP_TRUST_ENV` through `ApiSettings` instead of
|
|
54
|
+
hard-coding direct network access, matching the production runtime boundary.
|
|
55
|
+
The pinned TikTok provider includes HTTPX's SOCKS transport, so production
|
|
56
|
+
SOCKS4/5 proxy settings remain executable rather than failing during client
|
|
57
|
+
construction. The image also pins the timezone-aware/warning-free keble-db
|
|
58
|
+
checkpoint used by the local-full persistence proof.
|
|
59
|
+
|
|
60
|
+
## Runtime ownership
|
|
61
|
+
|
|
62
|
+
`keble_data_infra_api.main:create_app` constructs the only raw FastAPI app.
|
|
63
|
+
`api.router:create_router` builds discovery and provider routes from the
|
|
64
|
+
registry, and `api.error_handlers` performs typed exception conversion. The
|
|
65
|
+
Stand Out adapter resolves the single product-search capability through
|
|
66
|
+
`GET /v1/{channel}/providers` and then calls its generated operation. The old
|
|
67
|
+
`/v1/commerce/search` route and compatibility DTO snapshot are deleted.
|
|
68
|
+
|
|
69
|
+
API settings and immutable application values reuse
|
|
70
|
+
`keble-helpers.PydanticModelConfig` or the contract-owned `ContractModel`.
|
|
71
|
+
Durable aggregates use `SchemaBase`/`MongoObjectBase`; do not introduce a local
|
|
72
|
+
`ConfigDict` or direct persistence `BaseModel` policy.
|
|
73
|
+
|
|
74
|
+
Mongo/Redis settings enable the API-owned provider infrastructure. The local
|
|
75
|
+
env builder copies only `MONGO_DB_URI`, `REDIS_URI`, and named provider
|
|
76
|
+
credentials; API-owned database/collection/namespace defaults stay isolated.
|
|
77
|
+
Startup creates cache indexes before provider handlers run and shutdown closes
|
|
78
|
+
all sync/async Mongo and Redis clients.
|
|
79
|
+
|
|
80
|
+
Video Enrichment projection is also API-owned and requires Mongo whenever its
|
|
81
|
+
Result origin is configured. One document stores the source event, owner-token
|
|
82
|
+
claim, attempts, failure state, and optional projection. Unique event/result
|
|
83
|
+
indexes plus expiring claims make concurrent consumers safe; Redis entries are
|
|
84
|
+
acknowledged only after the document atomically reaches `completed`. The old
|
|
85
|
+
replica-local JSON state file is not supported.
|
|
86
|
+
|
|
87
|
+
The same typed gateway centralizes submission and job/result correlation:
|
|
88
|
+
|
|
89
|
+
```text
|
|
90
|
+
POST /internal/video-enrichments:submit
|
|
91
|
+
GET /internal/video-enrichment-jobs/{job_id}
|
|
92
|
+
GET /internal/video-enrichment-jobs/{job_id}/projection
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
The projection lookup uses the API-owned job/status/completion-time compound
|
|
96
|
+
index. Consumers never call the independent media service or infer `result_id`
|
|
97
|
+
from provider storage. A projection 404 means durable completion ingestion is
|
|
98
|
+
still pending and is safe for a bounded worker retry.
|
|
99
|
+
|
|
100
|
+
The submission gateway forwards the expected media SHA-256 as a dedicated
|
|
101
|
+
integrity field. When an authorized provider transcript exists, the contract binds
|
|
102
|
+
its normalized content checksum, source reference, and source media checksum before
|
|
103
|
+
the independent service may skip ASR. The API never treats a provider title or video
|
|
104
|
+
description as spoken captions.
|
|
105
|
+
|
|
106
|
+
The producer's Job/Result payload contains additional stage, artifact, usage,
|
|
107
|
+
frame, warning, and error fields. Data Infra validates the typed stable subset
|
|
108
|
+
through the contract's source-projection inheritance and ignores only those
|
|
109
|
+
additive producer fields. All submission and public normalized models retain
|
|
110
|
+
`extra="forbid"`; compatibility does not weaken command validation.
|
|
111
|
+
|
|
112
|
+
Configured provider routes include:
|
|
113
|
+
|
|
114
|
+
```text
|
|
115
|
+
POST /v1/amazon/providers/keepa/products/lookup
|
|
116
|
+
POST /v1/amazon/providers/keepa/products/search
|
|
117
|
+
POST /v1/amazon/providers/keepa/products/rank
|
|
118
|
+
POST /v1/tiktok/providers/fastmoss/products/search
|
|
119
|
+
POST /v1/tiktok/providers/echotik/products/rank
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
Run the isolated all-provider generated-route canary with:
|
|
123
|
+
|
|
124
|
+
```bash
|
|
125
|
+
python scripts/build_local_env.py
|
|
126
|
+
uv run --env-file .env pytest -q tests/live/test_provider_routes_live.py -m live
|
|
127
|
+
```
|
|
128
|
+
|
|
129
|
+
## Container and observability
|
|
130
|
+
|
|
131
|
+
The multi-stage image installs exact pushed provider commits and receives the
|
|
132
|
+
GitHub token only as a BuildKit secret. It runs as UID 10001, exposes `/health`
|
|
133
|
+
for liveness and `/ready` for configured Mongo/Redis readiness, and closes all
|
|
134
|
+
runtime clients through FastAPI lifespan.
|
|
135
|
+
|
|
136
|
+
Build the credential-preserving Docker env and start the profile on the existing
|
|
137
|
+
`keble_default` network:
|
|
138
|
+
|
|
139
|
+
```bash
|
|
140
|
+
uv run --directory keble-data-infra-api python scripts/build_local_env.py
|
|
141
|
+
uv run --directory keble-data-infra-api python -m scripts.build_local_docker_env
|
|
142
|
+
GH_TOKEN="$(gh auth token)" docker compose \
|
|
143
|
+
--env-file keble-data-infra-api/.env.docker \
|
|
144
|
+
-f deploy/compose.raw-api.yaml --profile raw-api up --build
|
|
145
|
+
```
|
|
146
|
+
|
|
147
|
+
`.env`, `.env.docker`, and the BuildKit secret stay outside Git and image
|
|
148
|
+
layers. Set `SENTRY_DSN`, `SENTRY_ENVIRONMENT`, and
|
|
149
|
+
`SENTRY_TRACES_SAMPLE_RATE` in the source env to enable error/performance
|
|
150
|
+
telemetry. Registry discovery, generated operations, cache outcomes, lease
|
|
151
|
+
contention, upstream work, batches, and video ingestion emit structural spans;
|
|
152
|
+
request/response content and credentials are not attached.
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
# keble-data-infra-api
|
|
2
|
+
|
|
3
|
+
Install the composed raw-data API with
|
|
4
|
+
`pip install "keble-data-infra-api>=0.1.0,<1"`. The published distribution
|
|
5
|
+
resolves the released Amazon and TikTok providers; editable local sources in
|
|
6
|
+
the uv workspace are development-only.
|
|
7
|
+
|
|
8
|
+
The sole FastAPI composition root for raw Amazon and TikTok provider data.
|
|
9
|
+
Provider packages export framework-free services and manifests; this package
|
|
10
|
+
owns route generation, exception mapping, runtime resources, and ingestion.
|
|
11
|
+
|
|
12
|
+
The Stand Out platform consumes this service over HTTP and does not import its
|
|
13
|
+
contracts or provider packages.
|
|
14
|
+
|
|
15
|
+
## Local provider environment
|
|
16
|
+
|
|
17
|
+
Build an ignored API-local `.env` from the existing Keble developer env files.
|
|
18
|
+
The helper copies only the required named settings and prints key names, never
|
|
19
|
+
credential values:
|
|
20
|
+
|
|
21
|
+
```bash
|
|
22
|
+
python scripts/build_local_env.py
|
|
23
|
+
uv run --env-file .env pytest -q tests/live -m live
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
The live suite uses independent `RUN_KEEPA_LIVE`, `RUN_FASTMOSS_LIVE`, and
|
|
27
|
+
`RUN_LIVE_ECHOTIK` gates. A gate is written only when its complete credential
|
|
28
|
+
set exists. Each configured provider runs as a separate canary, so one missing
|
|
29
|
+
credential cannot hide another provider's result. Provider failures are test
|
|
30
|
+
failures; missing opt-in credentials are the only valid reason to skip. Live
|
|
31
|
+
providers honor `PROVIDER_HTTP_TRUST_ENV` through `ApiSettings` instead of
|
|
32
|
+
hard-coding direct network access, matching the production runtime boundary.
|
|
33
|
+
The pinned TikTok provider includes HTTPX's SOCKS transport, so production
|
|
34
|
+
SOCKS4/5 proxy settings remain executable rather than failing during client
|
|
35
|
+
construction. The image also pins the timezone-aware/warning-free keble-db
|
|
36
|
+
checkpoint used by the local-full persistence proof.
|
|
37
|
+
|
|
38
|
+
## Runtime ownership
|
|
39
|
+
|
|
40
|
+
`keble_data_infra_api.main:create_app` constructs the only raw FastAPI app.
|
|
41
|
+
`api.router:create_router` builds discovery and provider routes from the
|
|
42
|
+
registry, and `api.error_handlers` performs typed exception conversion. The
|
|
43
|
+
Stand Out adapter resolves the single product-search capability through
|
|
44
|
+
`GET /v1/{channel}/providers` and then calls its generated operation. The old
|
|
45
|
+
`/v1/commerce/search` route and compatibility DTO snapshot are deleted.
|
|
46
|
+
|
|
47
|
+
API settings and immutable application values reuse
|
|
48
|
+
`keble-helpers.PydanticModelConfig` or the contract-owned `ContractModel`.
|
|
49
|
+
Durable aggregates use `SchemaBase`/`MongoObjectBase`; do not introduce a local
|
|
50
|
+
`ConfigDict` or direct persistence `BaseModel` policy.
|
|
51
|
+
|
|
52
|
+
Mongo/Redis settings enable the API-owned provider infrastructure. The local
|
|
53
|
+
env builder copies only `MONGO_DB_URI`, `REDIS_URI`, and named provider
|
|
54
|
+
credentials; API-owned database/collection/namespace defaults stay isolated.
|
|
55
|
+
Startup creates cache indexes before provider handlers run and shutdown closes
|
|
56
|
+
all sync/async Mongo and Redis clients.
|
|
57
|
+
|
|
58
|
+
Video Enrichment projection is also API-owned and requires Mongo whenever its
|
|
59
|
+
Result origin is configured. One document stores the source event, owner-token
|
|
60
|
+
claim, attempts, failure state, and optional projection. Unique event/result
|
|
61
|
+
indexes plus expiring claims make concurrent consumers safe; Redis entries are
|
|
62
|
+
acknowledged only after the document atomically reaches `completed`. The old
|
|
63
|
+
replica-local JSON state file is not supported.
|
|
64
|
+
|
|
65
|
+
The same typed gateway centralizes submission and job/result correlation:
|
|
66
|
+
|
|
67
|
+
```text
|
|
68
|
+
POST /internal/video-enrichments:submit
|
|
69
|
+
GET /internal/video-enrichment-jobs/{job_id}
|
|
70
|
+
GET /internal/video-enrichment-jobs/{job_id}/projection
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
The projection lookup uses the API-owned job/status/completion-time compound
|
|
74
|
+
index. Consumers never call the independent media service or infer `result_id`
|
|
75
|
+
from provider storage. A projection 404 means durable completion ingestion is
|
|
76
|
+
still pending and is safe for a bounded worker retry.
|
|
77
|
+
|
|
78
|
+
The submission gateway forwards the expected media SHA-256 as a dedicated
|
|
79
|
+
integrity field. When an authorized provider transcript exists, the contract binds
|
|
80
|
+
its normalized content checksum, source reference, and source media checksum before
|
|
81
|
+
the independent service may skip ASR. The API never treats a provider title or video
|
|
82
|
+
description as spoken captions.
|
|
83
|
+
|
|
84
|
+
The producer's Job/Result payload contains additional stage, artifact, usage,
|
|
85
|
+
frame, warning, and error fields. Data Infra validates the typed stable subset
|
|
86
|
+
through the contract's source-projection inheritance and ignores only those
|
|
87
|
+
additive producer fields. All submission and public normalized models retain
|
|
88
|
+
`extra="forbid"`; compatibility does not weaken command validation.
|
|
89
|
+
|
|
90
|
+
Configured provider routes include:
|
|
91
|
+
|
|
92
|
+
```text
|
|
93
|
+
POST /v1/amazon/providers/keepa/products/lookup
|
|
94
|
+
POST /v1/amazon/providers/keepa/products/search
|
|
95
|
+
POST /v1/amazon/providers/keepa/products/rank
|
|
96
|
+
POST /v1/tiktok/providers/fastmoss/products/search
|
|
97
|
+
POST /v1/tiktok/providers/echotik/products/rank
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
Run the isolated all-provider generated-route canary with:
|
|
101
|
+
|
|
102
|
+
```bash
|
|
103
|
+
python scripts/build_local_env.py
|
|
104
|
+
uv run --env-file .env pytest -q tests/live/test_provider_routes_live.py -m live
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
## Container and observability
|
|
108
|
+
|
|
109
|
+
The multi-stage image installs exact pushed provider commits and receives the
|
|
110
|
+
GitHub token only as a BuildKit secret. It runs as UID 10001, exposes `/health`
|
|
111
|
+
for liveness and `/ready` for configured Mongo/Redis readiness, and closes all
|
|
112
|
+
runtime clients through FastAPI lifespan.
|
|
113
|
+
|
|
114
|
+
Build the credential-preserving Docker env and start the profile on the existing
|
|
115
|
+
`keble_default` network:
|
|
116
|
+
|
|
117
|
+
```bash
|
|
118
|
+
uv run --directory keble-data-infra-api python scripts/build_local_env.py
|
|
119
|
+
uv run --directory keble-data-infra-api python -m scripts.build_local_docker_env
|
|
120
|
+
GH_TOKEN="$(gh auth token)" docker compose \
|
|
121
|
+
--env-file keble-data-infra-api/.env.docker \
|
|
122
|
+
-f deploy/compose.raw-api.yaml --profile raw-api up --build
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
`.env`, `.env.docker`, and the BuildKit secret stay outside Git and image
|
|
126
|
+
layers. Set `SENTRY_DSN`, `SENTRY_ENVIRONMENT`, and
|
|
127
|
+
`SENTRY_TRACES_SAMPLE_RATE` in the source env to enable error/performance
|
|
128
|
+
telemetry. Registry discovery, generated operations, cache outcomes, lease
|
|
129
|
+
contention, upstream work, batches, and video ingestion emit structural spans;
|
|
130
|
+
request/response content and credentials are not attached.
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "keble-data-infra-api"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Sole FastAPI composition root for Keble raw provider data."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.13,<3.14"
|
|
11
|
+
dependencies = [
|
|
12
|
+
"aiohttp>=3,<4",
|
|
13
|
+
"fastapi>=0.110,<1",
|
|
14
|
+
"httpx>=0.28,<1",
|
|
15
|
+
"keble-amz>=2.0.0,<3",
|
|
16
|
+
"keble-data-infra-contract>=0.1.0,<1",
|
|
17
|
+
"keble-db>=1.9.0,<2",
|
|
18
|
+
"keble-helpers>=1.12.2,<2",
|
|
19
|
+
"keble-keepa>=1.3.0,<2",
|
|
20
|
+
"keble-tiktok>=0.5.0,<1",
|
|
21
|
+
"motor>=3,<4",
|
|
22
|
+
"pymongo>=4.10,<5",
|
|
23
|
+
"redis>=5,<7",
|
|
24
|
+
"sentry-sdk>=2,<3",
|
|
25
|
+
"uvicorn>=0.29,<1",
|
|
26
|
+
]
|
|
27
|
+
|
|
28
|
+
[[project.authors]]
|
|
29
|
+
name = "zhenhao-ma"
|
|
30
|
+
email = "bob0103779@gmail.com"
|
|
31
|
+
|
|
32
|
+
[dependency-groups]
|
|
33
|
+
test = [
|
|
34
|
+
"pytest>=8.2,<9",
|
|
35
|
+
"pytest-asyncio>=1.4,<2",
|
|
36
|
+
"httpx>=0.28,<1",
|
|
37
|
+
"packaging>=24,<27",
|
|
38
|
+
]
|
|
39
|
+
|
|
40
|
+
[tool.uv]
|
|
41
|
+
default-groups = ["test"]
|
|
42
|
+
|
|
43
|
+
[tool.uv.sources]
|
|
44
|
+
keble-data-infra-contract = {workspace = true}
|
|
45
|
+
keble-amz = {path = "../../keble-amz"}
|
|
46
|
+
keble-db = {path = "../../../deps/keble-py/keble-db"}
|
|
47
|
+
keble-helpers = {path = "../../../deps/keble-py/keble-helpers"}
|
|
48
|
+
keble-keepa = {path = "../../../deps/keble-keepa"}
|
|
49
|
+
keble-tiktok = {path = "../../keble-tiktok"}
|
|
50
|
+
|
|
51
|
+
[tool.pytest.ini_options]
|
|
52
|
+
addopts = "--strict-markers"
|
|
53
|
+
testpaths = ["tests"]
|
|
54
|
+
asyncio_mode = "auto"
|
|
55
|
+
markers = [
|
|
56
|
+
"unit: fast deterministic tests with no network",
|
|
57
|
+
"contract: recorded or fixture-based external contract tests",
|
|
58
|
+
"integration: real local dependency or adapter tests",
|
|
59
|
+
"live: real external service calls",
|
|
60
|
+
"slow: slow tests",
|
|
61
|
+
"db: any real database test",
|
|
62
|
+
"mongo: MongoDB tests",
|
|
63
|
+
"redis: Redis tests",
|
|
64
|
+
"db_stack: multi-database integration tests",
|
|
65
|
+
"container: tests requiring throwaway containers",
|
|
66
|
+
"local_stack: tests requiring existing local Keble docker compose stack",
|
|
67
|
+
]
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Developer and release utilities for the raw API distribution."""
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
"""Create an ignored container-network env from the API's ignored local env."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from urllib.parse import urlsplit, urlunsplit
|
|
8
|
+
|
|
9
|
+
from .build_local_env import _read_env
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def _replace_endpoint(*, value: str, hostname: str, port: int) -> str:
|
|
13
|
+
"""Replace only URI host/port while retaining credentials, path and options."""
|
|
14
|
+
|
|
15
|
+
normalized = value.strip().strip("'\"")
|
|
16
|
+
parsed = urlsplit(normalized)
|
|
17
|
+
if parsed.hostname is None:
|
|
18
|
+
raise ValueError("database URI must include a hostname")
|
|
19
|
+
old_endpoint = (
|
|
20
|
+
parsed.hostname
|
|
21
|
+
if parsed.port is None
|
|
22
|
+
else f"{parsed.hostname}:{parsed.port}"
|
|
23
|
+
)
|
|
24
|
+
netloc = parsed.netloc.replace(old_endpoint, f"{hostname}:{port}")
|
|
25
|
+
return urlunsplit((parsed.scheme, netloc, parsed.path, parsed.query, parsed.fragment))
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def build_local_docker_env(*, source: Path, output: Path) -> tuple[str, ...]:
|
|
29
|
+
"""Retain provider settings and map DB endpoints to existing Docker aliases.
|
|
30
|
+
|
|
31
|
+
Side effects if changes:
|
|
32
|
+
- `deploy/compose.raw-api.yaml` connects to `keble_default` Mongo/Redis;
|
|
33
|
+
- URI credentials and query options survive without being printed;
|
|
34
|
+
- the ignored output must never enter the Docker build context or Git.
|
|
35
|
+
"""
|
|
36
|
+
|
|
37
|
+
assignments = _read_env(source)
|
|
38
|
+
mongo_uri = assignments.get("MONGO_DB_URI")
|
|
39
|
+
redis_uri = assignments.get("REDIS_URI")
|
|
40
|
+
if not mongo_uri or not redis_uri:
|
|
41
|
+
raise ValueError("local API env requires MONGO_DB_URI and REDIS_URI")
|
|
42
|
+
assignments["DATA_INFRA_MONGO_URI"] = _replace_endpoint(
|
|
43
|
+
value=mongo_uri,
|
|
44
|
+
hostname="mongodb",
|
|
45
|
+
port=27017,
|
|
46
|
+
)
|
|
47
|
+
assignments["DATA_INFRA_REDIS_URI"] = _replace_endpoint(
|
|
48
|
+
value=redis_uri,
|
|
49
|
+
hostname="redis",
|
|
50
|
+
port=6379,
|
|
51
|
+
)
|
|
52
|
+
output.parent.mkdir(parents=True, exist_ok=True)
|
|
53
|
+
temporary = output.with_suffix(f"{output.suffix}.tmp")
|
|
54
|
+
temporary.write_text(
|
|
55
|
+
"".join(f"{name}={value}\n" for name, value in sorted(assignments.items())),
|
|
56
|
+
encoding="utf-8",
|
|
57
|
+
)
|
|
58
|
+
temporary.replace(output)
|
|
59
|
+
return tuple(sorted(assignments))
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def main() -> None:
|
|
63
|
+
"""Build `.env.docker` beside the existing ignored API environment."""
|
|
64
|
+
|
|
65
|
+
api_root = Path(__file__).resolve().parents[1]
|
|
66
|
+
parser = argparse.ArgumentParser(description=__doc__)
|
|
67
|
+
parser.add_argument("--source", type=Path, default=api_root / ".env")
|
|
68
|
+
parser.add_argument("--output", type=Path, default=api_root / ".env.docker")
|
|
69
|
+
arguments = parser.parse_args()
|
|
70
|
+
written = build_local_docker_env(
|
|
71
|
+
source=arguments.source,
|
|
72
|
+
output=arguments.output,
|
|
73
|
+
)
|
|
74
|
+
print("Wrote Docker settings: " + ", ".join(written))
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
if __name__ == "__main__":
|
|
78
|
+
main()
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
"""Build an ignored local API environment from existing Keble provider env files."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
REQUIRED_PROVIDER_KEYS = (
|
|
10
|
+
"KEEPA_API_TOKEN",
|
|
11
|
+
"FASTMOSS_BASE_URL",
|
|
12
|
+
"FASTMOSS_CLIENT_SECRET",
|
|
13
|
+
"ECHOTIK_BASE_URL",
|
|
14
|
+
"ECHOTIK_USERNAME",
|
|
15
|
+
"ECHOTIK_PASSWORD",
|
|
16
|
+
"MONGO_DB_URI",
|
|
17
|
+
"REDIS_URI",
|
|
18
|
+
"SENTRY_DSN",
|
|
19
|
+
"SENTRY_ENVIRONMENT",
|
|
20
|
+
"SENTRY_TRACES_SAMPLE_RATE",
|
|
21
|
+
)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _parser(*, umbrella_root: Path) -> argparse.ArgumentParser:
|
|
25
|
+
"""Build explicit source and destination arguments for secret-safe env assembly.
|
|
26
|
+
|
|
27
|
+
Side effects if changes:
|
|
28
|
+
- local Keepa, FastMoss and EchoTik live-test setup;
|
|
29
|
+
- developer commands documented by ``keble-data-infra-api/README.md``.
|
|
30
|
+
"""
|
|
31
|
+
|
|
32
|
+
parser = argparse.ArgumentParser(description=__doc__)
|
|
33
|
+
parser.add_argument(
|
|
34
|
+
"--backend-env",
|
|
35
|
+
type=Path,
|
|
36
|
+
default=umbrella_root / "keble.backend" / ".env",
|
|
37
|
+
)
|
|
38
|
+
parser.add_argument(
|
|
39
|
+
"--tiktok-env",
|
|
40
|
+
type=Path,
|
|
41
|
+
default=umbrella_root / "data-infra" / "keble-tiktok" / ".env",
|
|
42
|
+
)
|
|
43
|
+
parser.add_argument(
|
|
44
|
+
"--output",
|
|
45
|
+
type=Path,
|
|
46
|
+
default=Path(__file__).resolve().parents[1] / ".env",
|
|
47
|
+
)
|
|
48
|
+
return parser
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _read_env(path: Path) -> dict[str, str]:
|
|
52
|
+
"""Read dotenv assignments without evaluating shell syntax or exposing values.
|
|
53
|
+
|
|
54
|
+
Side effects if changes:
|
|
55
|
+
- which existing local settings can enter the generated raw-API environment;
|
|
56
|
+
- whether quoted provider credentials retain their original dotenv encoding.
|
|
57
|
+
"""
|
|
58
|
+
|
|
59
|
+
assignments: dict[str, str] = {}
|
|
60
|
+
for line in path.read_text(encoding="utf-8").splitlines():
|
|
61
|
+
candidate = line.strip()
|
|
62
|
+
if not candidate or candidate.startswith("#") or "=" not in candidate:
|
|
63
|
+
continue
|
|
64
|
+
name, separator, value = candidate.partition("=")
|
|
65
|
+
if separator and name.strip() in REQUIRED_PROVIDER_KEYS:
|
|
66
|
+
assignments[name.strip()] = value.strip()
|
|
67
|
+
return assignments
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def _has_value(assignments: dict[str, str], name: str) -> bool:
|
|
71
|
+
"""Report whether one copied dotenv assignment contains a non-empty value."""
|
|
72
|
+
|
|
73
|
+
value = assignments.get(name, "").strip()
|
|
74
|
+
return bool(value.strip("'\""))
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def build_local_env(*, backend_env: Path, tiktok_env: Path, output: Path) -> tuple[str, ...]:
|
|
78
|
+
"""Copy only named provider settings and enable gates with complete credentials.
|
|
79
|
+
|
|
80
|
+
Steps:
|
|
81
|
+
1. parse the two existing local env files as inert dotenv text;
|
|
82
|
+
2. select the six provider variables owned by this API;
|
|
83
|
+
3. enable a live gate only when its complete credential set exists;
|
|
84
|
+
4. atomically replace the ignored destination and return key names only.
|
|
85
|
+
|
|
86
|
+
Side effects if changes:
|
|
87
|
+
- API-local provider credentials and live-test opt-in gates;
|
|
88
|
+
- no source env file is modified and no secret value is printed.
|
|
89
|
+
"""
|
|
90
|
+
|
|
91
|
+
assignments = {**_read_env(backend_env), **_read_env(tiktok_env)}
|
|
92
|
+
gates = {
|
|
93
|
+
"RUN_KEEPA_LIVE": _has_value(assignments, "KEEPA_API_TOKEN"),
|
|
94
|
+
"RUN_FASTMOSS_LIVE": all(
|
|
95
|
+
_has_value(assignments, name)
|
|
96
|
+
for name in ("FASTMOSS_BASE_URL", "FASTMOSS_CLIENT_SECRET")
|
|
97
|
+
),
|
|
98
|
+
"RUN_LIVE_ECHOTIK": all(
|
|
99
|
+
_has_value(assignments, name)
|
|
100
|
+
for name in ("ECHOTIK_BASE_URL", "ECHOTIK_USERNAME", "ECHOTIK_PASSWORD")
|
|
101
|
+
),
|
|
102
|
+
}
|
|
103
|
+
selected = {
|
|
104
|
+
name: assignments[name]
|
|
105
|
+
for name in REQUIRED_PROVIDER_KEYS
|
|
106
|
+
if _has_value(assignments, name)
|
|
107
|
+
}
|
|
108
|
+
selected.update({name: "1" for name, enabled in gates.items() if enabled})
|
|
109
|
+
|
|
110
|
+
output.parent.mkdir(parents=True, exist_ok=True)
|
|
111
|
+
temporary_output = output.with_suffix(f"{output.suffix}.tmp")
|
|
112
|
+
temporary_output.write_text(
|
|
113
|
+
"".join(f"{name}={value}\n" for name, value in sorted(selected.items())),
|
|
114
|
+
encoding="utf-8",
|
|
115
|
+
)
|
|
116
|
+
temporary_output.replace(output)
|
|
117
|
+
return tuple(sorted(selected))
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def main() -> None:
|
|
121
|
+
"""Resolve workspace defaults, build the env, and print key names only."""
|
|
122
|
+
|
|
123
|
+
umbrella_root = Path(__file__).resolve().parents[4]
|
|
124
|
+
arguments = _parser(umbrella_root=umbrella_root).parse_args()
|
|
125
|
+
written_keys = build_local_env(
|
|
126
|
+
backend_env=arguments.backend_env,
|
|
127
|
+
tiktok_env=arguments.tiktok_env,
|
|
128
|
+
output=arguments.output,
|
|
129
|
+
)
|
|
130
|
+
print("Wrote provider settings: " + ", ".join(written_keys))
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
if __name__ == "__main__":
|
|
134
|
+
main()
|