keble-data-infra-api 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. keble_data_infra_api-0.1.0/.gitignore +42 -0
  2. keble_data_infra_api-0.1.0/Dockerfile +45 -0
  3. keble_data_infra_api-0.1.0/PKG-INFO +152 -0
  4. keble_data_infra_api-0.1.0/README.md +130 -0
  5. keble_data_infra_api-0.1.0/pyproject.toml +67 -0
  6. keble_data_infra_api-0.1.0/scripts/__init__.py +1 -0
  7. keble_data_infra_api-0.1.0/scripts/build_local_docker_env.py +78 -0
  8. keble_data_infra_api-0.1.0/scripts/build_local_env.py +134 -0
  9. keble_data_infra_api-0.1.0/scripts/export_openapi.py +58 -0
  10. keble_data_infra_api-0.1.0/src/keble_data_infra_api/__init__.py +2 -0
  11. keble_data_infra_api-0.1.0/src/keble_data_infra_api/api/__init__.py +2 -0
  12. keble_data_infra_api-0.1.0/src/keble_data_infra_api/api/error_handlers.py +126 -0
  13. keble_data_infra_api-0.1.0/src/keble_data_infra_api/api/router.py +177 -0
  14. keble_data_infra_api-0.1.0/src/keble_data_infra_api/api/video_enrichment_router.py +169 -0
  15. keble_data_infra_api-0.1.0/src/keble_data_infra_api/integrations/__init__.py +2 -0
  16. keble_data_infra_api-0.1.0/src/keble_data_infra_api/integrations/video_enrichment/__init__.py +2 -0
  17. keble_data_infra_api-0.1.0/src/keble_data_infra_api/integrations/video_enrichment/service.py +452 -0
  18. keble_data_infra_api-0.1.0/src/keble_data_infra_api/main.py +66 -0
  19. keble_data_infra_api-0.1.0/src/keble_data_infra_api/runtime/__init__.py +1 -0
  20. keble_data_infra_api-0.1.0/src/keble_data_infra_api/runtime/bootstrap.py +347 -0
  21. keble_data_infra_api-0.1.0/src/keble_data_infra_api/runtime/observability.py +24 -0
  22. keble_data_infra_api-0.1.0/src/keble_data_infra_api/runtime/provider_registry.py +78 -0
  23. keble_data_infra_api-0.1.0/src/keble_data_infra_api/runtime/settings.py +216 -0
  24. keble_data_infra_api-0.1.0/src/keble_data_infra_api/runtime/video_enrichment_client.py +275 -0
  25. keble_data_infra_api-0.1.0/src/keble_data_infra_api/runtime/video_enrichment_redis.py +202 -0
  26. keble_data_infra_api-0.1.0/src/keble_data_infra_api/runtime/video_enrichment_store.py +507 -0
  27. keble_data_infra_api-0.1.0/tests/__init__.py +1 -0
  28. keble_data_infra_api-0.1.0/tests/conftest.py +23 -0
  29. keble_data_infra_api-0.1.0/tests/integration/rest/test_api_app.py +153 -0
  30. keble_data_infra_api-0.1.0/tests/integration/test_provider_runtime.py +94 -0
  31. keble_data_infra_api-0.1.0/tests/integration/test_video_enrichment_store.py +213 -0
  32. keble_data_infra_api-0.1.0/tests/live/test_provider_routes_live.py +123 -0
  33. keble_data_infra_api-0.1.0/tests/live/test_video_enrichment_result_read_live.py +42 -0
  34. keble_data_infra_api-0.1.0/tests/unit/integrations/video_enrichment/test_ingestion_service.py +345 -0
  35. keble_data_infra_api-0.1.0/tests/unit/test_bootstrap.py +64 -0
  36. keble_data_infra_api-0.1.0/tests/unit/test_dependency_direction.py +178 -0
  37. keble_data_infra_api-0.1.0/tests/unit/test_local_env_scripts.py +64 -0
  38. keble_data_infra_api-0.1.0/tests/unit/test_observability.py +49 -0
  39. keble_data_infra_api-0.1.0/tests/unit/test_package_imports.py +21 -0
  40. keble_data_infra_api-0.1.0/tests/unit/test_persistence_inheritance.py +40 -0
  41. keble_data_infra_api-0.1.0/tests/unit/test_provider_registry.py +82 -0
  42. keble_data_infra_api-0.1.0/tests/unit/test_settings.py +101 -0
  43. keble_data_infra_api-0.1.0/tests/unit/test_video_enrichment_client.py +289 -0
  44. keble_data_infra_api-0.1.0/tests/unit/test_video_enrichment_redis.py +176 -0
  45. keble_data_infra_api-0.1.0/tests/unit/test_video_enrichment_router.py +203 -0
@@ -0,0 +1,42 @@
1
+ # Generated by Cargo
2
+ # will have compiled files and executables
3
+ debug/
4
+ target/
5
+ cert/
6
+
7
+ # Remove Cargo.lock from gitignore if creating an executable, leave it for libraries
8
+ # More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html
9
+ Cargo.lock
10
+
11
+ # These are backup files generated by rustfmt
12
+ **/*.rs.bk
13
+
14
+ # MSVC Windows builds of rustc generate these, which store debugging information
15
+ *.pdb
16
+ *__ignored__*
17
+
18
+ *.env
19
+ **/.env.docker
20
+ *.DS_Store
21
+ **/.DS_Store
22
+
23
+ .vscode
24
+ .idea
25
+
26
+ *.crt
27
+
28
+ uploads/
29
+ !strapi-v5/public/uploads/
30
+ !strapi-v5/public/uploads/.gitkeep
31
+ test/
32
+ resources/
33
+ Brewfile
34
+ qdrant/
35
+ *.pyc
36
+ **.pyc.*
37
+ */traefik/certs
38
+ */traefik/certs/*
39
+ gcp.json
40
+ .history
41
+
42
+ prod.env.*
@@ -0,0 +1,45 @@
1
+ FROM ghcr.io/astral-sh/uv:python3.13-bookworm-slim AS builder
2
+
3
+ RUN apt-get update
4
+ RUN apt-get install --yes --no-install-recommends git build-essential
5
+ RUN rm -rf /var/lib/apt/lists/*
6
+
7
+ ARG KEBLE_AMZ_REF=b6ce544995cd14c6930bdfdfa809cd5ba10e7833
8
+ ARG KEBLE_TIKTOK_REF=f14a9c670d270a201f2359e608f6b1e2b0493825
9
+ ARG KEBLE_PY_REF=9db27c8c4cac25fcf4993620168c9f77dd95a0f5
10
+ ARG KEBLE_KEEPA_REF=5907fe43d3368fec6e64da4c30e33eddb7b1d885
11
+
12
+ WORKDIR /workspace
13
+ RUN --mount=type=secret,id=gh_token \
14
+ sh -eu -c 'TOKEN="$(cat /run/secrets/gh_token)"; BASIC="$(printf "x-access-token:%s" "${TOKEN}" | base64 -w0)"; git -c http.extraHeader="Authorization: Basic ${BASIC}" clone https://github.com/keble-ai/keble-amz.git data-infra/keble-amz'
15
+ RUN git -C data-infra/keble-amz checkout ${KEBLE_AMZ_REF}
16
+ RUN --mount=type=secret,id=gh_token \
17
+ sh -eu -c 'TOKEN="$(cat /run/secrets/gh_token)"; BASIC="$(printf "x-access-token:%s" "${TOKEN}" | base64 -w0)"; git -c http.extraHeader="Authorization: Basic ${BASIC}" clone https://github.com/keble-ai/keble-tiktok.git data-infra/keble-tiktok'
18
+ RUN git -C data-infra/keble-tiktok checkout ${KEBLE_TIKTOK_REF}
19
+ RUN --mount=type=secret,id=gh_token \
20
+ sh -eu -c 'TOKEN="$(cat /run/secrets/gh_token)"; BASIC="$(printf "x-access-token:%s" "${TOKEN}" | base64 -w0)"; git -c http.extraHeader="Authorization: Basic ${BASIC}" clone https://github.com/keble-ai/keble-py.git deps/keble-py'
21
+ RUN git -C deps/keble-py checkout ${KEBLE_PY_REF}
22
+ RUN --mount=type=secret,id=gh_token \
23
+ sh -eu -c 'TOKEN="$(cat /run/secrets/gh_token)"; BASIC="$(printf "x-access-token:%s" "${TOKEN}" | base64 -w0)"; git -c http.extraHeader="Authorization: Basic ${BASIC}" clone https://github.com/keble-ai/keble-keepa.git deps/keble-keepa'
24
+ RUN git -C deps/keble-keepa checkout ${KEBLE_KEEPA_REF}
25
+
26
+ COPY . /workspace/data-infra/keble-data-infra
27
+ WORKDIR /workspace/data-infra/keble-data-infra
28
+ RUN uv sync --no-editable --no-default-groups --package keble-data-infra-api
29
+
30
+ FROM python:3.13-slim-bookworm AS runtime
31
+
32
+ ENV PATH="/workspace/data-infra/keble-data-infra/.venv/bin:${PATH}" \
33
+ PYTHONDONTWRITEBYTECODE=1 \
34
+ PYTHONUNBUFFERED=1
35
+
36
+ RUN useradd --create-home --uid 10001 keble
37
+ WORKDIR /workspace/data-infra/keble-data-infra
38
+ COPY --from=builder --chown=keble:keble /workspace/data-infra/keble-data-infra/.venv /workspace/data-infra/keble-data-infra/.venv
39
+
40
+ USER keble
41
+ EXPOSE 8000
42
+ HEALTHCHECK --interval=15s --timeout=5s --start-period=15s --retries=4 \
43
+ CMD ["python", "-c", "from urllib.request import urlopen; urlopen('http://127.0.0.1:8000/ready', timeout=3).read()"]
44
+
45
+ CMD ["uvicorn", "keble_data_infra_api.main:app", "--host", "0.0.0.0", "--port", "8000"]
@@ -0,0 +1,152 @@
1
+ Metadata-Version: 2.4
2
+ Name: keble-data-infra-api
3
+ Version: 0.1.0
4
+ Summary: Sole FastAPI composition root for Keble raw provider data.
5
+ Author-email: zhenhao-ma <bob0103779@gmail.com>
6
+ Requires-Python: <3.14,>=3.13
7
+ Requires-Dist: aiohttp<4,>=3
8
+ Requires-Dist: fastapi<1,>=0.110
9
+ Requires-Dist: httpx<1,>=0.28
10
+ Requires-Dist: keble-amz<3,>=2.0.0
11
+ Requires-Dist: keble-data-infra-contract<1,>=0.1.0
12
+ Requires-Dist: keble-db<2,>=1.9.0
13
+ Requires-Dist: keble-helpers<2,>=1.12.2
14
+ Requires-Dist: keble-keepa<2,>=1.3.0
15
+ Requires-Dist: keble-tiktok<1,>=0.5.0
16
+ Requires-Dist: motor<4,>=3
17
+ Requires-Dist: pymongo<5,>=4.10
18
+ Requires-Dist: redis<7,>=5
19
+ Requires-Dist: sentry-sdk<3,>=2
20
+ Requires-Dist: uvicorn<1,>=0.29
21
+ Description-Content-Type: text/markdown
22
+
23
+ # keble-data-infra-api
24
+
25
+ Install the composed raw-data API with
26
+ `pip install "keble-data-infra-api>=0.1.0,<1"`. The published distribution
27
+ resolves the released Amazon and TikTok providers; editable local sources in
28
+ the uv workspace are development-only.
29
+
30
+ The sole FastAPI composition root for raw Amazon and TikTok provider data.
31
+ Provider packages export framework-free services and manifests; this package
32
+ owns route generation, exception mapping, runtime resources, and ingestion.
33
+
34
+ The Stand Out platform consumes this service over HTTP and does not import its
35
+ contracts or provider packages.
36
+
37
+ ## Local provider environment
38
+
39
+ Build an ignored API-local `.env` from the existing Keble developer env files.
40
+ The helper copies only the required named settings and prints key names, never
41
+ credential values:
42
+
43
+ ```bash
44
+ python scripts/build_local_env.py
45
+ uv run --env-file .env pytest -q tests/live -m live
46
+ ```
47
+
48
+ The live suite uses independent `RUN_KEEPA_LIVE`, `RUN_FASTMOSS_LIVE`, and
49
+ `RUN_LIVE_ECHOTIK` gates. A gate is written only when its complete credential
50
+ set exists. Each configured provider runs as a separate canary, so one missing
51
+ credential cannot hide another provider's result. Provider failures are test
52
+ failures; missing opt-in credentials are the only valid reason to skip. Live
53
+ providers honor `PROVIDER_HTTP_TRUST_ENV` through `ApiSettings` instead of
54
+ hard-coding direct network access, matching the production runtime boundary.
55
+ The pinned TikTok provider includes HTTPX's SOCKS transport, so production
56
+ SOCKS4/5 proxy settings remain executable rather than failing during client
57
+ construction. The image also pins the timezone-aware/warning-free keble-db
58
+ checkpoint used by the local-full persistence proof.
59
+
60
+ ## Runtime ownership
61
+
62
+ `keble_data_infra_api.main:create_app` constructs the only raw FastAPI app.
63
+ `api.router:create_router` builds discovery and provider routes from the
64
+ registry, and `api.error_handlers` performs typed exception conversion. The
65
+ Stand Out adapter resolves the single product-search capability through
66
+ `GET /v1/{channel}/providers` and then calls its generated operation. The old
67
+ `/v1/commerce/search` route and compatibility DTO snapshot are deleted.
68
+
69
+ API settings and immutable application values reuse
70
+ `keble-helpers.PydanticModelConfig` or the contract-owned `ContractModel`.
71
+ Durable aggregates use `SchemaBase`/`MongoObjectBase`; do not introduce a local
72
+ `ConfigDict` or direct persistence `BaseModel` policy.
73
+
74
+ Mongo/Redis settings enable the API-owned provider infrastructure. The local
75
+ env builder copies only `MONGO_DB_URI`, `REDIS_URI`, and named provider
76
+ credentials; API-owned database/collection/namespace defaults stay isolated.
77
+ Startup creates cache indexes before provider handlers run and shutdown closes
78
+ all sync/async Mongo and Redis clients.
79
+
80
+ Video Enrichment projection is also API-owned and requires Mongo whenever its
81
+ Result origin is configured. One document stores the source event, owner-token
82
+ claim, attempts, failure state, and optional projection. Unique event/result
83
+ indexes plus expiring claims make concurrent consumers safe; Redis entries are
84
+ acknowledged only after the document atomically reaches `completed`. The old
85
+ replica-local JSON state file is not supported.
86
+
87
+ The same typed gateway centralizes submission and job/result correlation:
88
+
89
+ ```text
90
+ POST /internal/video-enrichments:submit
91
+ GET /internal/video-enrichment-jobs/{job_id}
92
+ GET /internal/video-enrichment-jobs/{job_id}/projection
93
+ ```
94
+
95
+ The projection lookup uses the API-owned job/status/completion-time compound
96
+ index. Consumers never call the independent media service or infer `result_id`
97
+ from provider storage. A projection 404 means durable completion ingestion is
98
+ still pending and is safe for a bounded worker retry.
99
+
100
+ The submission gateway forwards the expected media SHA-256 as a dedicated
101
+ integrity field. When an authorized provider transcript exists, the contract binds
102
+ its normalized content checksum, source reference, and source media checksum before
103
+ the independent service may skip ASR. The API never treats a provider title or video
104
+ description as spoken captions.
105
+
106
+ The producer's Job/Result payload contains additional stage, artifact, usage,
107
+ frame, warning, and error fields. Data Infra validates the typed stable subset
108
+ through the contract's source-projection inheritance and ignores only those
109
+ additive producer fields. All submission and public normalized models retain
110
+ `extra="forbid"`; compatibility does not weaken command validation.
111
+
112
+ Configured provider routes include:
113
+
114
+ ```text
115
+ POST /v1/amazon/providers/keepa/products/lookup
116
+ POST /v1/amazon/providers/keepa/products/search
117
+ POST /v1/amazon/providers/keepa/products/rank
118
+ POST /v1/tiktok/providers/fastmoss/products/search
119
+ POST /v1/tiktok/providers/echotik/products/rank
120
+ ```
121
+
122
+ Run the isolated all-provider generated-route canary with:
123
+
124
+ ```bash
125
+ python scripts/build_local_env.py
126
+ uv run --env-file .env pytest -q tests/live/test_provider_routes_live.py -m live
127
+ ```
128
+
129
+ ## Container and observability
130
+
131
+ The multi-stage image installs exact pushed provider commits and receives the
132
+ GitHub token only as a BuildKit secret. It runs as UID 10001, exposes `/health`
133
+ for liveness and `/ready` for configured Mongo/Redis readiness, and closes all
134
+ runtime clients through FastAPI lifespan.
135
+
136
+ Build the credential-preserving Docker env and start the profile on the existing
137
+ `keble_default` network:
138
+
139
+ ```bash
140
+ uv run --directory keble-data-infra-api python scripts/build_local_env.py
141
+ uv run --directory keble-data-infra-api python -m scripts.build_local_docker_env
142
+ GH_TOKEN="$(gh auth token)" docker compose \
143
+ --env-file keble-data-infra-api/.env.docker \
144
+ -f deploy/compose.raw-api.yaml --profile raw-api up --build
145
+ ```
146
+
147
+ `.env`, `.env.docker`, and the BuildKit secret stay outside Git and image
148
+ layers. Set `SENTRY_DSN`, `SENTRY_ENVIRONMENT`, and
149
+ `SENTRY_TRACES_SAMPLE_RATE` in the source env to enable error/performance
150
+ telemetry. Registry discovery, generated operations, cache outcomes, lease
151
+ contention, upstream work, batches, and video ingestion emit structural spans;
152
+ request/response content and credentials are not attached.
@@ -0,0 +1,130 @@
1
+ # keble-data-infra-api
2
+
3
+ Install the composed raw-data API with
4
+ `pip install "keble-data-infra-api>=0.1.0,<1"`. The published distribution
5
+ resolves the released Amazon and TikTok providers; editable local sources in
6
+ the uv workspace are development-only.
7
+
8
+ The sole FastAPI composition root for raw Amazon and TikTok provider data.
9
+ Provider packages export framework-free services and manifests; this package
10
+ owns route generation, exception mapping, runtime resources, and ingestion.
11
+
12
+ The Stand Out platform consumes this service over HTTP and does not import its
13
+ contracts or provider packages.
14
+
15
+ ## Local provider environment
16
+
17
+ Build an ignored API-local `.env` from the existing Keble developer env files.
18
+ The helper copies only the required named settings and prints key names, never
19
+ credential values:
20
+
21
+ ```bash
22
+ python scripts/build_local_env.py
23
+ uv run --env-file .env pytest -q tests/live -m live
24
+ ```
25
+
26
+ The live suite uses independent `RUN_KEEPA_LIVE`, `RUN_FASTMOSS_LIVE`, and
27
+ `RUN_LIVE_ECHOTIK` gates. A gate is written only when its complete credential
28
+ set exists. Each configured provider runs as a separate canary, so one missing
29
+ credential cannot hide another provider's result. Provider failures are test
30
+ failures; missing opt-in credentials are the only valid reason to skip. Live
31
+ providers honor `PROVIDER_HTTP_TRUST_ENV` through `ApiSettings` instead of
32
+ hard-coding direct network access, matching the production runtime boundary.
33
+ The pinned TikTok provider includes HTTPX's SOCKS transport, so production
34
+ SOCKS4/5 proxy settings remain executable rather than failing during client
35
+ construction. The image also pins the timezone-aware/warning-free keble-db
36
+ checkpoint used by the local-full persistence proof.
37
+
38
+ ## Runtime ownership
39
+
40
+ `keble_data_infra_api.main:create_app` constructs the only raw FastAPI app.
41
+ `api.router:create_router` builds discovery and provider routes from the
42
+ registry, and `api.error_handlers` performs typed exception conversion. The
43
+ Stand Out adapter resolves the single product-search capability through
44
+ `GET /v1/{channel}/providers` and then calls its generated operation. The old
45
+ `/v1/commerce/search` route and compatibility DTO snapshot are deleted.
46
+
47
+ API settings and immutable application values reuse
48
+ `keble-helpers.PydanticModelConfig` or the contract-owned `ContractModel`.
49
+ Durable aggregates use `SchemaBase`/`MongoObjectBase`; do not introduce a local
50
+ `ConfigDict` or direct persistence `BaseModel` policy.
51
+
52
+ Mongo/Redis settings enable the API-owned provider infrastructure. The local
53
+ env builder copies only `MONGO_DB_URI`, `REDIS_URI`, and named provider
54
+ credentials; API-owned database/collection/namespace defaults stay isolated.
55
+ Startup creates cache indexes before provider handlers run and shutdown closes
56
+ all sync/async Mongo and Redis clients.
57
+
58
+ Video Enrichment projection is also API-owned and requires Mongo whenever its
59
+ Result origin is configured. One document stores the source event, owner-token
60
+ claim, attempts, failure state, and optional projection. Unique event/result
61
+ indexes plus expiring claims make concurrent consumers safe; Redis entries are
62
+ acknowledged only after the document atomically reaches `completed`. The old
63
+ replica-local JSON state file is not supported.
64
+
65
+ The same typed gateway centralizes submission and job/result correlation:
66
+
67
+ ```text
68
+ POST /internal/video-enrichments:submit
69
+ GET /internal/video-enrichment-jobs/{job_id}
70
+ GET /internal/video-enrichment-jobs/{job_id}/projection
71
+ ```
72
+
73
+ The projection lookup uses the API-owned job/status/completion-time compound
74
+ index. Consumers never call the independent media service or infer `result_id`
75
+ from provider storage. A projection 404 means durable completion ingestion is
76
+ still pending and is safe for a bounded worker retry.
77
+
78
+ The submission gateway forwards the expected media SHA-256 as a dedicated
79
+ integrity field. When an authorized provider transcript exists, the contract binds
80
+ its normalized content checksum, source reference, and source media checksum before
81
+ the independent service may skip ASR. The API never treats a provider title or video
82
+ description as spoken captions.
83
+
84
+ The producer's Job/Result payload contains additional stage, artifact, usage,
85
+ frame, warning, and error fields. Data Infra validates the typed stable subset
86
+ through the contract's source-projection inheritance and ignores only those
87
+ additive producer fields. All submission and public normalized models retain
88
+ `extra="forbid"`; compatibility does not weaken command validation.
89
+
90
+ Configured provider routes include:
91
+
92
+ ```text
93
+ POST /v1/amazon/providers/keepa/products/lookup
94
+ POST /v1/amazon/providers/keepa/products/search
95
+ POST /v1/amazon/providers/keepa/products/rank
96
+ POST /v1/tiktok/providers/fastmoss/products/search
97
+ POST /v1/tiktok/providers/echotik/products/rank
98
+ ```
99
+
100
+ Run the isolated all-provider generated-route canary with:
101
+
102
+ ```bash
103
+ python scripts/build_local_env.py
104
+ uv run --env-file .env pytest -q tests/live/test_provider_routes_live.py -m live
105
+ ```
106
+
107
+ ## Container and observability
108
+
109
+ The multi-stage image installs exact pushed provider commits and receives the
110
+ GitHub token only as a BuildKit secret. It runs as UID 10001, exposes `/health`
111
+ for liveness and `/ready` for configured Mongo/Redis readiness, and closes all
112
+ runtime clients through FastAPI lifespan.
113
+
114
+ Build the credential-preserving Docker env and start the profile on the existing
115
+ `keble_default` network:
116
+
117
+ ```bash
118
+ uv run --directory keble-data-infra-api python scripts/build_local_env.py
119
+ uv run --directory keble-data-infra-api python -m scripts.build_local_docker_env
120
+ GH_TOKEN="$(gh auth token)" docker compose \
121
+ --env-file keble-data-infra-api/.env.docker \
122
+ -f deploy/compose.raw-api.yaml --profile raw-api up --build
123
+ ```
124
+
125
+ `.env`, `.env.docker`, and the BuildKit secret stay outside Git and image
126
+ layers. Set `SENTRY_DSN`, `SENTRY_ENVIRONMENT`, and
127
+ `SENTRY_TRACES_SAMPLE_RATE` in the source env to enable error/performance
128
+ telemetry. Registry discovery, generated operations, cache outcomes, lease
129
+ contention, upstream work, batches, and video ingestion emit structural spans;
130
+ request/response content and credentials are not attached.
@@ -0,0 +1,67 @@
1
+ [build-system]
2
+ requires = ["hatchling"]
3
+ build-backend = "hatchling.build"
4
+
5
+ [project]
6
+ name = "keble-data-infra-api"
7
+ version = "0.1.0"
8
+ description = "Sole FastAPI composition root for Keble raw provider data."
9
+ readme = "README.md"
10
+ requires-python = ">=3.13,<3.14"
11
+ dependencies = [
12
+ "aiohttp>=3,<4",
13
+ "fastapi>=0.110,<1",
14
+ "httpx>=0.28,<1",
15
+ "keble-amz>=2.0.0,<3",
16
+ "keble-data-infra-contract>=0.1.0,<1",
17
+ "keble-db>=1.9.0,<2",
18
+ "keble-helpers>=1.12.2,<2",
19
+ "keble-keepa>=1.3.0,<2",
20
+ "keble-tiktok>=0.5.0,<1",
21
+ "motor>=3,<4",
22
+ "pymongo>=4.10,<5",
23
+ "redis>=5,<7",
24
+ "sentry-sdk>=2,<3",
25
+ "uvicorn>=0.29,<1",
26
+ ]
27
+
28
+ [[project.authors]]
29
+ name = "zhenhao-ma"
30
+ email = "bob0103779@gmail.com"
31
+
32
+ [dependency-groups]
33
+ test = [
34
+ "pytest>=8.2,<9",
35
+ "pytest-asyncio>=1.4,<2",
36
+ "httpx>=0.28,<1",
37
+ "packaging>=24,<27",
38
+ ]
39
+
40
+ [tool.uv]
41
+ default-groups = ["test"]
42
+
43
+ [tool.uv.sources]
44
+ keble-data-infra-contract = {workspace = true}
45
+ keble-amz = {path = "../../keble-amz"}
46
+ keble-db = {path = "../../../deps/keble-py/keble-db"}
47
+ keble-helpers = {path = "../../../deps/keble-py/keble-helpers"}
48
+ keble-keepa = {path = "../../../deps/keble-keepa"}
49
+ keble-tiktok = {path = "../../keble-tiktok"}
50
+
51
+ [tool.pytest.ini_options]
52
+ addopts = "--strict-markers"
53
+ testpaths = ["tests"]
54
+ asyncio_mode = "auto"
55
+ markers = [
56
+ "unit: fast deterministic tests with no network",
57
+ "contract: recorded or fixture-based external contract tests",
58
+ "integration: real local dependency or adapter tests",
59
+ "live: real external service calls",
60
+ "slow: slow tests",
61
+ "db: any real database test",
62
+ "mongo: MongoDB tests",
63
+ "redis: Redis tests",
64
+ "db_stack: multi-database integration tests",
65
+ "container: tests requiring throwaway containers",
66
+ "local_stack: tests requiring existing local Keble docker compose stack",
67
+ ]
@@ -0,0 +1 @@
1
+ """Developer and release utilities for the raw API distribution."""
@@ -0,0 +1,78 @@
1
+ """Create an ignored container-network env from the API's ignored local env."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import argparse
6
+ from pathlib import Path
7
+ from urllib.parse import urlsplit, urlunsplit
8
+
9
+ from .build_local_env import _read_env
10
+
11
+
12
+ def _replace_endpoint(*, value: str, hostname: str, port: int) -> str:
13
+ """Replace only URI host/port while retaining credentials, path and options."""
14
+
15
+ normalized = value.strip().strip("'\"")
16
+ parsed = urlsplit(normalized)
17
+ if parsed.hostname is None:
18
+ raise ValueError("database URI must include a hostname")
19
+ old_endpoint = (
20
+ parsed.hostname
21
+ if parsed.port is None
22
+ else f"{parsed.hostname}:{parsed.port}"
23
+ )
24
+ netloc = parsed.netloc.replace(old_endpoint, f"{hostname}:{port}")
25
+ return urlunsplit((parsed.scheme, netloc, parsed.path, parsed.query, parsed.fragment))
26
+
27
+
28
+ def build_local_docker_env(*, source: Path, output: Path) -> tuple[str, ...]:
29
+ """Retain provider settings and map DB endpoints to existing Docker aliases.
30
+
31
+ Side effects if changes:
32
+ - `deploy/compose.raw-api.yaml` connects to `keble_default` Mongo/Redis;
33
+ - URI credentials and query options survive without being printed;
34
+ - the ignored output must never enter the Docker build context or Git.
35
+ """
36
+
37
+ assignments = _read_env(source)
38
+ mongo_uri = assignments.get("MONGO_DB_URI")
39
+ redis_uri = assignments.get("REDIS_URI")
40
+ if not mongo_uri or not redis_uri:
41
+ raise ValueError("local API env requires MONGO_DB_URI and REDIS_URI")
42
+ assignments["DATA_INFRA_MONGO_URI"] = _replace_endpoint(
43
+ value=mongo_uri,
44
+ hostname="mongodb",
45
+ port=27017,
46
+ )
47
+ assignments["DATA_INFRA_REDIS_URI"] = _replace_endpoint(
48
+ value=redis_uri,
49
+ hostname="redis",
50
+ port=6379,
51
+ )
52
+ output.parent.mkdir(parents=True, exist_ok=True)
53
+ temporary = output.with_suffix(f"{output.suffix}.tmp")
54
+ temporary.write_text(
55
+ "".join(f"{name}={value}\n" for name, value in sorted(assignments.items())),
56
+ encoding="utf-8",
57
+ )
58
+ temporary.replace(output)
59
+ return tuple(sorted(assignments))
60
+
61
+
62
+ def main() -> None:
63
+ """Build `.env.docker` beside the existing ignored API environment."""
64
+
65
+ api_root = Path(__file__).resolve().parents[1]
66
+ parser = argparse.ArgumentParser(description=__doc__)
67
+ parser.add_argument("--source", type=Path, default=api_root / ".env")
68
+ parser.add_argument("--output", type=Path, default=api_root / ".env.docker")
69
+ arguments = parser.parse_args()
70
+ written = build_local_docker_env(
71
+ source=arguments.source,
72
+ output=arguments.output,
73
+ )
74
+ print("Wrote Docker settings: " + ", ".join(written))
75
+
76
+
77
+ if __name__ == "__main__":
78
+ main()
@@ -0,0 +1,134 @@
1
+ """Build an ignored local API environment from existing Keble provider env files."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import argparse
6
+ from pathlib import Path
7
+
8
+
9
+ REQUIRED_PROVIDER_KEYS = (
10
+ "KEEPA_API_TOKEN",
11
+ "FASTMOSS_BASE_URL",
12
+ "FASTMOSS_CLIENT_SECRET",
13
+ "ECHOTIK_BASE_URL",
14
+ "ECHOTIK_USERNAME",
15
+ "ECHOTIK_PASSWORD",
16
+ "MONGO_DB_URI",
17
+ "REDIS_URI",
18
+ "SENTRY_DSN",
19
+ "SENTRY_ENVIRONMENT",
20
+ "SENTRY_TRACES_SAMPLE_RATE",
21
+ )
22
+
23
+
24
+ def _parser(*, umbrella_root: Path) -> argparse.ArgumentParser:
25
+ """Build explicit source and destination arguments for secret-safe env assembly.
26
+
27
+ Side effects if changes:
28
+ - local Keepa, FastMoss and EchoTik live-test setup;
29
+ - developer commands documented by ``keble-data-infra-api/README.md``.
30
+ """
31
+
32
+ parser = argparse.ArgumentParser(description=__doc__)
33
+ parser.add_argument(
34
+ "--backend-env",
35
+ type=Path,
36
+ default=umbrella_root / "keble.backend" / ".env",
37
+ )
38
+ parser.add_argument(
39
+ "--tiktok-env",
40
+ type=Path,
41
+ default=umbrella_root / "data-infra" / "keble-tiktok" / ".env",
42
+ )
43
+ parser.add_argument(
44
+ "--output",
45
+ type=Path,
46
+ default=Path(__file__).resolve().parents[1] / ".env",
47
+ )
48
+ return parser
49
+
50
+
51
+ def _read_env(path: Path) -> dict[str, str]:
52
+ """Read dotenv assignments without evaluating shell syntax or exposing values.
53
+
54
+ Side effects if changes:
55
+ - which existing local settings can enter the generated raw-API environment;
56
+ - whether quoted provider credentials retain their original dotenv encoding.
57
+ """
58
+
59
+ assignments: dict[str, str] = {}
60
+ for line in path.read_text(encoding="utf-8").splitlines():
61
+ candidate = line.strip()
62
+ if not candidate or candidate.startswith("#") or "=" not in candidate:
63
+ continue
64
+ name, separator, value = candidate.partition("=")
65
+ if separator and name.strip() in REQUIRED_PROVIDER_KEYS:
66
+ assignments[name.strip()] = value.strip()
67
+ return assignments
68
+
69
+
70
+ def _has_value(assignments: dict[str, str], name: str) -> bool:
71
+ """Report whether one copied dotenv assignment contains a non-empty value."""
72
+
73
+ value = assignments.get(name, "").strip()
74
+ return bool(value.strip("'\""))
75
+
76
+
77
+ def build_local_env(*, backend_env: Path, tiktok_env: Path, output: Path) -> tuple[str, ...]:
78
+ """Copy only named provider settings and enable gates with complete credentials.
79
+
80
+ Steps:
81
+ 1. parse the two existing local env files as inert dotenv text;
82
+ 2. select the six provider variables owned by this API;
83
+ 3. enable a live gate only when its complete credential set exists;
84
+ 4. atomically replace the ignored destination and return key names only.
85
+
86
+ Side effects if changes:
87
+ - API-local provider credentials and live-test opt-in gates;
88
+ - no source env file is modified and no secret value is printed.
89
+ """
90
+
91
+ assignments = {**_read_env(backend_env), **_read_env(tiktok_env)}
92
+ gates = {
93
+ "RUN_KEEPA_LIVE": _has_value(assignments, "KEEPA_API_TOKEN"),
94
+ "RUN_FASTMOSS_LIVE": all(
95
+ _has_value(assignments, name)
96
+ for name in ("FASTMOSS_BASE_URL", "FASTMOSS_CLIENT_SECRET")
97
+ ),
98
+ "RUN_LIVE_ECHOTIK": all(
99
+ _has_value(assignments, name)
100
+ for name in ("ECHOTIK_BASE_URL", "ECHOTIK_USERNAME", "ECHOTIK_PASSWORD")
101
+ ),
102
+ }
103
+ selected = {
104
+ name: assignments[name]
105
+ for name in REQUIRED_PROVIDER_KEYS
106
+ if _has_value(assignments, name)
107
+ }
108
+ selected.update({name: "1" for name, enabled in gates.items() if enabled})
109
+
110
+ output.parent.mkdir(parents=True, exist_ok=True)
111
+ temporary_output = output.with_suffix(f"{output.suffix}.tmp")
112
+ temporary_output.write_text(
113
+ "".join(f"{name}={value}\n" for name, value in sorted(selected.items())),
114
+ encoding="utf-8",
115
+ )
116
+ temporary_output.replace(output)
117
+ return tuple(sorted(selected))
118
+
119
+
120
+ def main() -> None:
121
+ """Resolve workspace defaults, build the env, and print key names only."""
122
+
123
+ umbrella_root = Path(__file__).resolve().parents[4]
124
+ arguments = _parser(umbrella_root=umbrella_root).parse_args()
125
+ written_keys = build_local_env(
126
+ backend_env=arguments.backend_env,
127
+ tiktok_env=arguments.tiktok_env,
128
+ output=arguments.output,
129
+ )
130
+ print("Wrote provider settings: " + ", ".join(written_keys))
131
+
132
+
133
+ if __name__ == "__main__":
134
+ main()