api-dock 0.8.0__tar.gz → 0.8.2__tar.gz

{api_dock-0.8.0 → api_dock-0.8.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: api_dock
-Version: 0.8.0
+Version: 0.8.2
 Summary: A flexible API gateway that allows you to proxy requests to multiple remote APIs and Databases
 Author-email: Brookie Guzder-Williams <bguzder-williams@berkeley.edu>
 License-Expression: BSD-3-Clause
@@ -219,19 +219,33 @@ remotes:
 settings:
   add_trailing_slash: true              # Auto-add trailing slash to paths (default: true)
   follow_protocol_downgrades: false     # Allow HTTPS->HTTP redirects (default: false)
+  follow_redirects: true                # Follow remote redirects (default: true)
   timeout: 10                           # Upstream request timeout in seconds (default: 10)
+  base_path: /dock                      # Also serve the API under this prefix (default: none)
+  duckdb:                               # Options for database queries (default: none)
+    memory_limit: 700MB
+    threads: 2
+    max_concurrent_queries: 2
 ```
 
-### HTTP behavior Settings
+### Settings
 
-The optional `settings` section controls HTTP behavior:
+The optional `settings` section controls HTTP and query behavior:
 
 - **`add_trailing_slash`** (default: `true`): Automatically append a trailing slash to all proxied paths. This prevents 307/301 redirects from remote APIs that require trailing slashes (e.g., `/projects` → `/projects/`). Set to `false` to disable this behavior.
 
 - **`follow_protocol_downgrades`** (default: `false`): Control how HTTP redirects are handled. When `false` (recommended), HTTPS→HTTP redirects are blocked for security. When `true`, allows following redirects that downgrade from HTTPS to HTTP (not recommended for production).
 
+- **`follow_redirects`** (default: `true`): Whether redirects from a remote are followed by API Dock (`true`) or passed through to the client with their `Location` header (`false`). Set it to `false` when a remote answers with redirects the client should follow itself, such as presigned S3 URLs for large files.
+
 - **`timeout`** (default: `10`): Upstream request timeout in seconds, applied to both the streaming and buffered proxy paths. Raise it for slow upstreams (e.g. large aggregation queries) that would otherwise return a 502 on timeout. Set to `null` or `false` to disable the timeout entirely (not recommended — a stalled upstream can hold the connection open indefinitely).
 
+- **`base_path`** (default: none): An extra URL prefix the API is also served under, e.g. `/dock`. Use it when a proxy or CDN forwards a path on another domain without stripping it (say CloudFront routes `https://app.example.org/dock/*` to API Dock): `/dock/birdnet/latest/detections/` is then handled as `/birdnet/latest/detections/`. Paths without the prefix keep working, so direct calls and health checks are unaffected.
+
+- **`duckdb`** (default: none): Options for the DuckDB connection each database query runs on. Every key except `max_concurrent_queries` is applied as `SET <key> = <value>`, so any [DuckDB setting](https://duckdb.org/docs/configuration/overview) works; the useful ones on small servers are `memory_limit` (DuckDB spills to disk or fails the query instead of exceeding it), `threads`, and `temp_directory`. `max_concurrent_queries` caps how many database queries run at once in the process; further queries wait their turn. Memory limits apply per query, so on a small instance set `memory_limit × max_concurrent_queries` below the instance's memory.
+
+Database queries run in worker threads, so a slow query doesn't hold up other requests (including health checks on `/`).
+
 ### Catalog Endpoints (`expose`)
 
 The optional `expose` section adds read-only endpoints that list the models and versions of your configured databases, remotes, or both ("sources"). Listings are **opt-in** — with no `expose` key nothing is added.
@@ -563,6 +577,60 @@ Notes:
 - Storage credentials are set per table. Tables whose `region`/`public` differ from the rest get their own S3 secret scoped to their path, so one query can mix regions and public/private buckets.
 - Views are created only for the `[[schema.table]]` tables a query actually references.
 
+#### Querying across schemas (`[[*.table]]`, schema groups)
+
+Union references read the same table from several schemas at once:
+
+| Reference | Reads |
+|---|---|
+| `[[*.detections]]` | every shared schema that has a `detections` table, including the current one |
+| `[[*!.detections]]` | the same, minus the current database/version's `schema:` |
+| `[[group1.detections]]` | the schemas listed in `schema_groups.group1` |
+| `[[group1!.detections]]` | that group, minus the current schema |
+
+```yaml
+# api_dock_config/databases/config.yaml
+schema_groups:          # named lists of shared schemas
+  birdnet_models:
+    - birdnet_2p4
+    - birdnet_bullfrog_2p4v0p5
+```
+
+- A union expands, after `FROM`/`JOIN` only, to a parenthesized `UNION ALL BY NAME` over the member schemas, so give it an alias: `FROM [[*.detections]] detections`. Columns missing from some members come back as `NULL`.
+- `*` skips schemas without the table. A group whose member lacks the table, a group naming an unknown schema, and a group sharing a name with a schema are all errors. `!` applies only to `*` and groups; with no `schema:`, it removes nothing.
+- Every member gets its own S3 credentials (see above), so a union can mix regions and public/private buckets.
+
+**Source columns.** Union rows carry only the tables' real columns unless the route asks for more with `source_columns`. The available facts are `schema` (the member schema), and `name` and `version` (the database/version whose `schema:` is that schema, or `NULL` if none or several use it):
+
+```yaml
+source_columns: [schema, name, version]      # adds schema_name, name, version
+source_columns: {schema: _schema, name: model}   # pick a subset and rename
+```
+
+A source column that clashes with a real column raises an error. To use one for filtering without returning it, use DuckDB's `EXCLUDE`: `SELECT detections.* EXCLUDE (schema_name) ...`.
+
+**`{{self.*}}` placeholders.** `{{self.schema}}`, `{{self.name}}` and `{{self.version}}` are the current database/version's schema, name and version, as SQL literals (`NULL` when unknown).
+
+Together they make an "overlaps" route that every database/version can share. It returns every detection overlapping the given one, across all schemas, except that detection itself; other overlapping rows in the same schema are kept:
+
+```yaml
+routes:
+  - route: detections/{{id}}/overlaps
+    source_columns: [schema, name, version]
+    sql: |
+      WITH src AS (
+        SELECT recording_id, start_time, end_time FROM [[detections]] WHERE id = {{id}}
+      )
+      SELECT detections.*
+      FROM [[*.detections]] detections
+      JOIN src ON detections.recording_id = src.recording_id
+              AND detections.start_time < src.end_time
+              AND detections.end_time   > src.start_time
+      WHERE NOT (detections.schema_name = {{self.schema}} AND detections.id = {{id}})
+```
+
+Aliasing the union as `detections` also lets shared filters such as `[[detections]].confidence >= {{confidence}}` apply to the overlapping rows.
+
 #### Inline database configs (`slugs`)
 
 Simple database/version configs (often just a description and a `schema`) can live in the shared file instead of in their own files. Config files keep working, and the two can be mixed, even for the same database:
@@ -1543,47 +1611,52 @@ pixi run python scripts/hello_world.py
 
 ## Publishing a Release
 
+Publishing a GitHub Release is what publishes to PyPI: `.github/workflows/publish_to_pypi.yml` runs on `release: published`, builds the sdist and wheel with `uv build`, and uploads them using PyPI trusted publishing (OIDC). There's no local build, no API token, and no `twine`. conda-forge follows automatically: its bot opens a version PR on [conda-forge/api_dock-feedstock](https://github.com/conda-forge/api_dock-feedstock), which a maintainer merges.
+
 ```bash
-# 0. Make sure you are on `main` and merged with any changes
+# 0. Start from a clean, up-to-date main
+export VERSION=0.8.2          # the NEW version, no leading "v"
+git checkout main
+git pull origin main
+git status
 
-# 1. Bump version in pyproject.toml
+# 1. Set `version` in pyproject.toml to $VERSION
 
-# 2. Commit everything
+# 2. Run the tests
+pixi run -e dev pytest -q
+
+# 3. Commit, tag, push (the commit command adds the "v$VERSION: " prefix)
+export COMMIT_MESSAGE='query worker threads, duckdb settings, base_path, proxy host fix'
 git add -A
-git commit -m "v0.6.1: stream proxy responses (fix large-response 502 + content-encoding)"
-
-# 3. Tag and push
-git tag v0.6.1
-git push origin main v0.6.1
-
-# 4. Build the wheel (requires the `dev` pixi environment)
-rm -rf dist/
-find . -name "__pycache__" -type d -exec rm -rf {} +
-find . -name "*.pyc" -delete
-pixi run -e dev python -m build --wheel
-ls dist/*.whl
-
-# 5. Create GitHub release with the wheel attached
-gh release create v0.6.1 dist/api_dock-0.6.1-py3-none-any.whl \
-    --title "v0.6.1" --notes "$(cat <<'EOF'
+git commit -m "v$VERSION: $COMMIT_MESSAGE"
+git tag "v$VERSION"
+git push origin main "v$VERSION"
+
+# 4. Publish the GitHub Release; this triggers the PyPI upload.
+#    Don't use --draft (the workflow only runs on a published release); no wheel needs attaching.
+gh release create "v$VERSION" \
+  --title "v$VERSION" \
+  --notes "$(cat <<'EOF'
 * new features
-    - Remote proxy responses are now streamed (FastAPI) — upstream bytes are piped to the client as they arrive instead of being buffered fully in memory
-    - New `timeout` setting (default 10s) for the upstream request; set to `null`/`false` to disable
+    - `settings.duckdb`: DuckDB options applied to every database query (`memory_limit`, `threads`, `temp_directory`, or any other DuckDB setting), plus `max_concurrent_queries` to cap how many queries run at once
+    - `settings.base_path`: also serve the API under a URL prefix (e.g. `/dock`), for a CDN/proxy path such as CloudFront routing `https://app.example.org/dock/*` to API Dock; unprefixed paths keep working
 * bug fixes
-    - Large upstream responses no longer return 502 — streamed via `StreamingResponse` instead of reading the whole body into memory
-    - `Content-Encoding` (gzip/br/deflate) is now preserved on compressed responses — raw bytes are streamed via `aiter_raw()` so the header stays valid and the client can decompress
-    - Slow upstreams (e.g. large aggregation queries) no longer 502 at httpx's hardcoded 5s default — the timeout is now configurable via the `timeout` setting
+    - Database queries now run in worker threads, so one slow query no longer stalls every other request (including health checks on `/`)
+    - Remote proxying no longer forwards the client's `Host` (or hop-by-hop) headers upstream; upstream redirects (e.g. trailing-slash 307s) no longer point back at the proxy with the wrong host and path
 * cleanup / other improvements
-    - Added `PreparedRequest` dataclass and split route validation/resolution into `RouteMapper.prepare_remote_request()`; the FastAPI adapter issues the streaming HTTP call
-    - `map_route()` (buffered) retained for the Flask/sync path
-    - Added streaming test coverage (`TestStreamUpstream`, plus `prepare_remote_request` and streaming-header tests) — 53 tests total
+    - README: document `base_path`, `duckdb`, and the previously undocumented `follow_redirects` setting
+    - Test suite grew from 227 to 253 tests (`test_runtime_settings.py`)
 EOF
 )"
 
-# 6. Publish to PyPI
-pixi run -e dev python -m twine upload dist/*.whl
-```
+# 5. Watch the publish workflow, then confirm PyPI has the new version
+gh run watch "$(gh run list --workflow=publish_to_pypi.yml -L1 --json databaseId -q '.[0].databaseId')" --repo SchmidtDSE/api_dock
+curl -s https://pypi.org/pypi/api-dock/json | python3 -c "import sys,json; print('PyPI latest:', json.load(sys.stdin)['info']['version'])"
 
+# 6. conda-forge: once the bot opens the v$VERSION PR (usually within hours), check that the recipe's
+#    run requirements match pyproject.toml dependencies (the bot only bumps version + sha256), then merge it
+gh pr list --repo conda-forge/api_dock-feedstock --state open
+```
 
 ---
 

{api_dock-0.8.0 → api_dock-0.8.2}/README.md

@@ -180,19 +180,33 @@ remotes:
 settings:
   add_trailing_slash: true              # Auto-add trailing slash to paths (default: true)
   follow_protocol_downgrades: false     # Allow HTTPS->HTTP redirects (default: false)
+  follow_redirects: true                # Follow remote redirects (default: true)
   timeout: 10                           # Upstream request timeout in seconds (default: 10)
+  base_path: /dock                      # Also serve the API under this prefix (default: none)
+  duckdb:                               # Options for database queries (default: none)
+    memory_limit: 700MB
+    threads: 2
+    max_concurrent_queries: 2
 ```
 
-### HTTP behavior Settings
+### Settings
 
-The optional `settings` section controls HTTP behavior:
+The optional `settings` section controls HTTP and query behavior:
 
 - **`add_trailing_slash`** (default: `true`): Automatically append a trailing slash to all proxied paths. This prevents 307/301 redirects from remote APIs that require trailing slashes (e.g., `/projects` → `/projects/`). Set to `false` to disable this behavior.
 
 - **`follow_protocol_downgrades`** (default: `false`): Control how HTTP redirects are handled. When `false` (recommended), HTTPS→HTTP redirects are blocked for security. When `true`, allows following redirects that downgrade from HTTPS to HTTP (not recommended for production).
 
+- **`follow_redirects`** (default: `true`): Whether redirects from a remote are followed by API Dock (`true`) or passed through to the client with their `Location` header (`false`). Set it to `false` when a remote answers with redirects the client should follow itself, such as presigned S3 URLs for large files.
+
 - **`timeout`** (default: `10`): Upstream request timeout in seconds, applied to both the streaming and buffered proxy paths. Raise it for slow upstreams (e.g. large aggregation queries) that would otherwise return a 502 on timeout. Set to `null` or `false` to disable the timeout entirely (not recommended — a stalled upstream can hold the connection open indefinitely).
 
+- **`base_path`** (default: none): An extra URL prefix the API is also served under, e.g. `/dock`. Use it when a proxy or CDN forwards a path on another domain without stripping it (say CloudFront routes `https://app.example.org/dock/*` to API Dock): `/dock/birdnet/latest/detections/` is then handled as `/birdnet/latest/detections/`. Paths without the prefix keep working, so direct calls and health checks are unaffected.
+
+- **`duckdb`** (default: none): Options for the DuckDB connection each database query runs on. Every key except `max_concurrent_queries` is applied as `SET <key> = <value>`, so any [DuckDB setting](https://duckdb.org/docs/configuration/overview) works; the useful ones on small servers are `memory_limit` (DuckDB spills to disk or fails the query instead of exceeding it), `threads`, and `temp_directory`. `max_concurrent_queries` caps how many database queries run at once in the process; further queries wait their turn. Memory limits apply per query, so on a small instance set `memory_limit × max_concurrent_queries` below the instance's memory.
+
+Database queries run in worker threads, so a slow query doesn't hold up other requests (including health checks on `/`).
+
 ### Catalog Endpoints (`expose`)
 
 The optional `expose` section adds read-only endpoints that list the models and versions of your configured databases, remotes, or both ("sources"). Listings are **opt-in** — with no `expose` key nothing is added.
@@ -524,6 +538,60 @@ Notes:
 - Storage credentials are set per table. Tables whose `region`/`public` differ from the rest get their own S3 secret scoped to their path, so one query can mix regions and public/private buckets.
 - Views are created only for the `[[schema.table]]` tables a query actually references.
 
+#### Querying across schemas (`[[*.table]]`, schema groups)
+
+Union references read the same table from several schemas at once:
+
+| Reference | Reads |
+|---|---|
+| `[[*.detections]]` | every shared schema that has a `detections` table, including the current one |
+| `[[*!.detections]]` | the same, minus the current database/version's `schema:` |
+| `[[group1.detections]]` | the schemas listed in `schema_groups.group1` |
+| `[[group1!.detections]]` | that group, minus the current schema |
+
+```yaml
+# api_dock_config/databases/config.yaml
+schema_groups:          # named lists of shared schemas
+  birdnet_models:
+    - birdnet_2p4
+    - birdnet_bullfrog_2p4v0p5
+```
+
+- A union expands, after `FROM`/`JOIN` only, to a parenthesized `UNION ALL BY NAME` over the member schemas, so give it an alias: `FROM [[*.detections]] detections`. Columns missing from some members come back as `NULL`.
+- `*` skips schemas without the table. A group whose member lacks the table, a group naming an unknown schema, and a group sharing a name with a schema are all errors. `!` applies only to `*` and groups; with no `schema:`, it removes nothing.
+- Every member gets its own S3 credentials (see above), so a union can mix regions and public/private buckets.
+
+**Source columns.** Union rows carry only the tables' real columns unless the route asks for more with `source_columns`. The available facts are `schema` (the member schema), and `name` and `version` (the database/version whose `schema:` is that schema, or `NULL` if none or several use it):
+
+```yaml
+source_columns: [schema, name, version]      # adds schema_name, name, version
+source_columns: {schema: _schema, name: model}   # pick a subset and rename
+```
+
+A source column that clashes with a real column raises an error. To use one for filtering without returning it, use DuckDB's `EXCLUDE`: `SELECT detections.* EXCLUDE (schema_name) ...`.
+
+**`{{self.*}}` placeholders.** `{{self.schema}}`, `{{self.name}}` and `{{self.version}}` are the current database/version's schema, name and version, as SQL literals (`NULL` when unknown).
+
+Together they make an "overlaps" route that every database/version can share. It returns every detection overlapping the given one, across all schemas, except that detection itself; other overlapping rows in the same schema are kept:
+
+```yaml
+routes:
+  - route: detections/{{id}}/overlaps
+    source_columns: [schema, name, version]
+    sql: |
+      WITH src AS (
+        SELECT recording_id, start_time, end_time FROM [[detections]] WHERE id = {{id}}
+      )
+      SELECT detections.*
+      FROM [[*.detections]] detections
+      JOIN src ON detections.recording_id = src.recording_id
+              AND detections.start_time < src.end_time
+              AND detections.end_time   > src.start_time
+      WHERE NOT (detections.schema_name = {{self.schema}} AND detections.id = {{id}})
+```
+
+Aliasing the union as `detections` also lets shared filters such as `[[detections]].confidence >= {{confidence}}` apply to the overlapping rows.
+
 #### Inline database configs (`slugs`)
 
 Simple database/version configs (often just a description and a `schema`) can live in the shared file instead of in their own files. Config files keep working, and the two can be mixed, even for the same database:
@@ -1504,47 +1572,52 @@ pixi run python scripts/hello_world.py
 
 ## Publishing a Release
 
+Publishing a GitHub Release is what publishes to PyPI: `.github/workflows/publish_to_pypi.yml` runs on `release: published`, builds the sdist and wheel with `uv build`, and uploads them using PyPI trusted publishing (OIDC). There's no local build, no API token, and no `twine`. conda-forge follows automatically: its bot opens a version PR on [conda-forge/api_dock-feedstock](https://github.com/conda-forge/api_dock-feedstock), which a maintainer merges.
+
 ```bash
-# 0. Make sure you are on `main` and merged with any changes
+# 0. Start from a clean, up-to-date main
+export VERSION=0.8.2          # the NEW version, no leading "v"
+git checkout main
+git pull origin main
+git status
 
-# 1. Bump version in pyproject.toml
+# 1. Set `version` in pyproject.toml to $VERSION
 
-# 2. Commit everything
+# 2. Run the tests
+pixi run -e dev pytest -q
+
+# 3. Commit, tag, push (the commit command adds the "v$VERSION: " prefix)
+export COMMIT_MESSAGE='query worker threads, duckdb settings, base_path, proxy host fix'
 git add -A
-git commit -m "v0.6.1: stream proxy responses (fix large-response 502 + content-encoding)"
-
-# 3. Tag and push
-git tag v0.6.1
-git push origin main v0.6.1
-
-# 4. Build the wheel (requires the `dev` pixi environment)
-rm -rf dist/
-find . -name "__pycache__" -type d -exec rm -rf {} +
-find . -name "*.pyc" -delete
-pixi run -e dev python -m build --wheel
-ls dist/*.whl
-
-# 5. Create GitHub release with the wheel attached
-gh release create v0.6.1 dist/api_dock-0.6.1-py3-none-any.whl \
-    --title "v0.6.1" --notes "$(cat <<'EOF'
+git commit -m "v$VERSION: $COMMIT_MESSAGE"
+git tag "v$VERSION"
+git push origin main "v$VERSION"
+
+# 4. Publish the GitHub Release; this triggers the PyPI upload.
+#    Don't use --draft (the workflow only runs on a published release); no wheel needs attaching.
+gh release create "v$VERSION" \
+  --title "v$VERSION" \
+  --notes "$(cat <<'EOF'
 * new features
-    - Remote proxy responses are now streamed (FastAPI) — upstream bytes are piped to the client as they arrive instead of being buffered fully in memory
-    - New `timeout` setting (default 10s) for the upstream request; set to `null`/`false` to disable
+    - `settings.duckdb`: DuckDB options applied to every database query (`memory_limit`, `threads`, `temp_directory`, or any other DuckDB setting), plus `max_concurrent_queries` to cap how many queries run at once
+    - `settings.base_path`: also serve the API under a URL prefix (e.g. `/dock`), for a CDN/proxy path such as CloudFront routing `https://app.example.org/dock/*` to API Dock; unprefixed paths keep working
 * bug fixes
-    - Large upstream responses no longer return 502 — streamed via `StreamingResponse` instead of reading the whole body into memory
-    - `Content-Encoding` (gzip/br/deflate) is now preserved on compressed responses — raw bytes are streamed via `aiter_raw()` so the header stays valid and the client can decompress
-    - Slow upstreams (e.g. large aggregation queries) no longer 502 at httpx's hardcoded 5s default — the timeout is now configurable via the `timeout` setting
+    - Database queries now run in worker threads, so one slow query no longer stalls every other request (including health checks on `/`)
+    - Remote proxying no longer forwards the client's `Host` (or hop-by-hop) headers upstream; upstream redirects (e.g. trailing-slash 307s) no longer point back at the proxy with the wrong host and path
 * cleanup / other improvements
-    - Added `PreparedRequest` dataclass and split route validation/resolution into `RouteMapper.prepare_remote_request()`; the FastAPI adapter issues the streaming HTTP call
-    - `map_route()` (buffered) retained for the Flask/sync path
-    - Added streaming test coverage (`TestStreamUpstream`, plus `prepare_remote_request` and streaming-header tests) — 53 tests total
+    - README: document `base_path`, `duckdb`, and the previously undocumented `follow_redirects` setting
+    - Test suite grew from 227 to 253 tests (`test_runtime_settings.py`)
 EOF
 )"
 
-# 6. Publish to PyPI
-pixi run -e dev python -m twine upload dist/*.whl
-```
+# 5. Watch the publish workflow, then confirm PyPI has the new version
+gh run watch "$(gh run list --workflow=publish_to_pypi.yml -L1 --json databaseId -q '.[0].databaseId')" --repo SchmidtDSE/api_dock
+curl -s https://pypi.org/pypi/api-dock/json | python3 -c "import sys,json; print('PyPI latest:', json.load(sys.stdin)['info']['version'])"
 
+# 6. conda-forge: once the bot opens the v$VERSION PR (usually within hours), check that the recipe's
+#    run requirements match pyproject.toml dependencies (the bot only bumps version + sha256), then merge it
+gh pr list --repo conda-forge/api_dock-feedstock --state open
+```
 
 ---
 

{api_dock-0.8.0 → api_dock-0.8.2}/api_dock/database_config.py

@@ -56,6 +56,15 @@ SLUG_NAME_KEY: str = "name"
 SLUG_VERSION_KEY: str = "version"
 SLUG_VERSIONS_KEY: str = "versions"
 
+# Named groups of shared schemas: {group: [schema, ...]}, referenced in SQL as
+# [[group.table]]. Group names may not collide with schema names.
+SCHEMA_GROUPS_KEY: str = "schema_groups"
+
+# Union selectors: [[*.table]] = every shared schema with that table; a trailing
+# "!" ([[*!.table]], [[group!.table]]) drops the current version's schema.
+ALL_SCHEMAS: str = "*"
+EXCLUDE_SELF_SUFFIX: str = "!"
+
 # Exclusion version wildcard: matches every version (and unversioned databases).
 ALL_VERSIONS: str = "*"
 
@@ -212,7 +221,7 @@ def load_shared_config(config_dir: Optional[str] = None) -> Dict[str, Any]:
     """Load the whole shared database config file (``databases/config.yaml``).
 
     Top-level keys: ``database`` (tables, ``meta``, ``schema``), ``slugs``
-    (inline database/version configs), ``routes`` and
+    (inline database/version configs), ``schema_groups``, ``routes`` and
     ``query_params`` (added to every database/version), and the
     ``route_inclusions`` / ``query_inclusions`` and ``route_exclusions`` /
     ``query_exclusions`` lists.
@@ -250,12 +259,15 @@ def load_shared_config(config_dir: Optional[str] = None) -> Dict[str, Any]:
         ROUTE_EXCLUSIONS_KEY: list,
         QUERY_EXCLUSIONS_KEY: list,
         SLUGS_KEY: list,
+        SCHEMA_GROUPS_KEY: dict,
     }
     for key, expected in expected_types.items():
         value = contents.get(key) or expected()
         if not isinstance(value, expected):
             raise ValueError(f"'{key}' in {shared_path} must be a {expected.__name__}")
         normalized[key] = value
+
+    _validate_schema_groups(normalized, shared_path)
     return normalized
 
 
@@ -418,6 +430,92 @@ def resolve_table_reference(
     return None
 
 
+def resolve_schema_union(
+        selector: str,
+        table_name: str,
+        shared_config: Optional[Dict[str, Any]] = None,
+        schema_groups: Optional[Dict[str, List[str]]] = None) -> Optional[List[TableReference]]:
+    """Resolve a union selector (``*`` or a schema group) to its member tables.
+
+    Args:
+        selector: ``*`` for every shared schema, or a ``schema_groups`` name
+            (without any trailing ``!``).
+        table_name: Table to read from each member schema.
+        shared_config: The shared ``database`` mapping, or None.
+        schema_groups: The shared ``schema_groups`` mapping, or None.
+
+    Returns:
+        Qualified TableReferences in schema/group order, or None if the
+        selector is neither ``*`` nor a group (i.e. it names a single schema).
+
+    Raises:
+        ValueError: If no schema has the table (``*``), or a group member is
+            not a schema or lacks the table.
+    """
+    schemas = (shared_config or {}).get(SHARED_SCHEMA_KEY) or {}
+    groups = schema_groups or {}
+
+    if selector == ALL_SCHEMAS:
+        members = [name for name, tables in schemas.items() if table_name in (tables or {})]
+        if not members:
+            raise ValueError(f"No shared schema has a table named '{table_name}'")
+    elif selector in groups:
+        members = list(groups[selector])
+        for schema_name in members:
+            if table_name not in (schemas.get(schema_name) or {}):
+                raise ValueError(
+                    f"Schema '{schema_name}' in group '{selector}' has no table '{table_name}'"
+                )
+    else:
+        return None
+
+    references = []
+    for schema_name in members:
+        reference = resolve_table_reference(
+            f"{schema_name}{SCHEMA_SEPARATOR}{table_name}", {}, shared_config
+        )
+        if reference is None:
+            raise ValueError(f"Table '{schema_name}.{table_name}' not found")
+        references.append(reference)
+    return references
+
+
+def get_schema_sources(
+        database_names: List[str],
+        config_dir: Optional[str] = None) -> Dict[str, Tuple[str, Optional[str]]]:
+    """Map each shared schema to the database/version that uses it.
+
+    Args:
+        database_names: Served database names (from the main config).
+        config_dir: Base config directory. If None, uses default.
+
+    Returns:
+        Schema name -> (database name, version or None). Schemas used by more
+        than one database/version are left out (their source is ambiguous).
+    """
+    sources: Dict[str, Tuple[str, Optional[str]]] = {}
+    ambiguous = set()
+
+    for database_name in database_names:
+        if is_versioned_database(database_name, config_dir):
+            versions: List[Optional[str]] = list(get_database_versions(database_name, config_dir))
+        else:
+            versions = [None]
+        for version in versions:
+            try:
+                config = load_database_config(database_name, config_dir, version)
+            except FileNotFoundError:
+                continue
+            schema_name = config.get(DATABASE_SCHEMA_KEY) if isinstance(config, dict) else None
+            if not schema_name:
+                continue
+            if schema_name in sources:
+                ambiguous.add(schema_name)
+            sources[schema_name] = (database_name, version)
+
+    return {k: v for k, v in sources.items() if k not in ambiguous}
+
+
 def get_local_table_references(
         database_config: Dict[str, Any],
         shared_config: Optional[Dict[str, Any]] = None) -> List[TableReference]:
@@ -789,6 +887,37 @@ def _load_yaml_file(file_path: str) -> Dict[str, Any]:
         raise yaml.YAMLError(f"Invalid YAML in {file_path}: {e}")
 
 
+def _validate_schema_groups(shared_file: Dict[str, Any], shared_path: str) -> None:
+    """Validate the shared ``schema_groups`` mapping.
+
+    Args:
+        shared_file: The normalized shared config.
+        shared_path: Path of the shared config (for error messages).
+
+    Raises:
+        ValueError: If a group name isn't a plain identifier or collides with a
+            schema name, or a group isn't a non-empty list of known schemas.
+    """
+    schemas = (shared_file.get(SHARED_CONFIG_KEY) or {}).get(SHARED_SCHEMA_KEY) or {}
+    for group, members in (shared_file.get(SCHEMA_GROUPS_KEY) or {}).items():
+        if not IDENTIFIER_PATTERN.match(str(group)):
+            raise ValueError(f"{SCHEMA_GROUPS_KEY}: invalid group name '{group}' in {shared_path}")
+        if group in schemas:
+            raise ValueError(
+                f"{SCHEMA_GROUPS_KEY}: '{group}' is also a schema name in {shared_path}"
+            )
+        if not isinstance(members, list) or not members:
+            raise ValueError(
+                f"{SCHEMA_GROUPS_KEY}: '{group}' must be a non-empty list in {shared_path}"
+            )
+        for member in members:
+            if member not in schemas:
+                raise ValueError(
+                    f"{SCHEMA_GROUPS_KEY}: '{group}' names unknown schema '{member}' "
+                    f"in {shared_path}"
+                )
+
+
 def _is_selected(
         inclusions: Any,
         exclusions: Any,

{api_dock-0.8.0 → api_dock-0.8.2}/api_dock/example_api_dock_config/config.yaml

@@ -15,6 +15,11 @@ settings:
   add_trailing_slash: false   # Set true to auto-append trailing slash to proxied paths
   follow_redirects: true      # Set false to pass 3xx redirects through to the client
   timeout: 10                 # Upstream request timeout (seconds); null/false to disable
+  # base_path: /dock          # Also serve the API under this prefix (e.g. behind a CDN path)
+  # duckdb:                   # Database query options (DuckDB SET options + a concurrency cap)
+  #   memory_limit: 700MB
+  #   threads: 2
+  #   max_concurrent_queries: 2
 
 # Global route restrictions — applied to all remotes unless overridden per-remote.
 # Uncomment to block DELETE on every remote:

{api_dock-0.8.0 → api_dock-0.8.2}/api_dock/example_api_dock_config/databases/config.yaml

@@ -36,6 +36,11 @@
 #       items:
 #         uri: s3://your-bucket/v2/items/**/*.parquet
 #
+# # named groups of schemas: [[all_items.items]] reads both; [[all_items!.items]]
+# # skips the current version's schema; [[*.items]] reads every schema
+# schema_groups:
+#   all_items: [items_v1, items_v2]
+#
 # # database/version configs defined inline instead of as files (list them under
 # # `databases:` in the main config like any other database)
 # slugs:
@@ -68,6 +73,13 @@
 #   - route: items/
 #     include: ['example_db']   # ONLY add this route to these slug/versions
 #     sql: SELECT [[items]].id FROM [[items]]
+#   # every items row in the same group as this one, from any schema, tagged with its source
+#   - route: items/{{id}}/related
+#     source_columns: [schema, name, version]
+#     sql: >
+#       SELECT items.* FROM [[*.items]] items
+#       WHERE items.group_id = (SELECT group_id FROM [[items]] WHERE id = {{id}})
+#         AND NOT (items.schema_name = {{self.schema}} AND items.id = {{id}})
 #
 # query_params:
 #   - limit:

{api_dock-0.8.0 → api_dock-0.8.2}/api_dock/fast_api.py

@@ -16,9 +16,14 @@ import warnings
 import httpx
 from fastapi import FastAPI, Request
 from fastapi.responses import JSONResponse, Response, StreamingResponse
-from typing import Any, Dict, Optional
-
-from api_dock.route_mapper import collect_multi_query_params, HOP_BY_HOP_HEADERS, RouteMapper
+from typing import Any, Callable, Dict, Optional
+
+from api_dock.route_mapper import (
+    collect_multi_query_params,
+    HOP_BY_HOP_HEADERS,
+    RouteMapper,
+    strip_base_path,
+)
 from api_dock.types import PreparedRequest, ProxyResponse
 
 
@@ -66,12 +71,36 @@ def create_app(config_path: Optional[str] = None) -> FastAPI:
     _add_remote_routes(app, route_mapper)
     _add_error_handlers(app)
 
+    if route_mapper.base_path:
+        app.add_middleware(_StripBasePath, base_path=route_mapper.base_path)
+
     return app
 
 
 #
 # INTERNAL
 #
+class _StripBasePath:
+    """ASGI middleware serving the app under ``settings.base_path`` as well.
+
+    Requests whose path starts with the base path (e.g. ``/dock/birdnet/...``)
+    are routed as if it weren't there; other paths pass through unchanged.
+    """
+
+    def __init__(self, app: Callable, base_path: str) -> None:
+        self.app = app
+        self.base_path = base_path
+
+    async def __call__(self, scope: Dict[str, Any], receive: Callable, send: Callable) -> None:
+        """Rewrite the request path, then call the wrapped app."""
+        if scope.get("type") in ("http", "websocket"):
+            path = strip_base_path(scope.get("path", ""), self.base_path)
+            if path != scope.get("path"):
+                scope = {**scope, "path": path, "raw_path": path.encode()}
+        await self.app(scope, receive, send)
+
+
+
 def _add_main_routes(app: FastAPI, route_mapper: RouteMapper) -> None:
     """Add main API routes to the FastAPI app.