bdns-sync 0.1.0__py3-none-any.whl

bdns/sync/__init__.py

@@ -0,0 +1,5 @@
+"""BDNS Sync - keeps target databases in SCD2 form from the BDNS API."""
+
+__version__ = "0.1.0"
+
+__all__ = ["__version__"]

bdns/sync/__main__.py

@@ -0,0 +1,14 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+"""Main entry point for the BDNS Sync command line interface.
+
+Also what the installed `bdns-sync` console script points to
+(`pyproject.toml`). It imports `app` directly rather than running this
+module as `__main__`, so logging setup lives in `cli.py`'s callback
+instead of here (see its docstring).
+"""
+
+from bdns.sync.cli import app
+
+if __name__ == "__main__":
+    app()

bdns/sync/cli.py

@@ -0,0 +1,158 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+"""bdns-sync is a pure parameterized tool: one endpoint per invocation, no
+config file, no cadence knowledge. Which endpoints to sync and when is an
+orchestration concern that lives outside this package (see scripts/).
+"""
+
+import logging
+import sys
+from datetime import date
+from typing import Optional
+
+import typer
+
+import bdns.fetch.client as _bdns_fetch_client
+from bdns.fetch import BDNSClient
+from bdns.sync import __version__
+from bdns.sync.generic import WINDOWS
+from bdns.sync.sinks import get_sink
+from bdns.sync.syncers import FULL_SYNCERS, SEARCH_SYNCERS
+
+app = typer.Typer(
+    name="bdns-sync",
+    help="Sync one BDNS API endpoint into a target database in SCD2 form.",
+    add_completion=False,
+)
+
+
+def _version_callback(value: bool) -> None:
+    if value:
+        typer.echo(f"bdns-sync {__version__}")
+        raise typer.Exit()
+
+
+@app.callback()
+def main(
+    version: bool = typer.Option(
+        False,
+        "--version",
+        callback=_version_callback,
+        is_eager=True,
+        help="Show the version and exit.",
+    ),
+) -> None:
+    """BDNS Sync command line interface.
+
+    Configures logging here, not in `__main__.py`'s `if __name__ ==
+    "__main__":` guard. The installed `bdns-sync` console script
+    (`pyproject.toml`) imports and calls this Typer `app` directly, so that
+    guard never runs. Typer always runs this callback before any
+    subcommand regardless of entry point, so this is the one place
+    guaranteed to run every time.
+    """
+    logging.basicConfig(
+        level=logging.INFO,
+        format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
+        force=True,
+    )
+    if not sys.stderr.isatty():
+        # bdns-fetch's pagination progress bar writes bare `\r` updates with
+        # no trailing newline, meant for an interactive terminal. Piped to a
+        # log file (cron, background runs) that leaves a stale progress
+        # fragment glued to the front of the next log line. No public knob
+        # on BDNSClient to disable it, so silence it here instead. Only
+        # applies when output isn't a real terminal; interactive use keeps it.
+        _bdns_fetch_client.tqdm = lambda iterable, *args, **kwargs: iterable
+
+
+TARGET_URL_OPTION = typer.Option(
+    ...,
+    "--target-url",
+    envvar="BDNS_SYNC_TARGET_URL",
+    help="SQLAlchemy target DB URL (e.g. bigquery://project/dataset).",
+)
+
+
+def _parse_iso_date(value: Optional[str], flag: str) -> Optional[date]:
+    if value is None:
+        return None
+    try:
+        return date.fromisoformat(value)
+    except ValueError:
+        raise typer.BadParameter(f"{flag} must be an ISO date (YYYY-MM-DD), got {value!r}") from None
+
+
+@app.command()
+def sync(
+    endpoint: str = typer.Argument(..., help="Endpoint/entity name to sync."),
+    target_url: str = TARGET_URL_OPTION,
+    window: str = typer.Option(
+        None,
+        "--window",
+        help=f"Cascade window for incremental endpoints: one of {', '.join(WINDOWS)}.",
+    ),
+    since: str = typer.Option(
+        None,
+        "--since",
+        help="Backfill start date (YYYY-MM-DD) for incremental endpoints. "
+        "Overrides --window; syncs [since, until]. See scripts/full_load.sh.",
+    ),
+    until: str = typer.Option(
+        None,
+        "--until",
+        help="Backfill end date (YYYY-MM-DD). Defaults to yesterday. Only with --since.",
+    ),
+) -> None:
+    """Sync one endpoint.
+
+    Incremental endpoints (the big search endpoints plus convocatorias) need
+    a reg-date range: either a cascade `--window` (daily/weekly/monthly/annual)
+    or an explicit `--since [--until]` backfill range. Full-replace endpoints
+    ignore all of these.
+    """
+    sink = get_sink(target_url)
+    # Defaults (3 retries, 2s fixed wait) give up after ~1 minute of server
+    # trouble; a real multi-hour backfill died live to one request timing
+    # out 3 times in a row. 8 x 15s rides out a ~2-minute server rough patch;
+    # the only cost is extra delay before a genuinely permanent failure.
+    client = BDNSClient(max_retries=8, wait_time=15)
+
+    since_date = _parse_iso_date(since, "--since")
+    until_date = _parse_iso_date(until, "--until")
+
+    # Outcome (row counts, duration) is logged by bookkeeping.run_with_bookkeeping;
+    # no separate echo here to avoid printing the same summary twice.
+    if endpoint in SEARCH_SYNCERS:
+        sync_fn = SEARCH_SYNCERS[endpoint]
+        if since_date is not None:
+            if window is not None:
+                raise typer.BadParameter("use either --window or --since, not both")
+            if until_date is not None and until_date < since_date:
+                raise typer.BadParameter("--until must not be before --since")
+            sync_fn(sink, client, since=since_date, until=until_date)
+        elif window is not None:
+            if window not in WINDOWS:
+                raise typer.BadParameter(f"window must be one of {', '.join(WINDOWS)}")
+            sync_fn(sink, client, window)
+        else:
+            raise typer.BadParameter(f"{endpoint} requires --window or --since")
+    elif endpoint in FULL_SYNCERS:
+        FULL_SYNCERS[endpoint](sink, client)
+    else:
+        raise typer.BadParameter(f"unknown endpoint: {endpoint}")
+
+
+@app.command(name="list")
+def list_endpoints(
+    kind: str = typer.Option("full", "--kind", help="one of: full, search"),
+) -> None:
+    """List known endpoint names, one per line, for scripting (no hardcoded lists)."""
+    if kind == "full":
+        for name in FULL_SYNCERS:
+            typer.echo(name)
+    elif kind == "search":
+        for name in SEARCH_SYNCERS:
+            typer.echo(name)
+    else:
+        raise typer.BadParameter("kind must be one of: full, search")

bdns/sync/generic.py

@@ -0,0 +1,233 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+"""Shared sync shapes reused by several entity modules. Each entity module
+still owns its own name, key fields, and any real one-off logic. Only the
+mechanical "fetch, then apply" plumbing lives here.
+"""
+
+import inspect
+import logging
+from collections.abc import Iterator, Sequence
+from datetime import date, timedelta
+from typing import Optional
+
+from bdns.fetch import BDNSClient
+from bdns.sync.sinks import Sink
+
+logger = logging.getLogger(__name__)
+logger.addHandler(logging.NullHandler())
+
+
+def all_pages(fetch):
+    """Wrap a client fetch method so paginated endpoints return EVERY page.
+
+    bdns-fetch's `num_pages` option defaults to 1, which silently truncates
+    any response bigger than one page (pageSize max 10,000). Caught live:
+    grandesbeneficiarios_busqueda returned 10,000 of 142,260 rows. Passing
+    `num_pages=0` ("all pages") on every call is the fix, but only the
+    paginated methods accept the parameter, so it's added by signature
+    inspection instead of blindly.
+    """
+    params = inspect.signature(fetch).parameters
+    if "num_pages" not in params:
+        return fetch
+
+    def fetch_all(*args, **kwargs):
+        kwargs.setdefault("num_pages", 0)
+        return fetch(*args, **kwargs)
+
+    return fetch_all
+
+# window name -> reg-date window size in days. Shared by every entity that
+# does cascading re-verification (concesiones, ayudasestado, minimis,
+# partidospoliticos, convocatorias).
+WINDOWS: dict[str, int] = {
+    "daily": 1,
+    "weekly": 7,
+    "monthly": 30,
+    "annual": 365,
+}
+
+# Every reg-date fetch is chunked into pieces this wide, regardless of the
+# requested window. Live-tested against concesiones_busqueda: a 7-day range
+# pulled cleanly (147,856 rows, 0 errors), while a 4-year range failed
+# intermittently with ERR_MANTENIMIENTO_BBDD at every page depth. `daily`
+# and `weekly` are already this size or smaller, so chunking is a no-op for
+# them; `monthly` and `annual` are the ones this actually protects.
+CHUNK_DAYS = 7
+
+
+def iter_date_chunks(start: date, end: date, chunk_days: int = CHUNK_DAYS) -> Iterator[tuple[date, date]]:
+    """Split [start, end] (inclusive) into chunks of at most `chunk_days`,
+    contiguous and non-overlapping. Yields (start, end) is always at least
+    one item even if `start == end`.
+    """
+    current = start
+    while current <= end:
+        chunk_end = min(current + timedelta(days=chunk_days - 1), end)
+        yield current, chunk_end
+        current = chunk_end + timedelta(days=1)
+
+
+def to_api_upper_bound(inclusive_end: date) -> date:
+    """Translate an inclusive end date (how windows and chunks are expressed
+    everywhere else in this codebase) into the value the four `fechaRegFin`
+    search endpoints expect.
+
+    Those endpoints (concesiones/ayudasestado/minimis/partidospoliticos)
+    treat `fechaRegFin` as EXCLUSIVE: it matches registrations strictly
+    before 00:00 of the given date, so it does NOT include the date's own
+    day. Confirmed live: querying `fechaRegFin=D` returned 0 rows for day D
+    (concesiones leaked a single midnight-exact row), while `fechaRegFin=D+1`
+    returned the whole day (~58,000 for concesiones). Left as a bare
+    `fechaRegFin=end`, `daily` (start == end) fetches almost nothing, and
+    every wider window silently drops its most recent day per chunk.
+
+    To include every registration made on `inclusive_end`, ask the API for
+    the day after. This function is the only place that crosses from the
+    codebase's inclusive convention to the API's half-open one. Keep it
+    here, not inlined as a `+ 1` at call sites.
+
+    IMPORTANT: this is NOT universal. convocatorias' discovery uses a
+    different parameter, `fechaHasta`, which is INCLUSIVE (also confirmed
+    live: `fechaHasta=D` returns the full day D). That path must NOT call
+    this function; see `syncers.discover_convocatoria_codes`.
+    """
+    return inclusive_end + timedelta(days=1)
+
+
+def sync_full_catalog(
+    sink: Sink, client: BDNSClient, endpoint_name: str, fetch_method_name: str, key_fields: Sequence[str]
+) -> dict[str, int]:
+    """Fetch everything with one no-arg call, full-reconcile every run."""
+    fetch = all_pages(getattr(client, fetch_method_name))
+    return sink.sync_full(endpoint_name, fetch(), key_fields)
+
+
+def sync_swept_catalog(
+    sink: Sink,
+    client: BDNSClient,
+    endpoint_name: str,
+    fetch_method_name: str,
+    sweep_param: str,
+    sweep_values: Sequence[str],
+    key_fields: Sequence[str],
+) -> dict[str, int]:
+    """Sweep `sweep_param` across `sweep_values`, merging into one table
+    before reconciling. Reconciling per sweep value would wrongly close out
+    the other values' rows as "missing". The sweep value is tagged onto the
+    payload under `sweep_param` since the API doesn't echo it back.
+    """
+    fetch = all_pages(getattr(client, fetch_method_name))
+
+    def rows():
+        for value in sweep_values:
+            for item in fetch(**{sweep_param: value}):
+                item = dict(item)
+                item[sweep_param] = value
+                yield item
+
+    return sink.sync_full(endpoint_name, rows(), key_fields)
+
+
+def window_bounds(window: str) -> tuple[date, date]:
+    """Map a cascade window name to its inclusive `[start, end]` range. `end`
+    is always yesterday (see `to_api_upper_bound` for why today is excluded).
+    """
+    days = WINDOWS[window]
+    end = date.today() - timedelta(days=1)
+    return end - timedelta(days=days - 1), end
+
+
+def resolve_when(
+    window: Optional[str], since: Optional[date], until: Optional[date]
+) -> tuple[date, date, str]:
+    """Turn the two ways of asking for a reg-date range (a named cascade
+    `window`, or an explicit `since`/`until` backfill range) into a single
+    `(start, end, run_type)` triple.
+
+    An explicit `since` wins over `window` and marks the run as "backfill"
+    in `_sync_runs` (`until` defaults to yesterday). This is what lets the
+    tool stay a pure primitive: cadence and history bounds are the caller's
+    business (`scripts/`), the engine just syncs whatever range it's told.
+    """
+    if since is not None:
+        end = until if until is not None else date.today() - timedelta(days=1)
+        return since, end, "backfill"
+    if window is not None:
+        start, end = window_bounds(window)
+        return start, end, window
+    raise ValueError("a reg-date sync needs either a window or a since date")
+
+
+def sync_search_range(
+    sink: Sink,
+    client: BDNSClient,
+    endpoint_name: str,
+    fetch_method_name: str,
+    key_fields: Sequence[str],
+    start: date,
+    end: date,
+    run_type: str,
+    reg_date_field: Optional[str] = None,
+) -> dict[str, int]:
+    """Fetch the reg-date range `[start, end]` and apply incrementally. Used
+    for both cascade windows (a few days back) and backfills (years back);
+    same machinery, only the range and `run_type` label differ.
+
+    By default this never closes out keys, since a range is a subset of the
+    table, not its full current state. `reg_date_field` opts into
+    window-scoped deletion detection (see `scd2.apply_incremental`); only
+    entities that expose their own registration date can use it, confirmed
+    live per entity.
+
+    The fetch is chunked into `CHUNK_DAYS`-wide pieces (see `iter_date_chunks`
+    and `to_api_upper_bound`). `window_start`/`window_end` given to
+    `apply_incremental` still span the whole `[start, end]`, since deletion
+    scoping cares about the full range, not how it was split to fetch it.
+    """
+    fetch = all_pages(getattr(client, fetch_method_name))
+
+    def rows():
+        for chunk_start, chunk_end in iter_date_chunks(start, end):
+            logger.info("%s: chunk [%s .. %s]", endpoint_name, chunk_start, chunk_end)
+            yield from fetch(
+                fechaRegInicio=chunk_start, fechaRegFin=to_api_upper_bound(chunk_end)
+            )
+
+    return sink.sync_window(
+        endpoint_name, rows(), key_fields,
+        window_start=start, window_end=end, run_type=run_type, reg_date_field=reg_date_field,
+    )
+
+
+def sync_search_range_inclusive(
+    sink: Sink,
+    client: BDNSClient,
+    endpoint_name: str,
+    fetch_method_name: str,
+    key_fields: Sequence[str],
+    start: date,
+    end: date,
+    run_type: str,
+    reg_date_field: Optional[str] = None,
+) -> dict[str, int]:
+    """Same shape as `sync_search_range`, for the OTHER date-parameter family:
+    `fechaDesde`/`fechaHasta`, which is INCLUSIVE on the upper bound (unlike
+    `fechaRegFin`; see `to_api_upper_bound` and section 2 of
+    docs/bdns-api-behavior.md).
+    `chunk_end` is passed as-is, with NO `to_api_upper_bound` bridge.
+    Calling it here would over-fetch one extra day past the window.
+    """
+    fetch = all_pages(getattr(client, fetch_method_name))
+
+    def rows():
+        for chunk_start, chunk_end in iter_date_chunks(start, end):
+            logger.info("%s: chunk [%s .. %s]", endpoint_name, chunk_start, chunk_end)
+            yield from fetch(fechaDesde=chunk_start, fechaHasta=chunk_end)
+
+    return sink.sync_window(
+        endpoint_name, rows(), key_fields,
+        window_start=start, window_end=end, run_type=run_type, reg_date_field=reg_date_field,
+    )
+

bdns/sync/hashing.py

@@ -0,0 +1,49 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+"""Canonical JSON, row hashing, and natural key derivation for SCD2 versioning."""
+
+import hashlib
+import json
+from collections.abc import Iterable, Sequence
+from typing import Any, Optional
+
+
+def _order_independent(value: Any) -> Any:
+    """Recursively sort list elements so hashing doesn't care about array
+    order. Confirmed live on `regiones`: the API returns the same tree
+    `children` in a different element order across calls, with no field
+    actually changed. Without this normalization, every re-sync produced a
+    spurious SCD2 version for any key with a reordered nested array.
+
+    Dict key order is already handled by `json.dumps(sort_keys=True)`
+    (recursively); only list element order needs normalizing here, by
+    sorting on each element's own canonical JSON string.
+    """
+    if isinstance(value, dict):
+        return {k: _order_independent(v) for k, v in value.items()}
+    if isinstance(value, list):
+        items = [_order_independent(v) for v in value]
+        return sorted(items, key=lambda v: json.dumps(v, sort_keys=True, ensure_ascii=False, default=str))
+    return value
+
+
+def canonical_json(
+    payload: dict[str, Any], exclude_fields: Optional[Iterable[str]] = None
+) -> str:
+    if exclude_fields:
+        excluded = set(exclude_fields)
+        payload = {k: v for k, v in payload.items() if k not in excluded}
+    return json.dumps(
+        _order_independent(payload), sort_keys=True, separators=(",", ":"), ensure_ascii=False, default=str
+    )
+
+
+def row_hash(payload: dict[str, Any], exclude_fields: Optional[Iterable[str]] = None) -> str:
+    digest = canonical_json(payload, exclude_fields).encode("utf-8")
+    return hashlib.sha256(digest).hexdigest()
+
+
+def natural_key(payload: dict[str, Any], key_fields: Sequence[str]) -> str:
+    """Build a stable string key from one or more fields (composite keys supported)."""
+    values = [payload[field] for field in key_fields]
+    return json.dumps(values, separators=(",", ":"), ensure_ascii=False, default=str)

bdns/sync/pipeline.py

@@ -0,0 +1,129 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+"""Concurrency helpers shared by the fetch and storage code.
+
+Nothing here knows about the BDNS API or SQL. Callers pass plain
+iterables and callables, which also keeps these functions easy to test.
+"""
+
+import concurrent.futures
+import itertools
+import queue
+import threading
+import time
+from collections.abc import Callable, Iterable, Iterator
+from typing import Any
+
+
+def chunked(items: Iterable[Any], chunk_size: int) -> Iterator[list[Any]]:
+    """Group `items` into lists of at most `chunk_size`. Pure: no threads."""
+    chunk = []
+    for item in items:
+        chunk.append(item)
+        if len(chunk) >= chunk_size:
+            yield chunk
+            chunk = []
+    if chunk:
+        yield chunk
+
+
+def prefetch(iterable: Iterable[Any]) -> Iterator[Any]:
+    """Yield the items of `iterable`, pulling them on a helper thread that
+    reads ahead of the caller.
+
+    While the caller processes one item, the helper is already producing
+    the next. The queue holds at most two items: if the caller falls
+    behind, the helper blocks instead of filling memory.
+
+    The caller does its work on its own thread. That matters for SQLite
+    connections, which must stay on the thread that created them.
+
+    If the helper raises, the exception is re-raised here. If the caller
+    stops iterating early, the helper is unblocked and joined before the
+    generator exits.
+    """
+    item_queue: queue.Queue = queue.Queue(maxsize=2)
+    done = object()
+    consumer_gone = threading.Event()
+
+    def put(item) -> bool:
+        # Re-check every second whether the consumer went away. Without
+        # this, a full queue would block the helper thread forever.
+        while not consumer_gone.is_set():
+            try:
+                item_queue.put(item, timeout=1)
+                return True
+            except queue.Full:
+                continue
+        return False
+
+    def read_ahead():
+        try:
+            for item in iterable:
+                if not put(item):
+                    return  # consumer is gone, nothing left to do
+            put(done)
+        except Exception as exc:
+            put(exc)  # re-raised on the caller's thread below
+
+    helper = threading.Thread(target=read_ahead, daemon=True)
+    helper.start()
+    try:
+        while True:
+            item = item_queue.get()
+            if item is done:
+                return
+            if isinstance(item, Exception):
+                raise item
+            yield item
+    finally:
+        consumer_gone.set()
+        helper.join()
+
+
+def rate_limited_map(
+    keys: Iterable[Any],
+    fn: Callable[[Any], Any],
+    spacing_seconds: float,
+    max_workers: int,
+) -> Iterator[tuple[Any, Any]]:
+    """Run `fn(key)` on a thread pool and yield `(key, result)` as calls
+    finish.
+
+    Call starts are spaced at least `spacing_seconds` apart across all
+    workers. Rate-limited servers reject bursts, not averages: a fresh
+    pool firing all its workers at once gets 429s even when the average
+    rate is fine. With spaced starts, `max_workers` only needs to be
+    large enough to cover call latency.
+
+    At most `2 * max_workers` calls are submitted ahead of the consumer,
+    so a large key set never piles up results in memory. If `fn` raises,
+    the exception propagates and stops the iteration.
+    """
+    lock = threading.Lock()
+    next_start = 0.0
+
+    def run_one(key):
+        nonlocal next_start
+        with lock:
+            now = time.monotonic()
+            wait = max(0.0, next_start - now)
+            next_start = now + wait + spacing_seconds
+        if wait:
+            time.sleep(wait)
+        return key, fn(key)
+
+    keys_iter = iter(keys)
+    with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
+        pending = {
+            executor.submit(run_one, key)
+            for key in itertools.islice(keys_iter, max_workers * 2)
+        }
+        while pending:
+            finished, pending = concurrent.futures.wait(
+                pending, return_when=concurrent.futures.FIRST_COMPLETED
+            )
+            for key in itertools.islice(keys_iter, len(finished)):
+                pending.add(executor.submit(run_one, key))
+            for future in finished:
+                yield future.result()