osmsg 1.2.4__tar.gz → 1.3.0__tar.gz

{osmsg-1.2.4 → osmsg-1.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: osmsg
-Version: 1.2.4
+Version: 1.3.0
 Summary: OpenStreetMap Stats Generator: Commandline
 Keywords: osm,stats,commandline,openstreetmap
 Author: Kshitij Raj Sharma
@@ -178,7 +178,8 @@ litestar --app api.app:app run --host 0.0.0.0 --port 8000
 
 ```text
 GET /health
-GET /api/v1/user-stats?start=2026-05-01T00:00:00Z&end=2026-05-02T00:00:00Z
+GET /api/v2/hashtag/hotosm/summary
+GET /api/v2/hashtag/hotosm/leaderboard
 GET /docs
 ```
 
@@ -230,7 +231,7 @@ docker-compose `environment:` block all reach the same setting. CLI flag wins ov
 | `--country` | `OSMSG_COUNTRY` | unset | Geofabrik region id(s). Comma-separated when set via env. |
 | `--boundary` | `OSMSG_BOUNDARY` | unset | GeoJSON path or inline GeoJSON. |
 | `--url` | `OSMSG_URL` | `minute` | `minute`/`hour`/`day` shortcut or full URL. Comma-separated when set via env. |
-| `--workers` | `OSMSG_WORKERS` | cpu count | Parallel workers. |
+| `--workers` | `OSMSG_WORKERS` | cpu count | Parallel parse workers. |
 | `--cache-dir` | `OSMSG_CACHE_DIR` | platform cache | Where downloaded OSM files are kept across runs. |
 | `--output-dir` | `OSMSG_OUTPUT_DIR` | `.` | Where `<name>.duckdb` and exports are written. |
 | `--format` / `-f` | `OSMSG_FORMAT` | `parquet` | Repeat for multiple. Comma-separated when set via env. |

{osmsg-1.2.4 → osmsg-1.3.0}/README.md

@@ -146,7 +146,8 @@ litestar --app api.app:app run --host 0.0.0.0 --port 8000
 
 ```text
 GET /health
-GET /api/v1/user-stats?start=2026-05-01T00:00:00Z&end=2026-05-02T00:00:00Z
+GET /api/v2/hashtag/hotosm/summary
+GET /api/v2/hashtag/hotosm/leaderboard
 GET /docs
 ```
 
@@ -198,7 +199,7 @@ docker-compose `environment:` block all reach the same setting. CLI flag wins ov
 | `--country` | `OSMSG_COUNTRY` | unset | Geofabrik region id(s). Comma-separated when set via env. |
 | `--boundary` | `OSMSG_BOUNDARY` | unset | GeoJSON path or inline GeoJSON. |
 | `--url` | `OSMSG_URL` | `minute` | `minute`/`hour`/`day` shortcut or full URL. Comma-separated when set via env. |
-| `--workers` | `OSMSG_WORKERS` | cpu count | Parallel workers. |
+| `--workers` | `OSMSG_WORKERS` | cpu count | Parallel parse workers. |
 | `--cache-dir` | `OSMSG_CACHE_DIR` | platform cache | Where downloaded OSM files are kept across runs. |
 | `--output-dir` | `OSMSG_OUTPUT_DIR` | `.` | Where `<name>.duckdb` and exports are written. |
 | `--format` / `-f` | `OSMSG_FORMAT` | `parquet` | Repeat for multiple. Comma-separated when set via env. |

osmsg-1.3.0/osmsg/__version__.py

@@ -0,0 +1 @@
+__version__ = "1.3.0"

{osmsg-1.2.4 → osmsg-1.3.0}/osmsg/_tick.py

@@ -22,6 +22,20 @@ def _has_state(db_path: Path, source_url: str) -> bool:
     return result
 
 
+def _has_any_state(db_path: Path) -> bool:
+    """True if the store has a resume position for ANY source. A `--insert --seed-only` seeds the source
+    at the granularity matched to the gap (e.g. day), which differs from this tick's default source
+    (minute); `--update` reads the store's own tracked source and continues/auto-refines it, so any
+    seeded state means we continue rather than bootstrap to now."""
+    if not db_path.exists():
+        return False
+    conn = connect(str(db_path))
+    create_tables(conn)
+    row = conn.execute("SELECT count(*) FROM state").fetchone()
+    conn.close()
+    return bool(row and row[0])
+
+
 def _parse_arg(args: list[str], flag: str) -> str | None:
     for i, arg in enumerate(args):
         if arg == flag and i + 1 < len(args):
@@ -31,13 +45,12 @@ def _parse_arg(args: list[str], flag: str) -> str | None:
 
 def main() -> int:
     extra_args = shlex.split(os.environ.get("OSMSG_EXTRA_ARGS", ""))
-    bootstrap = os.environ.get("OSMSG_BOOTSTRAP", "hour")
-    bootstrap_days = os.environ.get("OSMSG_BOOTSTRAP_DAYS")
+    bootstrap_days = os.environ.get("OSMSG_BOOTSTRAP_DAYS", "1")
     name = _parse_arg(extra_args, "--name") or "stats"
     out = Path(_parse_arg(extra_args, "--output-dir") or "/var/lib/osmsg")
     country = _parse_arg(extra_args, "--country")
     explicit_url = _parse_arg(extra_args, "--url")
-    url = explicit_url or "minute"
+    url = explicit_url or "day"
 
     out.mkdir(parents=True, exist_ok=True)
 
@@ -61,12 +74,17 @@ def main() -> int:
         if not (extra_set & {"--all", "--keys"}):
             cmd.append("--all")
 
-        if _has_state(db_path, source_url):
+        # A country run tracks one specific geofabrik source; the planet run continues whatever the store
+        # was seeded with (possibly a coarser source than this tick's default), so accept any state.
+        has_state = _has_state(db_path, source_url) if country else _has_any_state(db_path)
+        if has_state:
             cmd.append("--update")
-        elif bootstrap_days:
-            cmd.extend(["--days", bootstrap_days])
         else:
-            cmd.extend(["--last", bootstrap])
+            # Cold start at day granularity (coarse, few files); --update then refines day->hour->minute
+            # as the backlog shrinks. A normal deployment seeds the same way via `--insert`.
+            if explicit_url is None and not country:
+                cmd.extend(["--url", "day"])
+            cmd.extend(["--days", bootstrap_days])
 
         print(f"[osmsg-tick] {' '.join(cmd)}", flush=True)
         return subprocess.call(cmd)

osmsg-1.3.0/osmsg/catalog.py

@@ -0,0 +1,212 @@
+"""Assemble query sources into one relation so `stats.py` runs a single query regardless of where
+data lives. History (older than the published frontier) comes from the `hashtag_changeset` rollup;
+the recent tail (from the frontier on) is derived on the fly from the base tables, filtered by the
+hashtag first so only matching changesets are read. The two are unioned and split at the frontier, so
+nothing is counted twice and nothing is dropped, and the recent side is always as fresh as the base.
+
+The base is just relations: a local DuckDB store's `changeset_stats`/`changesets`, or an attached
+Postgres's `pg.changeset_stats`/`pg.changesets`. Either way the query runs in DuckDB.
+"""
+
+from __future__ import annotations
+
+import datetime as dt
+
+from .stats import COUNT_COLS
+
+_PROJECTION = f"changeset_id, uid, editor, created_at, {', '.join(COUNT_COLS)}, tags"
+
+
+def _window_clause(start: dt.datetime | None, end: dt.datetime | None) -> tuple[str, list[object]]:
+    """Optional `created_at` bounds as SQL fragments plus their params, in order. Half-open [start, end):
+    inclusive lower, exclusive upper, matching the frontier split and the v1 API's window semantics."""
+    parts: list[str] = []
+    params: list[object] = []
+    if start is not None:
+        parts.append(" AND created_at >= ?")
+        params.append(start)
+    if end is not None:
+        parts.append(" AND created_at < ?")
+        params.append(end)
+    return "".join(parts), params
+
+
+def _recent_from_base(stats_rel: str, changesets_rel: str, window_sql: str, prefix_pred: str) -> str:
+    """Per-changeset rows built on the fly from the base tables for changesets whose hashtags match any
+    requested prefix and are on/after the frontier. Counts are summed across a changeset's seq rows and
+    the native `tags` lists merged, matching the rollup shape exactly (one tag representation, one
+    breakdown path). The changeset filter runs first, so only matching changesets are read. `prefix_pred`
+    is the OR of `lower(h) >= ? AND lower(h) < ?` range tests on the lambda var. Placeholders, in order:
+    frontier, (window bounds), (prefix bounds)."""
+    sums = ", ".join(f"SUM(s.{c}) AS {c}" for c in COUNT_COLS)
+    payload = ", ".join(f"p.{c}" for c in COUNT_COLS)
+    return f"""
+        WITH matched AS (
+            SELECT changeset_id, uid, editor, created_at
+            FROM {changesets_rel}
+            WHERE created_at >= ?{window_sql} AND len(list_filter(hashtags, h -> {prefix_pred})) > 0
+        ),
+        per_cs AS (
+            SELECT s.changeset_id, {sums}
+            FROM {stats_rel} s JOIN matched USING (changeset_id) GROUP BY s.changeset_id
+        ),
+        tag_rows AS (
+            SELECT changeset_id, t.k AS k, t.v AS v,
+                   SUM(t.c) AS c, SUM(t.m) AS m, SUM(t.len_m) AS len_m
+            FROM (
+                SELECT s.changeset_id, UNNEST(s.tags) AS t
+                FROM {stats_rel} s JOIN matched USING (changeset_id)
+                WHERE s.tags IS NOT NULL AND len(s.tags) > 0
+            )
+            GROUP BY changeset_id, t.k, t.v
+        ),
+        tags AS (
+            SELECT changeset_id, list(struct_pack(k := k, v := v, c := c, m := m, len_m := len_m)) AS tags
+            FROM tag_rows GROUP BY changeset_id
+        )
+        SELECT m.changeset_id, m.uid, m.editor, m.created_at, {payload}, COALESCE(t.tags, []) AS tags
+        FROM matched m JOIN per_cs p USING (changeset_id) LEFT JOIN tags t USING (changeset_id)
+    """
+
+
+def hashtag_scope(
+    history_rel: str,
+    recent_stats_rel: str,
+    recent_changesets_rel: str,
+    *,
+    prefixes: list[tuple[str, str]],
+    frontier: dt.datetime,
+    start: dt.datetime | None = None,
+    end: dt.datetime | None = None,
+) -> tuple[str, list[object]]:
+    """One deduped per-changeset relation for one or more hashtag prefix ranges [lo, hi): the rollup for
+    history (created_at < frontier, hashtag range prunes row groups) unioned with the recent tail
+    derived from the base (created_at >= frontier). A changeset matching more than one prefix (or
+    carrying two matching hashtags) is deduped by changeset_id, so it counts once. An optional half-open
+    [start, end) window bounds created_at on both sides, so it intersects the frontier split cleanly (a
+    window entirely before the frontier reads only history; entirely after, only the base). `prefixes`
+    must be non-empty. Returns (sql, params)."""
+    if not prefixes:
+        raise ValueError("prefixes must be non-empty")
+    window_sql, window_params = _window_clause(start, end)
+    prefix_params = [bound for pair in prefixes for bound in pair]
+    hist_pred = " OR ".join("(hashtag >= ? AND hashtag < ?)" for _ in prefixes)
+    recent_pred = " OR ".join("(lower(h) >= ? AND lower(h) < ?)" for _ in prefixes)
+    recent = _recent_from_base(recent_stats_rel, recent_changesets_rel, window_sql, recent_pred)
+    sql = f"""
+        SELECT DISTINCT ON (changeset_id) {_PROJECTION} FROM (
+            SELECT {_PROJECTION} FROM {history_rel}
+                WHERE ({hist_pred}) AND created_at < ?{window_sql}
+            UNION ALL
+            ({recent})
+        )
+    """
+    params: list[object] = [
+        *prefix_params,
+        frontier,
+        *window_params,
+        frontier,
+        *window_params,
+        *prefix_params,
+    ]
+    return sql, params
+
+
+def history_dedup_scope(
+    history_rel: str,
+    *,
+    prefixes: list[tuple[str, str]],
+    frontier: dt.datetime,
+    start: dt.datetime | None = None,
+    end: dt.datetime | None = None,
+) -> tuple[str, list[object]]:
+    """The deduped per-changeset HISTORY relation alone (created_at < frontier), as its own SELECT so a
+    caller can aggregate it and the recent side SEPARATELY and combine the small results. This avoids
+    DISTINCT ON over the history+recent UNION, which forces DuckDB to materialize the whole 14M-row
+    intermediate for a big hashtag; aggregating each side first keeps the history dedup streaming.
+    `prefixes` must be non-empty. Returns (sql, params)."""
+    if not prefixes:
+        raise ValueError("prefixes must be non-empty")
+    window_sql, window_params = _window_clause(start, end)
+    prefix_params = [bound for pair in prefixes for bound in pair]
+    hist_pred = " OR ".join("(hashtag >= ? AND hashtag < ?)" for _ in prefixes)
+    sql = (
+        f"SELECT DISTINCT ON (changeset_id) {_PROJECTION} FROM {history_rel} "
+        f"WHERE ({hist_pred}) AND created_at < ?{window_sql}"
+    )
+    return sql, [*prefix_params, frontier, *window_params]
+
+
+def history_scope_count(
+    history_rel: str,
+    *,
+    prefixes: list[tuple[str, str]],
+    frontier: dt.datetime,
+    start: dt.datetime | None = None,
+    end: dt.datetime | None = None,
+) -> tuple[str, list[object]]:
+    """A CHEAP raw `count(*)` of history rows matching the hashtag scope + window (no dedup, hashtag range
+    prunes row groups: ~sub-second even for a 14M-changeset hashtag). Used to decide whether a per-user
+    tag breakdown is affordable before paying for it. Returns (sql, params)."""
+    if not prefixes:
+        raise ValueError("prefixes must be non-empty")
+    window_sql, window_params = _window_clause(start, end)
+    prefix_params = [bound for pair in prefixes for bound in pair]
+    hist_pred = " OR ".join("(hashtag >= ? AND hashtag < ?)" for _ in prefixes)
+    sql = f"SELECT count(*) FROM {history_rel} WHERE ({hist_pred}) AND created_at < ?{window_sql}"
+    return sql, [*prefix_params, frontier, *window_params]
+
+
+def recent_scope(
+    recent_stats_rel: str,
+    recent_changesets_rel: str,
+    *,
+    prefixes: list[tuple[str, str]],
+    frontier: dt.datetime,
+    start: dt.datetime | None = None,
+    end: dt.datetime | None = None,
+) -> tuple[str, list[object]]:
+    """The recent per-changeset relation alone (created_at >= frontier), already one row per changeset.
+    Companion to `history_dedup_scope` for the aggregate-then-combine path. Returns (sql, params)."""
+    if not prefixes:
+        raise ValueError("prefixes must be non-empty")
+    window_sql, window_params = _window_clause(start, end)
+    prefix_params = [bound for pair in prefixes for bound in pair]
+    recent_pred = " OR ".join("(lower(h) >= ? AND lower(h) < ?)" for _ in prefixes)
+    sql = "(" + _recent_from_base(recent_stats_rel, recent_changesets_rel, window_sql, recent_pred) + ")"
+    return sql, [frontier, *window_params, *prefix_params]
+
+
+def map_scope(
+    history_rollup_rel: str,
+    recent_changesets_rel: str,
+    *,
+    prefixes: list[tuple[str, str]],
+    frontier: dt.datetime,
+    start: dt.datetime | None = None,
+    end: dt.datetime | None = None,
+) -> tuple[str, list[object]]:
+    """Deduped changeset centroids `(changeset_id, uid, lon, lat)` for the hashtag union, for the map.
+    History centroids come from the rollup's `lon`/`lat`; recent centroids from the base changesets bbox
+    midpoint (`recent_changesets_rel` must expose min_lon/min_lat/max_lon/max_lat, e.g. the published
+    changesets dataset or the Postgres `changesets` table). Changesets with no bbox are dropped. Same
+    frontier split and optional [start, end) window as `hashtag_scope`. `prefixes` must be non-empty."""
+    if not prefixes:
+        raise ValueError("prefixes must be non-empty")
+    window_sql, window_params = _window_clause(start, end)
+    prefix_params = [bound for pair in prefixes for bound in pair]
+    hist_pred = " OR ".join("(hashtag >= ? AND hashtag < ?)" for _ in prefixes)
+    recent_pred = " OR ".join("(lower(h) >= ? AND lower(h) < ?)" for _ in prefixes)
+    sql = f"""
+        SELECT DISTINCT ON (changeset_id) changeset_id, uid, lon, lat FROM (
+            SELECT changeset_id, uid, lon, lat FROM {history_rollup_rel}
+                WHERE ({hist_pred}) AND created_at < ?{window_sql} AND lon IS NOT NULL
+            UNION ALL
+            SELECT changeset_id, uid, (min_lon + max_lon) / 2.0 AS lon, (min_lat + max_lat) / 2.0 AS lat
+                FROM {recent_changesets_rel}
+                WHERE created_at >= ?{window_sql} AND min_lon IS NOT NULL
+                  AND len(list_filter(hashtags, h -> {recent_pred})) > 0
+        )
+    """
+    params: list[object] = [*prefix_params, frontier, *window_params, frontier, *window_params, *prefix_params]
+    return sql, params

{osmsg-1.2.4 → osmsg-1.3.0}/osmsg/cli.py

@@ -150,7 +150,7 @@ def main(
     ] = None,
     workers: Annotated[
         int | None,
-        typer.Option(envvar="OSMSG_WORKERS", help="Parallel workers (default: cpu count)."),
+        typer.Option(envvar="OSMSG_WORKERS", help="Parallel parse workers (default: cpu count)."),
     ] = None,
     rows: Annotated[
         int | None,
@@ -263,6 +263,14 @@ def main(
             "whole published history; --start/--end loads a slice. Follow with --update to catch up.",
         ),
     ] = False,
+    seed_only: Annotated[
+        bool,
+        typer.Option(
+            "--seed-only",
+            help="With --insert: seed resume state at the published frontier without ingesting any "
+            "history (it stays remote), so --update only ever fetches the live tail. The lightest setup.",
+        ),
+    ] = False,
     osh_file: Annotated[
         str | None,
         typer.Option("--osh-file", help="Insert from a local .osh.pbf instead of the published dataset."),
@@ -296,6 +304,9 @@ def main(
     if insert and update:
         error("--insert and --update are mutually exclusive; insert first, then update.")
         raise typer.Exit(code=2)
+    if seed_only and not insert:
+        error("--seed-only is only valid with --insert.")
+        raise typer.Exit(code=2)
     if insert and (last is not None or days is not None):
         error("--insert takes --start/--end (or no window), not --last/--days.")
         raise typer.Exit(code=2)
@@ -344,6 +355,7 @@ def main(
         history_mode="auto" if history else "off",
         history_url=history_url,
         insert=insert,
+        seed_only=seed_only,
         osh_file=osh_file,
         changeset_file=changeset_file,
         overwrite=overwrite,
@@ -395,10 +407,10 @@ def main(
             "name",
             "changesets",
             "map_changes",
-            "nodes_create",
-            "ways_create",
-            "rels_create",
-            "poi_create",
+            "nodes_created",
+            "ways_created",
+            "rels_created",
+            "poi_created",
             "hashtags",
         ),
         title=f"Top users (showing {display_n} of {result['rows']})",

{osmsg-1.2.4 → osmsg-1.3.0}/osmsg/db/duckdb_schema.py

@@ -1,4 +1,5 @@
 # No FKs: DuckDB rejects UPDATE on FK-referenced LIST/GEOMETRY columns, which would block changeset upgrades.
+# `tags` is the native tag breakdown (osmsg.stats.TAG_STRUCT_DDL); kept in sync with that constant.
 DUCKDB_SCHEMA = """
 CREATE TABLE IF NOT EXISTS users (
     uid      BIGINT PRIMARY KEY,
@@ -28,7 +29,7 @@ CREATE TABLE IF NOT EXISTS changeset_stats (
     rels_deleted   INTEGER DEFAULT 0,
     poi_created    INTEGER DEFAULT 0,
     poi_modified   INTEGER DEFAULT 0,
-    tag_stats      JSON,
+    tags           STRUCT(k VARCHAR, v VARCHAR, c BIGINT, m BIGINT, len_m DOUBLE)[],
     PRIMARY KEY (seq_id, changeset_id)
 );
 CREATE INDEX IF NOT EXISTS idx_changeset_stats_uid ON changeset_stats(uid);

{osmsg-1.2.4 → osmsg-1.3.0}/osmsg/db/ingest.py

@@ -10,6 +10,20 @@ import duckdb
 import pyarrow as pa
 import pyarrow.parquet as pq
 
+# Native shard column for the per-changeset tag breakdown, matching the store's
+# STRUCT(k VARCHAR, v VARCHAR, c BIGINT, m BIGINT, len_m DOUBLE)[] so ingest is a direct copy.
+_TAG_PA_TYPE = pa.list_(
+    pa.struct(
+        [
+            pa.field("k", pa.string()),
+            pa.field("v", pa.string()),
+            pa.field("c", pa.int64()),
+            pa.field("m", pa.int64()),
+            pa.field("len_m", pa.float64()),
+        ]
+    )
+)
+
 
 def _quarantine_corrupt(parquet_dir: Path) -> None:
     """Rename unreadable parquet shards out of the way so the bulk read doesn't abort."""
@@ -59,7 +73,7 @@ CHANGESET_STATS_SCHEMA = pa.schema(
         pa.field("rels_deleted", pa.int32()),
         pa.field("poi_created", pa.int32()),
         pa.field("poi_modified", pa.int32()),
-        pa.field("tag_stats", pa.string()),
+        pa.field("tags", _TAG_PA_TYPE),
     ]
 )
 
@@ -109,7 +123,12 @@ def merge_parquet_files(conn: duckdb.DuckDBPyConnection, parquet_dir: Path, *, c
         # read_parquet() takes a literal, escape so quoted paths can't break out.
         return _sql_escape((parquet_dir / f"temp_*_{name}_*.parquet").as_posix())
 
-    conn.execute("BEGIN")
+    # Each step auto-commits (no enclosing transaction): DuckDB keeps an uncommitted transaction's changes
+    # and the non-spillable PK index in memory until COMMIT, so wrapping a whole multi-day window in one
+    # transaction exhausts a constrained memory_limit. Every statement here is idempotent (INSERT OR IGNORE,
+    # COALESCE update), so committing per step - and re-merging leftover shards after a crash - is safe.
+    # preserve_insertion_order=false lets the merges stream instead of buffering rows to preserve order.
+    conn.execute("SET preserve_insertion_order = false")
     try:
         if any(parquet_dir.glob("temp_*_users_*.parquet")):
             conn.execute(f"INSERT OR IGNORE INTO users SELECT uid, username FROM read_parquet('{pattern('users')}')")
@@ -153,6 +172,8 @@ def merge_parquet_files(conn: duckdb.DuckDBPyConnection, parquet_dir: Path, *, c
                 """
             )
         if any(parquet_dir.glob("temp_*_changeset_stats_*.parquet")):
+            # The shard stores `tags` as a native LIST<STRUCT> (built in the handler), so ingest is a
+            # direct column copy.
             conn.execute(
                 f"""
                 INSERT OR IGNORE INTO changeset_stats
@@ -161,14 +182,12 @@ def merge_parquet_files(conn: duckdb.DuckDBPyConnection, parquet_dir: Path, *, c
                        ways_created,  ways_modified,  ways_deleted,
                        rels_created,  rels_modified,  rels_deleted,
                        poi_created,   poi_modified,
-                       tag_stats::JSON
+                       tags
                 FROM read_parquet('{pattern("changeset_stats")}')
                 """
             )
-        conn.execute("COMMIT")
-    except Exception:
-        conn.execute("ROLLBACK")
-        raise
+    finally:
+        conn.execute("SET preserve_insertion_order = true")
 
     if cleanup:
         shutil.rmtree(parquet_dir, ignore_errors=True)

{osmsg-1.2.4 → osmsg-1.3.0}/osmsg/db/queries.py

@@ -2,42 +2,41 @@
 
 from __future__ import annotations
 
-import json
 from typing import Any
 
 import duckdb
 
+from ..stats import map_changes_sum, sum_cols
+
 
 def _rows(result) -> list[dict[str, Any]]:
     cols = [d[0] for d in result.description]
     return [dict(zip(cols, r, strict=True)) for r in result.fetchall()]
 
 
+def _tags_to_nested(tags: list[dict[str, Any]] | None) -> dict[str, dict[str, dict[str, Any]]]:
+    """The native `tags` list (list of {k, v, c, m, len_m}) as the nested {key: {value: {c, m, len}}}
+    shape `_accumulate_tags` sums over. len is omitted when absent."""
+    out: dict[str, dict[str, dict[str, Any]]] = {}
+    for t in tags or []:
+        entry: dict[str, Any] = {"c": t["c"], "m": t["m"]}
+        if t["len_m"] is not None:
+            entry["len"] = t["len_m"]
+        out.setdefault(t["k"], {})[t["v"]] = entry
+    return out
+
+
 def user_stats(conn: duckdb.DuckDBPyConnection, top_n: int | None = None) -> list[dict[str, Any]]:
-    """One row per user, ranked by total map changes."""
+    """One row per user, ranked by total map changes. Counts come from the shared stats vocabulary."""
     rows = _rows(
         conn.execute(
-            """
+            f"""
             SELECT
                 u.uid,
                 u.username                                  AS name,
                 COUNT(DISTINCT cs.changeset_id)             AS changesets,
-                SUM(cs.nodes_created)                       AS nodes_create,
-                SUM(cs.nodes_modified)                      AS nodes_modify,
-                SUM(cs.nodes_deleted)                       AS nodes_delete,
-                SUM(cs.ways_created)                        AS ways_create,
-                SUM(cs.ways_modified)                       AS ways_modify,
-                SUM(cs.ways_deleted)                        AS ways_delete,
-                SUM(cs.rels_created)                        AS rels_create,
-                SUM(cs.rels_modified)                       AS rels_modify,
-                SUM(cs.rels_deleted)                        AS rels_delete,
-                SUM(cs.poi_created)                         AS poi_create,
-                SUM(cs.poi_modified)                        AS poi_modify,
-                SUM(
-                    cs.nodes_created + cs.nodes_modified + cs.nodes_deleted +
-                    cs.ways_created  + cs.ways_modified  + cs.ways_deleted  +
-                    cs.rels_created  + cs.rels_modified  + cs.rels_deleted
-                )                                           AS map_changes
+                {sum_cols("cs")},
+                {map_changes_sum("cs")}
             FROM users u
             JOIN changeset_stats cs ON u.uid = cs.uid
             GROUP BY u.uid, u.username
@@ -121,7 +120,7 @@ def attach_tag_stats(
     tag_mode: str = "none",
     length_tags: list[str] | None = None,
 ) -> None:
-    """In-place: parse the JSON tag_stats column once per row, then aggregate per user."""
+    """In-place: read the native `tags` list column once per row, then aggregate per user."""
     if not rows:
         return
     if not (additional_tags or tag_mode != "none" or length_tags):
@@ -138,18 +137,14 @@ def attach_tag_stats(
         for k in length_tags or []:
             r.setdefault(f"{k}_len_m", 0)
 
-    for uid, tag_stats_json in conn.execute(
-        "SELECT uid, tag_stats FROM changeset_stats WHERE tag_stats IS NOT NULL"
+    for uid, tags in conn.execute(
+        "SELECT uid, tags FROM changeset_stats WHERE tags IS NOT NULL AND len(tags) > 0"
     ).fetchall():
-        if uid not in by_uid or not tag_stats_json:
-            continue
-        try:
-            payload = json.loads(tag_stats_json) if isinstance(tag_stats_json, str) else tag_stats_json
-        except (json.JSONDecodeError, TypeError):
+        if uid not in by_uid or not tags:
             continue
         _accumulate_tags(
             by_uid[uid],
-            payload,
+            _tags_to_nested(tags),
             additional_tags=additional_tags,
             tag_mode=tag_mode,
             length_tags=length_tags,
@@ -171,27 +166,13 @@ def daily_summary(
     """One row per UTC day. Requires `changesets` populated (--changeset / --hashtags)."""
     rows = _rows(
         conn.execute(
-            """
+            f"""
             SELECT
                 CAST(DATE_TRUNC('day', cs.created_at) AS DATE)::VARCHAR AS date,
                 COUNT(DISTINCT cs.changeset_id) AS changesets,
                 COUNT(DISTINCT cs.uid)          AS users,
-                SUM(st.nodes_created)           AS nodes_create,
-                SUM(st.nodes_modified)          AS nodes_modify,
-                SUM(st.nodes_deleted)           AS nodes_delete,
-                SUM(st.ways_created)            AS ways_create,
-                SUM(st.ways_modified)           AS ways_modify,
-                SUM(st.ways_deleted)            AS ways_delete,
-                SUM(st.rels_created)            AS rels_create,
-                SUM(st.rels_modified)           AS rels_modify,
-                SUM(st.rels_deleted)            AS rels_delete,
-                SUM(st.poi_created)             AS poi_create,
-                SUM(st.poi_modified)            AS poi_modify,
-                SUM(
-                    st.nodes_created + st.nodes_modified + st.nodes_deleted +
-                    st.ways_created  + st.ways_modified  + st.ways_deleted  +
-                    st.rels_created  + st.rels_modified  + st.rels_deleted
-                )                               AS map_changes
+                {sum_cols("st")},
+                {map_changes_sum("st")}
             FROM changesets cs
             JOIN changeset_stats st ON cs.changeset_id = st.changeset_id
             GROUP BY DATE_TRUNC('day', cs.created_at)
@@ -227,22 +208,18 @@ def daily_summary(
         for k in length_tags or []:
             r.setdefault(f"{k}_len_m", 0)
 
-    for date, tag_stats_json in conn.execute(
+    for date, tags in conn.execute(
         """
-        SELECT CAST(DATE_TRUNC('day', cs.created_at) AS DATE)::VARCHAR, st.tag_stats
+        SELECT CAST(DATE_TRUNC('day', cs.created_at) AS DATE)::VARCHAR, st.tags
         FROM changesets cs JOIN changeset_stats st ON cs.changeset_id = st.changeset_id
-        WHERE st.tag_stats IS NOT NULL
+        WHERE st.tags IS NOT NULL AND len(st.tags) > 0
         """
     ).fetchall():
-        if date not in by_date or not tag_stats_json:
-            continue
-        try:
-            payload = json.loads(tag_stats_json) if isinstance(tag_stats_json, str) else tag_stats_json
-        except (json.JSONDecodeError, TypeError):
+        if date not in by_date or not tags:
             continue
         _accumulate_tags(
             by_date[date],
-            payload,
+            _tags_to_nested(tags),
             additional_tags=additional_tags,
             tag_mode=tag_mode,
             length_tags=length_tags,

{osmsg-1.2.4 → osmsg-1.3.0}/osmsg/db/schema.py

@@ -1,14 +1,41 @@
 from __future__ import annotations
 
+import os
+import re
 from typing import Any
 
 import duckdb
 
 from .duckdb_schema import DUCKDB_SCHEMA
 
+_MEMORY_LIMIT_RE = re.compile(r"^\d+(\.\d+)?\s?(B|KB|MB|GB|TB|KiB|MiB|GiB|TiB)$", re.IGNORECASE)
+
+
+def _apply_runtime_pragmas(conn: duckdb.DuckDBPyConnection) -> None:
+    """Bound DuckDB memory and point spilling at a roomy disk, from operator env.
+
+    Unset means DuckDB defaults, so library and test behaviour is unchanged. On a
+    memory-capped host these keep a large merge/aggregation spilling to disk instead
+    of aborting with an out-of-memory error.
+    """
+    memory_limit = os.environ.get("OSMSG_DUCKDB_MEMORY_LIMIT")
+    if memory_limit:
+        if not _MEMORY_LIMIT_RE.match(memory_limit):
+            raise ValueError(f"OSMSG_DUCKDB_MEMORY_LIMIT must be like '1GB', got {memory_limit!r}")
+        conn.execute(f"SET memory_limit='{memory_limit}'")
+    threads = os.environ.get("OSMSG_DUCKDB_THREADS")
+    if threads:
+        conn.execute(f"SET threads={int(threads)}")
+    temp_directory = os.environ.get("OSMSG_DUCKDB_TEMP_DIR")
+    if temp_directory:
+        os.makedirs(temp_directory, exist_ok=True)
+        conn.execute(f"SET temp_directory='{temp_directory.replace(chr(39), chr(39) * 2)}'")
+
 
 def connect(db_path: str) -> duckdb.DuckDBPyConnection:
-    return duckdb.connect(db_path)
+    conn = duckdb.connect(db_path)
+    _apply_runtime_pragmas(conn)
+    return conn
 
 
 def close(conn: duckdb.DuckDBPyConnection) -> None: