osmsg 1.2.4__tar.gz → 1.2.5__tar.gz

{osmsg-1.2.4 → osmsg-1.2.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: osmsg
-Version: 1.2.4
+Version: 1.2.5
 Summary: OpenStreetMap Stats Generator: Commandline
 Keywords: osm,stats,commandline,openstreetmap
 Author: Kshitij Raj Sharma
@@ -230,7 +230,7 @@ docker-compose `environment:` block all reach the same setting. CLI flag wins ov
 | `--country` | `OSMSG_COUNTRY` | unset | Geofabrik region id(s). Comma-separated when set via env. |
 | `--boundary` | `OSMSG_BOUNDARY` | unset | GeoJSON path or inline GeoJSON. |
 | `--url` | `OSMSG_URL` | `minute` | `minute`/`hour`/`day` shortcut or full URL. Comma-separated when set via env. |
-| `--workers` | `OSMSG_WORKERS` | cpu count | Parallel workers. |
+| `--workers` | `OSMSG_WORKERS` | cpu count | Parallel parse workers. |
 | `--cache-dir` | `OSMSG_CACHE_DIR` | platform cache | Where downloaded OSM files are kept across runs. |
 | `--output-dir` | `OSMSG_OUTPUT_DIR` | `.` | Where `<name>.duckdb` and exports are written. |
 | `--format` / `-f` | `OSMSG_FORMAT` | `parquet` | Repeat for multiple. Comma-separated when set via env. |

{osmsg-1.2.4 → osmsg-1.2.5}/README.md

@@ -198,7 +198,7 @@ docker-compose `environment:` block all reach the same setting. CLI flag wins ov
 | `--country` | `OSMSG_COUNTRY` | unset | Geofabrik region id(s). Comma-separated when set via env. |
 | `--boundary` | `OSMSG_BOUNDARY` | unset | GeoJSON path or inline GeoJSON. |
 | `--url` | `OSMSG_URL` | `minute` | `minute`/`hour`/`day` shortcut or full URL. Comma-separated when set via env. |
-| `--workers` | `OSMSG_WORKERS` | cpu count | Parallel workers. |
+| `--workers` | `OSMSG_WORKERS` | cpu count | Parallel parse workers. |
 | `--cache-dir` | `OSMSG_CACHE_DIR` | platform cache | Where downloaded OSM files are kept across runs. |
 | `--output-dir` | `OSMSG_OUTPUT_DIR` | `.` | Where `<name>.duckdb` and exports are written. |
 | `--format` / `-f` | `OSMSG_FORMAT` | `parquet` | Repeat for multiple. Comma-separated when set via env. |

osmsg-1.2.5/osmsg/__version__.py

@@ -0,0 +1 @@
+__version__ = "1.2.5"

{osmsg-1.2.4 → osmsg-1.2.5}/osmsg/cli.py

@@ -150,7 +150,7 @@ def main(
     ] = None,
     workers: Annotated[
         int | None,
-        typer.Option(envvar="OSMSG_WORKERS", help="Parallel workers (default: cpu count)."),
+        typer.Option(envvar="OSMSG_WORKERS", help="Parallel parse workers (default: cpu count)."),
     ] = None,
     rows: Annotated[
         int | None,

{osmsg-1.2.4 → osmsg-1.2.5}/osmsg/gui.py

@@ -61,6 +61,19 @@ def _split(value: str | None) -> list[str] | None:
     return items if items else None
 
 
+def _parse_int(value: object, field: str) -> int | None:
+    text = str(value or "").strip()
+    if not text:
+        return None
+    try:
+        number = int(text)
+    except ValueError as exc:
+        raise OsmsgError(f"{field} must be a whole number.") from exc
+    if number < 1:
+        raise OsmsgError(f"{field} must be at least 1.")
+    return number
+
+
 def build_config(form: dict[str, object], output_dir: str) -> RunConfig:
     """Map the form fields to a RunConfig, raising OsmsgError on invalid input."""
     formats = [name for name in FORMATS if form.get(name)]
@@ -78,6 +91,7 @@ def build_config(form: dict[str, object], output_dir: str) -> RunConfig:
         tag_mode="all" if form.get("all_tags") else "none",
         summary=bool(form.get("summary")),
         formats=formats,
+        workers=_parse_int(form.get("workers"), "Workers"),
         output_dir=Path(output_dir or "."),
     )
 
@@ -133,6 +147,7 @@ class App:
             ("End (blank = now)", "end", ""),
             ("Hashtags (comma-sep)", "hashtags", ""),
             ("Tags (comma-sep)", "tags", ""),
+            ("Workers", "workers", str(os.cpu_count() or 4)),
         ]
         for i, (label, key, default) in enumerate(rows):
             ttk.Label(frame, text=label).grid(row=i, column=0, sticky="w", pady=2)
@@ -141,7 +156,7 @@ class App:
             self.vars[key] = var
 
         preset_frame = ttk.LabelFrame(frame, text="Quick range", padding=6)
-        preset_frame.grid(row=5, column=0, columnspan=4, sticky="we", pady=6)
+        preset_frame.grid(row=6, column=0, columnspan=4, sticky="we", pady=6)
         for i, name in enumerate(PRESETS):
             ttk.Button(preset_frame, text=name, width=11, command=lambda n=name: self._apply_preset(n)).grid(
                 row=0, column=i, padx=2
@@ -149,32 +164,32 @@ class App:
 
         self.vars["all_tags"] = tk.BooleanVar()
         self.vars["summary"] = tk.BooleanVar()
-        ttk.Checkbutton(frame, text="All tags", variable=self.vars["all_tags"]).grid(row=6, column=0, sticky="w")
-        ttk.Checkbutton(frame, text="Daily summary", variable=self.vars["summary"]).grid(row=6, column=1, sticky="w")
+        ttk.Checkbutton(frame, text="All tags", variable=self.vars["all_tags"]).grid(row=7, column=0, sticky="w")
+        ttk.Checkbutton(frame, text="Daily summary", variable=self.vars["summary"]).grid(row=7, column=1, sticky="w")
 
         fmt_frame = ttk.LabelFrame(frame, text="Formats", padding=6)
-        fmt_frame.grid(row=7, column=0, columnspan=4, sticky="we", pady=6)
+        fmt_frame.grid(row=8, column=0, columnspan=4, sticky="we", pady=6)
         for i, name in enumerate(FORMATS):
             var = tk.BooleanVar(value=name in ("parquet", "csv"))
             ttk.Checkbutton(fmt_frame, text=name, variable=var).grid(row=0, column=i, padx=4)
             self.vars[name] = var
 
         self.out_label = ttk.Label(frame, text=f"Output: {self.out_dir}")
-        self.out_label.grid(row=8, column=0, columnspan=3, sticky="w")
-        ttk.Button(frame, text="Choose folder", command=self._choose_folder).grid(row=8, column=3, sticky="e")
+        self.out_label.grid(row=9, column=0, columnspan=3, sticky="w")
+        ttk.Button(frame, text="Choose folder", command=self._choose_folder).grid(row=9, column=3, sticky="e")
 
         self.run_btn = ttk.Button(frame, text="Compute", command=self._on_run)
-        self.run_btn.grid(row=9, column=0, pady=8, sticky="w")
+        self.run_btn.grid(row=10, column=0, pady=8, sticky="w")
         self.open_btn = ttk.Button(frame, text="Open output folder", command=lambda: _open_folder(Path(self.out_dir)))
-        self.open_btn.grid(row=9, column=1, pady=8, sticky="w")
+        self.open_btn.grid(row=10, column=1, pady=8, sticky="w")
         self.spinner = ttk.Progressbar(frame, mode="indeterminate", length=160)
-        self.spinner.grid(row=9, column=2, columnspan=2, pady=8, sticky="we")
+        self.spinner.grid(row=10, column=2, columnspan=2, pady=8, sticky="we")
 
         self.log = scrolledtext.ScrolledText(frame, width=70, height=14, state="disabled")
-        self.log.grid(row=10, column=0, columnspan=4, sticky="nsew")
+        self.log.grid(row=11, column=0, columnspan=4, sticky="nsew")
 
-        ttk.Button(frame, text="About", command=self._show_about).grid(row=11, column=0, pady=(6, 0), sticky="w")
-        ttk.Label(frame, text="A project of OSGeo Nepal").grid(row=11, column=1, columnspan=3, pady=(6, 0), sticky="e")
+        ttk.Button(frame, text="About", command=self._show_about).grid(row=12, column=0, pady=(6, 0), sticky="w")
+        ttk.Label(frame, text="A project of OSGeo Nepal").grid(row=12, column=1, columnspan=3, pady=(6, 0), sticky="e")
         self.root.after(120, self._drain)
 
     def _show_about(self) -> None:

{osmsg-1.2.4 → osmsg-1.2.5}/osmsg/history.py

@@ -5,6 +5,7 @@ path (a glob would make DuckDB list every partition over the HF API)."""
 import datetime as dt
 import json
 import pathlib
+import time
 from dataclasses import dataclass
 
 import duckdb
@@ -16,6 +17,7 @@ UTC = dt.UTC
 SCHEMA_VERSION = 1
 DEFAULT_HISTORY_URL = "hf://datasets/kshitijrajsharma/osmsg-history"
 HISTORY_SEQ_ID = 0
+MONTH_READ_ATTEMPTS = 4
 
 
 @dataclass
@@ -185,36 +187,47 @@ def ingest_remote(
 
     info(f"history: remote ingest {start_iso} -> {end_iso} ({len(months)} month partitions) from {history_url}")
 
+    def ingest_month(month: tuple[int, int]) -> None:
+        changesets_src = _partition_list(history_url, "changesets", [month])
+        changefiles_src = _partition_list(history_url, "changefiles", [month])
+        if changesets_src is not None:
+            conn.execute(
+                f"""INSERT INTO users
+                    SELECT uid, any_value(username) FROM {changesets_src}
+                    WHERE {in_window} AND username IS NOT NULL
+                    GROUP BY uid ON CONFLICT (uid) DO NOTHING"""
+            )
+            conn.execute(
+                f"""INSERT INTO changesets
+                    SELECT changeset_id, uid, created_at, hashtags, editor,
+                           CASE WHEN min_lon IS NOT NULL
+                                THEN ST_MakeEnvelope(min_lon, min_lat, max_lon, max_lat) END
+                    FROM {changesets_src} WHERE {changeset_where}
+                    ON CONFLICT (changeset_id) DO NOTHING"""
+            )
+        if changefiles_src is not None:
+            conn.execute(
+                f"""INSERT INTO changeset_stats
+                    SELECT changeset_id, {HISTORY_SEQ_ID} AS seq_id, uid,
+                           nodes_created, nodes_modified, nodes_deleted,
+                           ways_created, ways_modified, ways_deleted,
+                           rels_created, rels_modified, rels_deleted,
+                           poi_created, poi_modified, tag_stats
+                    FROM {changefiles_src} WHERE {stats_where}
+                    ON CONFLICT (seq_id, changeset_id) DO NOTHING"""
+            )
+
     with progress_bar(len(months), unit="months", description="Reading history") as advance:
         for month in months:
-            changesets_src = _partition_list(history_url, "changesets", [month])
-            changefiles_src = _partition_list(history_url, "changefiles", [month])
-            if changesets_src is not None:
-                conn.execute(
-                    f"""INSERT INTO users
-                        SELECT uid, any_value(username) FROM {changesets_src}
-                        WHERE {in_window} AND username IS NOT NULL
-                        GROUP BY uid ON CONFLICT (uid) DO NOTHING"""
-                )
-                conn.execute(
-                    f"""INSERT INTO changesets
-                        SELECT changeset_id, uid, created_at, hashtags, editor,
-                               CASE WHEN min_lon IS NOT NULL
-                                    THEN ST_MakeEnvelope(min_lon, min_lat, max_lon, max_lat) END
-                        FROM {changesets_src} WHERE {changeset_where}
-                        ON CONFLICT (changeset_id) DO NOTHING"""
-                )
-            if changefiles_src is not None:
-                conn.execute(
-                    f"""INSERT INTO changeset_stats
-                        SELECT changeset_id, {HISTORY_SEQ_ID} AS seq_id, uid,
-                               nodes_created, nodes_modified, nodes_deleted,
-                               ways_created, ways_modified, ways_deleted,
-                               rels_created, rels_modified, rels_deleted,
-                               poi_created, poi_modified, tag_stats
-                        FROM {changefiles_src} WHERE {stats_where}
-                        ON CONFLICT (seq_id, changeset_id) DO NOTHING"""
-                )
+            for attempt in range(MONTH_READ_ATTEMPTS):
+                try:
+                    ingest_month(month)
+                    break
+                except duckdb.Error as exc:
+                    if attempt == MONTH_READ_ATTEMPTS - 1:
+                        raise
+                    warn(f"history: {month[0]}-{month[1]:02d} read failed ({type(exc).__name__}); retrying.")
+                    time.sleep(2 * (attempt + 1))
             advance()
 
     row = conn.execute(f"SELECT count(*) FROM changeset_stats WHERE seq_id = {HISTORY_SEQ_ID}").fetchone()

{osmsg-1.2.4 → osmsg-1.2.5}/osmsg/pipeline.py

@@ -540,21 +540,18 @@ def _processing_config(cfg: RunConfig, *, parquet_dir: Path, geom_wkt: str | Non
     }
 
 
-# Replication servers throttle many concurrent connections, so downloads stay polite regardless of
-# the worker count used for local parsing. Already-downloaded files are cached, so a rerun resumes.
 _DOWNLOAD_WORKERS = 4
 
 
 def _download_all(
     urls: list[str],
     mode: str,
-    max_workers: int,
+    workers: int,
     cookie: str | None,
     cache_dir: Path,
     label: str,
     description: str = "downloading",
 ) -> None:
-    workers = min(max_workers, _DOWNLOAD_WORKERS)
     try:
         with (
             progress_bar(len(urls), unit=label, description=description) as advance,
@@ -714,10 +711,13 @@ def run(cfg: RunConfig) -> dict[str, Any]:
                         if run_live:
                             _auto_switch_replication(cfg, cfg.end_date - cfg.start_date)
                     except duckdb.Error as exc:
-                        warn(f"history: remote ingest failed ({type(exc).__name__}: {exc}); using live path.")
                         for tbl in ("changeset_stats", "changesets", "users"):
                             conn.execute(f"DELETE FROM {tbl}")
-                        run_live = True
+                        dbmod.close(conn)
+                        raise OsmsgError(
+                            f"Reading the published history failed after retries ({type(exc).__name__}). "
+                            "Re-run to try again, narrow the date range, or pass --no-history for the live path."
+                        ) from exc
 
     max_workers = cfg.workers or _cpu_count()
     info(f"Workers: {max_workers}")
@@ -756,7 +756,13 @@ def run(cfg: RunConfig) -> dict[str, Any]:
             cs_config["window_start_utc"] = cfg.start_date.astimezone(UTC)
 
             _download_all(
-                urls, "changeset", max_workers, None, cfg.cache_dir, "changesets", description="Downloading changesets"
+                urls,
+                "changeset",
+                _DOWNLOAD_WORKERS,
+                None,
+                cfg.cache_dir,
+                "changesets",
+                description="Downloading changesets",
             )
             _process_all(
                 urls,
@@ -819,7 +825,7 @@ def run(cfg: RunConfig) -> dict[str, Any]:
         _download_all(
             urls,
             "changefiles",
-            max_workers,
+            _DOWNLOAD_WORKERS,
             cookie,
             cfg.cache_dir,
             "changefiles",

{osmsg-1.2.4 → osmsg-1.2.5}/osmsg/replication.py

@@ -145,7 +145,7 @@ class ChangesetReplication:
     def timestamp_to_sequence(self, ts: datetime) -> int:
         cur_seq, last_run = self._state()
         wanted = int((ts - last_run).total_seconds() / 60) + cur_seq
-        return min(wanted, cur_seq)
+        return max(1, min(wanted, cur_seq))
 
     def sequence_to_timestamp(self, seq: int) -> datetime:
         txt = session.get(self.state_url(seq)).text

{osmsg-1.2.4 → osmsg-1.2.5}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "osmsg"
-version = "1.2.4"
+version = "1.2.5"
 description = "OpenStreetMap Stats Generator: Commandline"
 readme = "README.md"
 authors = [

osmsg-1.2.4/osmsg/__version__.py

@@ -1 +0,0 @@
-__version__ = "1.2.4"