osmsg 1.1.0__tar.gz → 1.1.2__tar.gz

{osmsg-1.1.0 → osmsg-1.1.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: osmsg
-Version: 1.1.0
+Version: 1.1.2
 Summary: OpenStreetMap Stats Generator: Commandline
 Keywords: osm,stats,commandline,openstreetmap
 Author: Kshitij Raj Sharma

osmsg-1.1.2/osmsg/__version__.py

@@ -0,0 +1 @@
+__version__ = "1.1.2"

{osmsg-1.1.0 → osmsg-1.1.2}/osmsg/_tick.py

@@ -36,7 +36,8 @@ def main() -> int:
     name = _parse_arg(extra_args, "--name") or "stats"
     out = Path(_parse_arg(extra_args, "--output-dir") or "/var/lib/osmsg")
     country = _parse_arg(extra_args, "--country")
-    url = _parse_arg(extra_args, "--url") or "minute"
+    explicit_url = _parse_arg(extra_args, "--url")
+    url = explicit_url or "minute"
 
     out.mkdir(parents=True, exist_ok=True)
 
@@ -45,26 +46,32 @@ def main() -> int:
     try:
         fcntl.flock(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
     except BlockingIOError:
+        os.close(lock_fd)
         print("[osmsg-tick] previous tick still running, skipping", flush=True)
         return 0
 
-    source_url = country_update_url(country) if country else resolve_url(url)
-    db_path = out / f"{name}.duckdb"
-
-    extra_set = set(extra_args)
-    cmd = ["osmsg"] + extra_args
-    if not (extra_set & {"--all", "--keys"}):
-        cmd.append("--all")
-
-    if _has_state(db_path, source_url):
-        cmd.append("--update")
-    elif bootstrap_days:
-        cmd.extend(["--days", bootstrap_days])
-    else:
-        cmd.extend(["--last", bootstrap])
-
-    print(f"[osmsg-tick] {' '.join(cmd)}", flush=True)
-    return subprocess.call(cmd)
+    try:
+        # Mirror pipeline._normalize_urls: explicit --url wins over --country's geofabrik default,
+        # otherwise --update can't find the state row and the DuckDB gets wiped every tick.
+        source_url = country_update_url(country) if country and explicit_url is None else resolve_url(url)
+        db_path = out / f"{name}.duckdb"
+
+        extra_set = set(extra_args)
+        cmd = ["osmsg"] + extra_args
+        if not (extra_set & {"--all", "--keys"}):
+            cmd.append("--all")
+
+        if _has_state(db_path, source_url):
+            cmd.append("--update")
+        elif bootstrap_days:
+            cmd.extend(["--days", bootstrap_days])
+        else:
+            cmd.extend(["--last", bootstrap])
+
+        print(f"[osmsg-tick] {' '.join(cmd)}", flush=True)
+        return subprocess.call(cmd)
+    finally:
+        os.close(lock_fd)
 
 
 if __name__ == "__main__":

{osmsg-1.1.0 → osmsg-1.1.2}/osmsg/cli.py

@@ -233,6 +233,9 @@ def main(
     if sum(1 for x in (start, last, days) if x) > 1:
         error("--start, --last, and --days are mutually exclusive — pick one.")
         raise typer.Exit(code=2)
+    if update and any(x is not None for x in (start, end, last, days)):
+        error("--update resumes from prior state and runs to head; it ignores --start/--end/--last/--days.")
+        raise typer.Exit(code=2)
     if Format.psql in formats and not psql_dsn:
         error("-f psql requires --psql-dsn (libpq connection string, e.g. 'host=localhost dbname=osm user=osm').")
         raise typer.Exit(code=2)

{osmsg-1.1.0 → osmsg-1.1.2}/osmsg/export/markdown.py

@@ -17,15 +17,16 @@ def _stringify(v: Any) -> str:
     return str(v)
 
 
-def table_markdown(rows: list[dict[str, Any]], headers: list[str] | None = None) -> str:
+def table_markdown(rows: list[dict[str, Any]], output_path: Path, headers: list[str] | None = None) -> Path:
     """Return a GitHub-flavored markdown table for the given rows."""
-    if not rows:
-        return ""
     headers = headers or list(rows[0].keys())
     lines = ["| " + " | ".join(headers) + " |", "| " + " | ".join("---" for _ in headers) + " |"]
     for r in rows:
         lines.append("| " + " | ".join(_stringify(r.get(h)) for h in headers) + " |")
-    return "\n".join(lines)
+    output_path = Path(output_path)
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    output_path.write_text("\n".join(lines), encoding="utf-8")
+    return output_path
 
 
 def _human(n: int) -> str:

{osmsg-1.1.0 → osmsg-1.1.2}/osmsg/handlers.py

@@ -32,23 +32,15 @@ class ChangesetHandler(osmium.SimpleHandler):
             return
         cfg = self.config
 
-        # Drop padding-pulled changesets that don't overlap the window — otherwise
-        # attach_metadata leaks their hashtags onto users with in-window edits.
-        # `c.open` gate is required: osmium uses 1970 as the closed_at sentinel.
+        # Drop closed-before-window changesets so attach_metadata can't leak their
+        # hashtags onto in-window users.
         start = cfg.get("window_start_utc")
-        end = cfg.get("window_end_utc")
-        if start is not None and end is not None:
-            created = c.created_at
-            if created.tzinfo is None:
-                created = created.replace(tzinfo=dt.UTC)
-            if created >= end:
+        if start is not None and not c.open:
+            closed = c.closed_at
+            if closed.tzinfo is None:
+                closed = closed.replace(tzinfo=dt.UTC)
+            if closed <= start:
                 return
-            if not c.open:
-                closed = c.closed_at
-                if closed.tzinfo is None:
-                    closed = closed.replace(tzinfo=dt.UTC)
-                if closed <= start:
-                    return
 
         if self._geom is not None:
             if not c.bounds.valid():
@@ -119,7 +111,6 @@ class ChangefileHandler(osmium.SimpleHandler):
         super().__init__()
         self.config = config
         self.start = config["start_date_utc"]
-        self.end = config["end_date_utc"]
         self.seq_id = sequence_id
         # None == no filter; empty set == filter matched nothing (collect nothing).
         self.valid_changesets = valid_changesets
@@ -188,15 +179,20 @@ class ChangefileHandler(osmium.SimpleHandler):
                 if track_length and k in length_keys:
                     tv.add_length(len_m)
 
+    def _in_window(self, ts) -> bool:
+        # Lower-bound only; disjoint coverage between ticks comes from the seq boundary
+        # (next tick resumes at last_seq+1, with state.last_ts = state_ts(last_seq)).
+        return ts >= self.start
+
     def node(self, n) -> None:
-        if not (self.start <= n.timestamp < self.end):
+        if not self._in_window(n.timestamp):
             return
         if not self._should_collect(n.user, n.changeset):
             return
         self._accumulate(n.uid, n.user, n.changeset, 0 if n.deleted else n.version, n.tags, "nodes")
 
     def way(self, w) -> None:
-        if not (self.start <= w.timestamp < self.end):
+        if not self._in_window(w.timestamp):
             return
         if not self._should_collect(w.user, w.changeset):
             return
@@ -204,7 +200,7 @@ class ChangefileHandler(osmium.SimpleHandler):
         self._accumulate(w.uid, w.user, w.changeset, 0 if w.deleted else w.version, w.tags, "ways", nodes)
 
     def relation(self, r) -> None:
-        if not (self.start <= r.timestamp < self.end):
+        if not self._in_window(r.timestamp):
             return
         if not self._should_collect(r.user, r.changeset):
             return

{osmsg-1.1.0 → osmsg-1.1.2}/osmsg/pipeline.py

@@ -22,7 +22,7 @@ from .boundary import load_boundary
 from .db.queries import attach_metadata, attach_tag_stats, daily_summary, list_changesets, user_stats
 from .db.schema import get_state, upsert_state
 from .exceptions import CredentialsRequiredError, NoDataFoundError, OsmsgError
-from .export import summary_markdown, to_csv, to_json, to_parquet, to_psql
+from .export import summary_markdown, table_markdown, to_csv, to_json, to_parquet, to_psql
 from .fetch import download_osm_file
 from .geofabrik import country_geometry, country_update_url
 from .replication import (
@@ -30,6 +30,7 @@ from .replication import (
     SHORTCUTS,
     ChangesetReplication,
     changefile_download_urls,
+    changefile_seq_timestamp,
     resolve_url,
 )
 from .ui import info, progress_bar, warn
@@ -371,6 +372,8 @@ def run(cfg: RunConfig) -> dict[str, Any]:
     valid_changesets: set[int] | None = None
     start_seq: int | None = None
     end_seq: int | None = None
+    # Threaded into changefile_download_urls so a tick never advances cf past cs.
+    cs_frontier_ts: dt.datetime | None = None
 
     if cfg.hashtags or cfg.changeset:
         cs_repl = ChangesetReplication(pad_hours=cfg.changeset_pad_hours)
@@ -384,11 +387,12 @@ def run(cfg: RunConfig) -> dict[str, Any]:
         )
         info(f"Changesets: {len(urls)} files (seq {cs_start}-{cs_end}), {pad_note}.")
 
+        cs_frontier_ts = cs_repl.sequence_to_timestamp(cs_end)
+
         if urls:
             cs_dir.mkdir(parents=True, exist_ok=True)
             cs_config = _processing_config(cfg, parquet_dir=cs_dir, geom_wkt=geom_wkt)
             cs_config["window_start_utc"] = cfg.start_date.astimezone(UTC)
-            cs_config["window_end_utc"] = cfg.end_date.astimezone(UTC)
 
             _download_all(
                 urls, "changeset", max_workers, None, cfg.cache_dir, "changesets", description="Downloading changesets"
@@ -408,7 +412,7 @@ def run(cfg: RunConfig) -> dict[str, Any]:
                 conn,
                 source_url=CHANGESETS_REPLICATION,
                 last_seq=cs_end,
-                last_ts=cfg.end_date.astimezone(UTC),
+                last_ts=cs_frontier_ts,
                 updated_at=dt.datetime.now(UTC),
             )
             info("Changeset processing complete.")
@@ -421,15 +425,12 @@ def run(cfg: RunConfig) -> dict[str, Any]:
         info(f"Changefiles ← {url}")
         url_start, resume_seq = url_starts[url]
         urls, server_ts, src_start_seq, src_end_seq, _, _ = changefile_download_urls(
-            url_start, cfg.end_date, url, resume_seq=resume_seq
+            url_start, cfg.end_date, url, resume_seq=resume_seq, cs_ts=cs_frontier_ts
         )
         if start_seq is None:
             start_seq = src_start_seq
         end_seq = src_end_seq
-        # Cap per URL only — never mutate cfg.end_date or sibling URLs lose their window.
-        url_end_date = min(cfg.end_date, server_ts)
         url_start_date_utc = url_start.astimezone(UTC)
-        url_end_date_utc = url_end_date.astimezone(UTC)
 
         gap = server_ts - url_start_date_utc
         info(
@@ -439,12 +440,20 @@ def run(cfg: RunConfig) -> dict[str, Any]:
 
         if not urls:
             info(f"  {url}: already up-to-date")
+            if resume_seq is not None:
+                # Heartbeat: bump updated_at so /health can tell "alive, idle" apart from "stuck".
+                upsert_state(
+                    conn,
+                    source_url=url,
+                    last_seq=resume_seq - 1,
+                    last_ts=url_start,
+                    updated_at=dt.datetime.now(UTC),
+                )
             continue
 
         cf_dir.mkdir(parents=True, exist_ok=True)
         cf_config = _processing_config(cfg, parquet_dir=cf_dir, geom_wkt=None)
         cf_config["start_date_utc"] = url_start_date_utc
-        cf_config["end_date_utc"] = url_end_date_utc
 
         _download_all(
             urls,
@@ -469,15 +478,18 @@ def run(cfg: RunConfig) -> dict[str, Any]:
             description="Processing changefiles",
         )
         dbmod.merge_parquet_files(conn, cf_dir, cleanup=True)
+        # state.last_ts is the seq_ts of last_seq so the next tick's lower-bound filter
+        # aligns with the seq boundary.
+        state_last_ts = changefile_seq_timestamp(url, src_end_seq)
         upsert_state(
             conn,
             source_url=url,
             last_seq=src_end_seq,
-            last_ts=url_end_date,
+            last_ts=state_last_ts,
             updated_at=dt.datetime.now(UTC),
         )
-        lag = server_ts - url_end_date_utc
-        info(f"  DB now current to: {url_end_date_utc.isoformat()}  |  lag from server: {lag}")
+        lag = server_ts - state_last_ts
+        info(f"  DB now current to: {state_last_ts.isoformat()}  |  lag from server: {lag}")
         info(f"Changefile processing complete: {url}")
 
     if cfg.delete_temp:
@@ -521,19 +533,10 @@ def run(cfg: RunConfig) -> dict[str, Any]:
         written["json"] = str(to_json(rows, out / f"{cfg.name}.json"))
 
     if "markdown" in cfg.formats:
-        from .export.markdown import summary_markdown as render_md
-
         md_path = out / f"{cfg.name}.md"
-        render_md(
+        table_markdown(
             rows,
             output_path=md_path,
-            start_date=start_date_utc,
-            end_date=end_date_utc,
-            additional_tags=cfg.additional_tags,
-            length_tags=cfg.length_tags,
-            tag_mode=cfg.tag_mode,
-            fname=cfg.name,
-            tm_stats=cfg.tm_stats,
         )
         written["markdown"] = str(md_path)
 

{osmsg-1.1.0 → osmsg-1.1.2}/osmsg/replication.py

@@ -34,14 +34,25 @@ def seq_to_timestamp(state_url: str) -> datetime:
     return datetime.strptime(raw, "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=UTC)
 
 
+def changefile_seq_timestamp(base_url: str, seq: int) -> datetime:
+    """Timestamp of a changefile-replication seq (the diff's `state.txt` `timestamp=` line)."""
+    return seq_to_timestamp(ReplicationServer(base_url).get_state_url(seq))
+
+
 def changefile_download_urls(
     start_date: datetime | None,
     end_date: datetime,
     base_url: str,
     *,
     resume_seq: int | None = None,
+    cs_ts: datetime | None = None,
 ) -> tuple[list[str], datetime, int, int, str, str]:
-    """resume_seq starts exactly there (skipping the timestamp lookup + backward pad used on first runs)."""
+    """Resolve [start_seq, last_seq] for the time range, plus the URL list to fetch.
+
+    resume_seq, when given, is used verbatim (no backward pad). cs_ts gates the
+    upper bound on resume ticks: last_seq stays behind cs_ts so every (seq, cs) row
+    written has its parent changeset in place.
+    """
     repl = ReplicationServer(base_url)
 
     if resume_seq is not None:
@@ -75,15 +86,19 @@ def changefile_download_urls(
         end_seq = repl.timestamp_to_sequence(end_date)
         if end_seq is None:
             raise OsmsgError(f"Could not resolve end_date {end_date}")
-        last_seq = end_seq
-        # Pad forwards so the last requested timestamp is fully covered by the diffs we fetch.
-        if "minute" in base_url:
-            adjust = int((seq_to_timestamp(repl.get_state_url(end_seq)) - end_date).total_seconds() / 60)
-            last_seq = last_seq + adjust + 60
-        else:
-            last_seq += 1
-        if last_seq >= server_seq:
-            last_seq = server_seq
+        # +1 only when end_seq's state_ts is strictly before end_date, since that is the
+        # only diff that can contain edits in (state_ts(end_seq), end_date].
+        end_seq_ts = seq_to_timestamp(repl.get_state_url(end_seq))
+        if end_seq_ts < end_date:
+            end_seq += 1
+        last_seq = min(end_seq, server_seq)
+
+    # Hold cf one diff behind cs when cs is the slower stream, so every (seq, cs) row
+    # written has a parent in `changesets` already.
+    if resume_seq is not None and cs_ts is not None:
+        target_ts = end_date if end_date else server_ts
+        if target_ts > cs_ts:
+            last_seq -= 1
 
     if seq >= last_seq:
         return [], server_ts, start_seq, last_seq, start_seq_url, repl.get_state_url(last_seq)
@@ -168,10 +183,8 @@ class ChangesetReplication:
             end_seq = self.timestamp_to_sequence(end_date)
             end_ts = self.sequence_to_timestamp(end_seq)
             if end_date > end_ts:
+                # Step to the diff covering end_date, plus one so edits at end_date land.
                 end_seq += int((end_date - end_ts).total_seconds() / 60) + 1
-                end_ts = self.sequence_to_timestamp(end_seq)
-            if end_ts > end_date:
-                end_seq += int((end_ts - end_date).total_seconds() / 60) + 60
             end_seq = min(end_seq, cur_seq)
 
         if start_seq >= end_seq:

{osmsg-1.1.0 → osmsg-1.1.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "osmsg"
-version = "1.1.0"
+version = "1.1.2"
 description = "OpenStreetMap Stats Generator: Commandline"
 readme = "README.md"
 authors = [

osmsg-1.1.0/osmsg/__version__.py

@@ -1 +0,0 @@
-__version__ = "1.1.0"