osmsg 0.3.0__tar.gz → 1.0.3__tar.gz

osmsg-1.0.3/PKG-INFO

@@ -0,0 +1,177 @@
+Metadata-Version: 2.4
+Name: osmsg
+Version: 1.0.3
+Summary: OpenStreetMap Stats Generator: Commandline
+Keywords: osm,stats,commandline,openstreetmap
+Author: Kshitij Raj Sharma
+Author-email: Kshitij Raj Sharma <skshitizraj@gmail.com>
+License-Expression: MIT
+License-File: LICENSE
+Classifier: Topic :: Utilities
+Classifier: Topic :: Scientific/Engineering :: GIS
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Dist: duckdb>=1.5.2
+Requires-Dist: osmium>=4.3.1
+Requires-Dist: platformdirs>=4.5.1
+Requires-Dist: pyarrow>=24.0.0
+Requires-Dist: pydantic>=2.13.3
+Requires-Dist: python-dotenv>=1.2.2
+Requires-Dist: pytz>=2024.1
+Requires-Dist: requests>=2.32.5
+Requires-Dist: rich>=13.0
+Requires-Dist: shapely>=2.1.2
+Requires-Dist: typer>=0.25.0
+Requires-Dist: typer-config[yaml]>=1.5.1
+Requires-Python: >=3.11
+Project-URL: documentation, https://github.com/osgeonepal/osmsg
+Project-URL: homepage, https://github.com/osgeonepal/osmsg
+Project-URL: repository, https://github.com/osgeonepal/osmsg
+Description-Content-Type: text/markdown
+
+# osmsg
+
+[![CI](https://github.com/osgeonepal/osmsg/actions/workflows/ci.yml/badge.svg)](https://github.com/osgeonepal/osmsg/actions/workflows/ci.yml)
+[![Docker](https://github.com/osgeonepal/osmsg/actions/workflows/docker.yml/badge.svg)](https://github.com/osgeonepal/osmsg/actions/workflows/docker.yml)
+[![PyPI](https://img.shields.io/pypi/v/osmsg.svg)](https://pypi.org/project/osmsg/)
+[![Python](https://img.shields.io/badge/python-3.11%2B-blue.svg)](https://www.python.org/downloads/)
+[![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](./LICENSE)
+[![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
+[![uv](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/uv/main/assets/badge/v0.json)](https://github.com/astral-sh/uv)
+[![Container](https://img.shields.io/badge/ghcr.io-osgeonepal%2Fosmsg-2496ED?logo=docker)](https://github.com/osgeonepal/osmsg/pkgs/container/osmsg)
+
+**OpenStreetMap Stats Generator.** A tiny CLI (and Python library) that turns OSM history into per-user counts
+of nodes, ways, and relations created, modified, or deleted, written to parquet, csv, json, markdown, or Postgres.
+
+A Project of [OSGeo Nepal](https://osgeonepal.org).
+
+## What you get
+
+- Per-user create/modify/delete counts over any time window.
+- Tag and hashtag breakdowns (e.g. `building`, `#hotosm`).
+- Country and custom-boundary filters via Geofabrik.
+- Cron-friendly resume with `--update`.
+- Outputs you can query: parquet, csv, json, markdown, DuckDB, Postgres.
+
+## Install
+
+Pick the one that fits how you work.
+
+```bash
+uvx --from osmsg osmsg --last hour       # zero-install, one-shot run
+pip install osmsg                        # into your project
+uv tool install osmsg                    # standalone CLI
+docker run --rm -v "$PWD:/work" -w /work ghcr.io/osgeonepal/osmsg:latest --last hour
+```
+
+`uvx` can run osmsg in a throwaway environment , no install, no virtualenv to manage. Works
+with any flag combination, e.g. `uvx --from osmsg osmsg --last hour --tags building --summary -f parquet -f markdown`.
+
+## Quick start
+
+```bash
+osmsg --last hour                        # planet, last hour
+osmsg --last day --tags building         # last day with a tag breakdown
+osmsg --hashtags hotosm --last day       # only changesets tagged #hotosm
+```
+
+That's it. A `stats.duckdb` and a `stats.parquet` show up in your current folder.
+
+## Tutorials
+
+### 1. Stats for a country
+
+```bash
+osmsg --country nepal --last day
+```
+
+`--country` resolves through Geofabrik and needs an OSM account. Set `OSM_USERNAME` and `OSM_PASSWORD`
+in your shell or a `.env` file:
+
+```bash
+export OSM_USERNAME=you
+export OSM_PASSWORD=secret
+```
+
+### 2. A custom date range with summaries
+
+```bash
+osmsg --start "2026-04-01" --end "2026-04-08" \
+      --tags building --tags highway --summary
+```
+
+`--summary` adds a daily rollup file alongside the per-changeset stats.
+
+### 3. Run on a schedule
+
+```bash
+osmsg --country nepal --update           # picks up where the last run stopped
+```
+
+Drop that into cron or a GitHub Actions schedule. State is stored inside the DuckDB file, so reruns are safe.
+
+### 4. Query the output
+
+```bash
+duckdb stats.duckdb -c "SELECT username, SUM(nodes_created) AS n
+                        FROM users JOIN changeset_stats USING (uid)
+                        GROUP BY username ORDER BY n DESC LIMIT 10"
+```
+
+Same schema in DuckDB and Postgres: `users`, `changesets`, `changeset_stats`, `state`.
+
+### 5. Use it as a library
+
+```python
+from datetime import datetime, UTC
+from osmsg import RunConfig, run
+
+result = run(RunConfig(
+    name="nepal",
+    countries=["nepal"],
+    start_date=datetime(2026, 4, 25, tzinfo=UTC),
+    end_date=datetime(2026, 4, 26, tzinfo=UTC),
+))
+print(result["files"]["parquet"])
+```
+
+Same pipeline as the CLI.
+
+### 6. Long flag lists? Use a config
+
+```bash
+osmsg --config nepal.yaml
+```
+
+Any flag works as a YAML key. See [docs/Manual.md](./docs/Manual.md) for the full list.
+
+## Output formats
+
+Every run writes `stats.duckdb` (or `<--name>.duckdb`) plus the formats you ask for via
+`-f parquet|csv|json|markdown|psql`. Parquet is the default. Open it with duckdb, polars, pandas, anything.
+
+## Documentation
+
+- [Installation](./docs/Installation.md)
+- [Manual](./docs/Manual.md) (every flag, with examples)
+- [Version control / release notes](./docs/Version_control.md)
+
+## Contributing
+
+Pull requests are welcome. Quick path:
+
+```bash
+git clone https://github.com/osgeonepal/osmsg && cd osmsg
+git switch develop
+uv sync
+uv run pre-commit install
+uv run pytest -m "not network"
+```
+
+Please read [CONTRIBUTING.md](./CONTRIBUTING.md) and the [Code of Conduct](./CODE_OF_CONDUCT.md) before opening a PR.
+Use [Conventional Commits](https://www.conventionalcommits.org/) (`cz commit`).
+
+## License
+
+[MIT](./LICENSE) © OSGeo Nepal contributors.

osmsg-1.0.3/README.md

@@ -0,0 +1,145 @@
+# osmsg
+
+[![CI](https://github.com/osgeonepal/osmsg/actions/workflows/ci.yml/badge.svg)](https://github.com/osgeonepal/osmsg/actions/workflows/ci.yml)
+[![Docker](https://github.com/osgeonepal/osmsg/actions/workflows/docker.yml/badge.svg)](https://github.com/osgeonepal/osmsg/actions/workflows/docker.yml)
+[![PyPI](https://img.shields.io/pypi/v/osmsg.svg)](https://pypi.org/project/osmsg/)
+[![Python](https://img.shields.io/badge/python-3.11%2B-blue.svg)](https://www.python.org/downloads/)
+[![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](./LICENSE)
+[![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
+[![uv](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/uv/main/assets/badge/v0.json)](https://github.com/astral-sh/uv)
+[![Container](https://img.shields.io/badge/ghcr.io-osgeonepal%2Fosmsg-2496ED?logo=docker)](https://github.com/osgeonepal/osmsg/pkgs/container/osmsg)
+
+**OpenStreetMap Stats Generator.** A tiny CLI (and Python library) that turns OSM history into per-user counts
+of nodes, ways, and relations created, modified, or deleted, written to parquet, csv, json, markdown, or Postgres.
+
+A Project of [OSGeo Nepal](https://osgeonepal.org).
+
+## What you get
+
+- Per-user create/modify/delete counts over any time window.
+- Tag and hashtag breakdowns (e.g. `building`, `#hotosm`).
+- Country and custom-boundary filters via Geofabrik.
+- Cron-friendly resume with `--update`.
+- Outputs you can query: parquet, csv, json, markdown, DuckDB, Postgres.
+
+## Install
+
+Pick the one that fits how you work.
+
+```bash
+uvx --from osmsg osmsg --last hour       # zero-install, one-shot run
+pip install osmsg                        # into your project
+uv tool install osmsg                    # standalone CLI
+docker run --rm -v "$PWD:/work" -w /work ghcr.io/osgeonepal/osmsg:latest --last hour
+```
+
+`uvx` can run osmsg in a throwaway environment , no install, no virtualenv to manage. Works
+with any flag combination, e.g. `uvx --from osmsg osmsg --last hour --tags building --summary -f parquet -f markdown`.
+
+## Quick start
+
+```bash
+osmsg --last hour                        # planet, last hour
+osmsg --last day --tags building         # last day with a tag breakdown
+osmsg --hashtags hotosm --last day       # only changesets tagged #hotosm
+```
+
+That's it. A `stats.duckdb` and a `stats.parquet` show up in your current folder.
+
+## Tutorials
+
+### 1. Stats for a country
+
+```bash
+osmsg --country nepal --last day
+```
+
+`--country` resolves through Geofabrik and needs an OSM account. Set `OSM_USERNAME` and `OSM_PASSWORD`
+in your shell or a `.env` file:
+
+```bash
+export OSM_USERNAME=you
+export OSM_PASSWORD=secret
+```
+
+### 2. A custom date range with summaries
+
+```bash
+osmsg --start "2026-04-01" --end "2026-04-08" \
+      --tags building --tags highway --summary
+```
+
+`--summary` adds a daily rollup file alongside the per-changeset stats.
+
+### 3. Run on a schedule
+
+```bash
+osmsg --country nepal --update           # picks up where the last run stopped
+```
+
+Drop that into cron or a GitHub Actions schedule. State is stored inside the DuckDB file, so reruns are safe.
+
+### 4. Query the output
+
+```bash
+duckdb stats.duckdb -c "SELECT username, SUM(nodes_created) AS n
+                        FROM users JOIN changeset_stats USING (uid)
+                        GROUP BY username ORDER BY n DESC LIMIT 10"
+```
+
+Same schema in DuckDB and Postgres: `users`, `changesets`, `changeset_stats`, `state`.
+
+### 5. Use it as a library
+
+```python
+from datetime import datetime, UTC
+from osmsg import RunConfig, run
+
+result = run(RunConfig(
+    name="nepal",
+    countries=["nepal"],
+    start_date=datetime(2026, 4, 25, tzinfo=UTC),
+    end_date=datetime(2026, 4, 26, tzinfo=UTC),
+))
+print(result["files"]["parquet"])
+```
+
+Same pipeline as the CLI.
+
+### 6. Long flag lists? Use a config
+
+```bash
+osmsg --config nepal.yaml
+```
+
+Any flag works as a YAML key. See [docs/Manual.md](./docs/Manual.md) for the full list.
+
+## Output formats
+
+Every run writes `stats.duckdb` (or `<--name>.duckdb`) plus the formats you ask for via
+`-f parquet|csv|json|markdown|psql`. Parquet is the default. Open it with duckdb, polars, pandas, anything.
+
+## Documentation
+
+- [Installation](./docs/Installation.md)
+- [Manual](./docs/Manual.md) (every flag, with examples)
+- [Version control / release notes](./docs/Version_control.md)
+
+## Contributing
+
+Pull requests are welcome. Quick path:
+
+```bash
+git clone https://github.com/osgeonepal/osmsg && cd osmsg
+git switch develop
+uv sync
+uv run pre-commit install
+uv run pytest -m "not network"
+```
+
+Please read [CONTRIBUTING.md](./CONTRIBUTING.md) and the [Code of Conduct](./CODE_OF_CONDUCT.md) before opening a PR.
+Use [Conventional Commits](https://www.conventionalcommits.org/) (`cz commit`).
+
+## License
+
+[MIT](./LICENSE) © OSGeo Nepal contributors.

osmsg-1.0.3/osmsg/__init__.py

@@ -0,0 +1,95 @@
+"""OpenStreetMap stats generator. Parquet-first, OAuth 2.0, UTC-only.
+
+Library usage::
+
+    from osmsg import RunConfig, run, OsmsgError
+
+    cfg = RunConfig(
+        name="nepal",
+        countries=["nepal"],
+        start_date=datetime(2026, 4, 25, tzinfo=UTC),
+        end_date=datetime(2026, 4, 26, tzinfo=UTC),
+        formats=["parquet"],
+        osm_username="...",
+        osm_password="...",
+    )
+    try:
+        result = run(cfg)
+    except OsmsgError as exc:
+        ...
+    print(result["files"]["parquet"])      # → 'nepal.parquet'
+
+CLI entry point: ``osmsg`` (defined in ``osmsg.cli``).
+"""
+
+from __future__ import annotations
+
+from .__version__ import __version__
+from .db import (
+    attach_metadata,
+    attach_tag_stats,
+    connect,
+    create_tables,
+    daily_summary,
+    get_state,
+    upsert_state,
+    user_stats,
+)
+from .exceptions import (
+    CredentialsRequiredError,
+    GeofabrikAuthError,
+    NoDataFoundError,
+    OsmsgError,
+    UnknownRegionError,
+)
+from .export import (
+    summary_markdown,
+    table_markdown,
+    to_csv,
+    to_json,
+    to_parquet,
+    to_psql,
+)
+from .geofabrik import country_update_url, load_index
+from .models import (
+    Action,
+    Changeset,
+    ChangesetStats,
+    ElementStat,
+    TagValueStat,
+    User,
+)
+from .pipeline import RunConfig, run
+
+__all__ = [
+    "Action",
+    "Changeset",
+    "ChangesetStats",
+    "CredentialsRequiredError",
+    "ElementStat",
+    "GeofabrikAuthError",
+    "NoDataFoundError",
+    "OsmsgError",
+    "RunConfig",
+    "TagValueStat",
+    "UnknownRegionError",
+    "User",
+    "__version__",
+    "attach_metadata",
+    "attach_tag_stats",
+    "connect",
+    "country_update_url",
+    "create_tables",
+    "daily_summary",
+    "get_state",
+    "load_index",
+    "run",
+    "summary_markdown",
+    "table_markdown",
+    "to_csv",
+    "to_json",
+    "to_parquet",
+    "to_psql",
+    "upsert_state",
+    "user_stats",
+]

osmsg-1.0.3/osmsg/__version__.py

@@ -0,0 +1 @@
+__version__ = "1.0.3"

osmsg-1.0.3/osmsg/_http.py

@@ -0,0 +1,41 @@
+"""Shared `requests.Session` with retry policy + connect/read timeouts.
+
+Every HTTP call in osmsg goes through this session so retry behaviour and
+timeout defaults are consistent. Per-request `timeout=` still wins.
+"""
+
+from __future__ import annotations
+
+import requests
+from requests.adapters import HTTPAdapter
+from urllib3.util.retry import Retry
+
+USER_AGENT = "osmsg"
+DEFAULT_TIMEOUT = (10, 60)  # (connect, read) seconds
+
+
+class _TimeoutSession(requests.Session):
+    """Session that applies `DEFAULT_TIMEOUT` whenever the caller did not specify one."""
+
+    def request(self, method, url, *args, **kwargs):
+        kwargs.setdefault("timeout", DEFAULT_TIMEOUT)
+        return super().request(method, url, *args, **kwargs)
+
+
+def make_session() -> requests.Session:
+    """Fresh session with the standard timeout + retry policy (use when a flow needs its own cookie jar)."""
+    s = _TimeoutSession()
+    retry = Retry(
+        total=5,
+        backoff_factor=0.5,
+        status_forcelist=(429, 500, 502, 503, 504),
+        allowed_methods=frozenset({"GET", "POST", "HEAD"}),
+    )
+    adapter = HTTPAdapter(max_retries=retry, pool_maxsize=32)
+    s.mount("https://", adapter)
+    s.mount("http://", adapter)
+    s.headers["User-Agent"] = USER_AGENT
+    return s
+
+
+session = make_session()

osmsg-1.0.3/osmsg/auth.py

@@ -0,0 +1,118 @@
+"""OAuth 2.0 cookie client for Geofabrik internal download server.
+
+Mirrors https://github.com/geofabrik/sendfile_osm_oauth_protector
+"""
+
+from __future__ import annotations
+
+import urllib.parse
+from html.parser import HTMLParser
+
+from ._http import make_session
+from ._http import session as shared_session
+from .exceptions import GeofabrikAuthError
+
+DEFAULT_OSM_HOST = "https://www.openstreetmap.org"
+DEFAULT_CONSUMER_URL = "https://osm-internal.download.geofabrik.de/get_cookie"
+
+
+class _CsrfFinder(HTMLParser):
+    def __init__(self) -> None:
+        super().__init__()
+        self.token: str | None = None
+
+    def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
+        if tag != "meta" or self.token is not None:
+            return
+        a = dict(attrs)
+        if a.get("name") == "csrf-token":
+            content = a.get("content")
+            if content:
+                self.token = content
+
+
+def _csrf(html: str) -> str:
+    parser = _CsrfFinder()
+    parser.feed(html)
+    if parser.token is None:
+        raise GeofabrikAuthError("authenticity_token not found in OSM response")
+    return parser.token
+
+
+def get_geofabrik_cookie(
+    username: str,
+    password: str,
+    osm_host: str = DEFAULT_OSM_HOST,
+    consumer_url: str = DEFAULT_CONSUMER_URL,
+) -> str:
+    if not username or not password:
+        raise GeofabrikAuthError("OSM username and password required")
+
+    r = shared_session.post(f"{consumer_url}?action=get_authorization_url", timeout=30)
+    if r.status_code != 200:
+        raise GeofabrikAuthError(f"get_authorization_url returned HTTP {r.status_code}")
+    payload = r.json()
+    try:
+        authz_url = payload["authorization_url"]
+        state = payload["state"]
+        redirect_uri = payload["redirect_uri"]
+        client_id = payload["client_id"]
+    except KeyError as exc:
+        raise GeofabrikAuthError(f"missing field in authorization response: {exc}") from exc
+
+    s = make_session()
+
+    r = s.get(f"{osm_host}/login?cookie_test=true", timeout=30)
+    if r.status_code != 200:
+        raise GeofabrikAuthError(f"GET /login returned HTTP {r.status_code}")
+
+    r = s.post(
+        f"{osm_host}/login",
+        data={
+            "username": username,
+            "password": password,
+            "referer": "/",
+            "commit": "Login",
+            "authenticity_token": _csrf(r.text),
+        },
+        allow_redirects=False,
+        timeout=30,
+    )
+    if r.status_code != 302:
+        raise GeofabrikAuthError(f"OSM login failed (HTTP {r.status_code}); check credentials")
+
+    r = s.get(authz_url, allow_redirects=False, timeout=30)
+    if r.status_code != 302:
+        if r.status_code != 200:
+            raise GeofabrikAuthError(f"GET authorize returned HTTP {r.status_code}")
+        r = s.post(
+            authz_url,
+            data={
+                "client_id": client_id,
+                "redirect_uri": redirect_uri,
+                "authenticity_token": _csrf(r.text),
+                "state": state,
+                "response_type": "code",
+                "scope": "read_prefs",
+                "nonce": "",
+                "code_challenge": "",
+                "code_challenge_method": "",
+                "commit": "Authorize",
+            },
+            allow_redirects=False,
+            timeout=30,
+        )
+        if r.status_code != 302:
+            raise GeofabrikAuthError(f"POST authorize returned HTTP {r.status_code}")
+
+    location = r.headers.get("location") or ""
+    if "?" not in location:
+        raise GeofabrikAuthError("authorization redirect missing query string")
+
+    s.get(f"{osm_host}/logout", timeout=30)
+
+    final_url = f"{location}&{urllib.parse.urlencode({'format': 'http'})}"
+    r = shared_session.get(final_url, timeout=30)
+    if r.status_code != 200 or not r.text.strip():
+        raise GeofabrikAuthError(f"cookie exchange failed (HTTP {r.status_code})")
+    return r.text.strip()

osmsg-1.0.3/osmsg/boundary.py

@@ -0,0 +1,37 @@
+"""Geometry helpers: boundary parsing + bbox centroid."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+from shapely.geometry import MultiPolygon, Polygon, box, shape
+from shapely.geometry.base import BaseGeometry
+
+
+def load_boundary(input_data: str) -> BaseGeometry:
+    """Accept either inline GeoJSON text or a path to a GeoJSON file."""
+    try:
+        payload: Any = json.loads(input_data)
+    except json.JSONDecodeError as exc:
+        path = Path(input_data)
+        if not path.is_file():
+            raise ValueError(f"Not valid JSON or a file path: {input_data!r}") from exc
+        payload = json.loads(path.read_text())
+
+    geometry = payload.get("geometry") if "geometry" in payload else payload
+    if not geometry or geometry.get("type") not in ("Polygon", "MultiPolygon"):
+        raise ValueError("Boundary must be a Polygon or MultiPolygon GeoJSON.")
+    geom = shape(geometry)
+    if isinstance(geom, (Polygon, MultiPolygon)):
+        return geom
+    raise ValueError(f"Unexpected geometry type: {type(geom).__name__}")
+
+
+def bbox_centroid(bounds) -> tuple[float, float] | None:
+    """Centroid of an osmium bounding box, or None if invalid."""
+    if not bounds.valid():
+        return None
+    geom = box(bounds.bottom_left.lon, bounds.bottom_left.lat, bounds.top_right.lon, bounds.top_right.lat)
+    return geom.centroid.x, geom.centroid.y