xtransit 0.1.0a1__py3-none-any.whl

xtransit/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.1.0a1"

xtransit/__main__.py

@@ -0,0 +1,3 @@
+from xtransit.collect import main
+
+raise SystemExit(main())

xtransit/collect.py

@@ -0,0 +1,447 @@
+"""Download cataloged GTFS schedules and point-in-time realtime snapshots."""
+
+import argparse
+from collections import Counter
+from concurrent.futures import ThreadPoolExecutor, as_completed
+import csv
+from datetime import datetime, timezone
+import hashlib
+import io
+import json
+from pathlib import Path
+import re
+import shutil
+import subprocess
+import threading
+import time
+from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
+import zipfile
+from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
+
+
+CATALOG_URL = "https://files.mobilitydatabase.org/feeds_v2.csv"
+HOST_LOCK = threading.Lock()
+HOST_LIMITS = {}
+
+
+def utc_now():
+    return datetime.now(timezone.utc).isoformat()
+
+
+def fetch(url, destination, timeout=90, max_bytes=512 * 1024 * 1024):
+    """Use curl's TLS verification, wall-clock timeout, and size limit."""
+    if urlsplit(url).scheme not in {"https", "http"}:
+        raise ValueError("Only HTTP(S) feed URLs are supported")
+    destination.parent.mkdir(parents=True, exist_ok=True)
+    with HOST_LOCK:
+        limit = HOST_LIMITS.setdefault(urlsplit(url).netloc, threading.Semaphore(4))
+    with limit:
+        result = subprocess.run(
+            [
+                "curl",
+                "--fail",
+                "--silent",
+                "--show-error",
+                "--location",
+                "--proto",
+                "=http,https",
+                "--proto-redir",
+                "=http,https",
+                "--connect-timeout",
+                "15",
+                "--max-time",
+                str(timeout),
+                "--max-filesize",
+                str(max_bytes),
+                "--user-agent",
+                "xtransit/0.0 GTFS collector",
+                "--output",
+                str(destination),
+                "--url",
+                url,
+            ],
+            capture_output=True,
+            text=True,
+            timeout=timeout + 10,
+        )
+    if result.returncode:
+        # Do not put URLs or credentials echoed by remote servers in logs.
+        http_error = re.search(r"error: (\d{3})", result.stderr)
+        reason = (
+            f"HTTP {http_error[1]}" if http_error else f"curl exit {result.returncode}"
+        )
+        raise RuntimeError(reason)
+    if destination.stat().st_size > max_bytes:
+        raise ValueError("Feed exceeds configured size limit")
+
+
+def read_catalog(path):
+    with path.open(newline="", encoding="utf-8-sig") as file:
+        reader = csv.DictReader(file)
+        required = {
+            "id",
+            "data_type",
+            "location.country_code",
+            "status",
+            "urls.direct_download",
+        }
+        if not required.issubset(reader.fieldnames or []):
+            raise ValueError("Unrecognized MobilityDatabase CSV schema")
+        rows = list(reader)
+    if len({row["id"] for row in rows}) != len(rows):
+        raise ValueError("Catalog contains duplicate feed IDs")
+    return rows
+
+
+def write_catalog(path, rows):
+    if not rows:
+        raise ValueError("No catalog entries found for the requested country")
+    with path.open("w", newline="", encoding="utf-8") as file:
+        writer = csv.DictWriter(file, fieldnames=list(rows[0]))
+        writer.writeheader()
+        writer.writerows(rows)
+
+
+def feed_urls(row):
+    """Public schedule mirrors remain usable when the producer requires a key."""
+    urls = []
+    if row["data_type"] == "gtfs" and row.get("urls.latest"):
+        urls.append(row["urls.latest"])
+    if (
+        row.get("urls.authentication_type", "") in {"", "0"}
+        and row["urls.direct_download"]
+    ):
+        direct = row["urls.direct_download"]
+        urls.append(direct)
+        if row["data_type"] == "gtfs_rt":
+            parts = urlsplit(direct)
+            query = parse_qsl(parts.query, keep_blank_values=True)
+            if any(key.lower() == "debug" for key, _ in query):
+                # Some catalog URLs request a human-readable debug response.
+                query = [(key, value) for key, value in query if key.lower() != "debug"]
+                urls.append(urlunsplit(parts._replace(query=urlencode(query))))
+    return list(dict.fromkeys(urls))
+
+
+def inspect_schedule(path):
+    with zipfile.ZipFile(path) as archive:
+        required = {"agency.txt", "routes.txt", "trips.txt", "stop_times.txt"}
+        directories = {}
+        for name in archive.namelist():
+            if name.endswith("/") or name.startswith("__MACOSX/"):
+                continue
+            parent, _, basename = name.rpartition("/")
+            members = directories.setdefault(parent, {})
+            if basename.lower() in members:
+                raise ValueError("ZIP has ambiguous GTFS table names")
+            members[basename.lower()] = name
+        candidates = [
+            members for members in directories.values() if required.issubset(members)
+        ]
+        if not candidates:
+            raise ValueError("ZIP lacks required GTFS schedule tables")
+        if len(candidates) != 1:
+            raise ValueError("ZIP contains multiple GTFS datasets")
+        members = candidates[0]
+        if not {"calendar.txt", "calendar_dates.txt"}.intersection(members):
+            raise ValueError("ZIP lacks a GTFS service calendar")
+        dates = []
+        for name in ["calendar.txt", "calendar_dates.txt"]:
+            if name not in members:
+                continue
+            if archive.getinfo(members[name]).file_size > 128 * 1024 * 1024:
+                raise ValueError("Service calendar exceeds inspection limit")
+            with archive.open(members[name]) as raw:
+                for row in csv.DictReader(io.TextIOWrapper(raw, encoding="utf-8-sig")):
+                    if name == "calendar.txt":
+                        dates.extend(
+                            [row.get("start_date", ""), row.get("end_date", "")]
+                        )
+                    elif row.get("exception_type") == "1":
+                        dates.append(row.get("date", ""))
+        dates = [date for date in dates if re.fullmatch(r"\d{8}", date)]
+        agency_timezones = set()
+        with archive.open(members["agency.txt"]) as raw:
+            for row in csv.DictReader(io.TextIOWrapper(raw, encoding="utf-8-sig")):
+                if row.get("agency_timezone"):
+                    agency_timezones.add(row["agency_timezone"])
+        local_dates = []
+        for agency_timezone in agency_timezones:
+            try:
+                local_dates.append(
+                    datetime.now(ZoneInfo(agency_timezone)).strftime("%Y%m%d")
+                )
+            except ZoneInfoNotFoundError:
+                pass
+        if not local_dates:
+            local_dates.append(datetime.now(timezone.utc).strftime("%Y%m%d"))
+        return {
+            "tables": sorted(members),
+            "archive_table_paths": members,
+            "nonstandard_table_paths": any(
+                name != member for name, member in members.items()
+            ),
+            "uncompressed_bytes": sum(info.file_size for info in archive.infolist()),
+            "service_date_min": min(dates) if dates else None,
+            "service_date_max": max(dates) if dates else None,
+            "agency_timezones": sorted(agency_timezones),
+            "calendar_range_expired": bool(dates and max(dates) < min(local_dates)),
+            "calendar_range_future": bool(dates and min(dates) > max(local_dates)),
+        }
+
+
+def inspect_realtime(path):
+    from google.protobuf.json_format import MessageToDict
+    from google.transit import gtfs_realtime_pb2
+
+    feed = gtfs_realtime_pb2.FeedMessage()
+    feed.ParseFromString(path.read_bytes())
+    if not feed.IsInitialized() or not feed.HasField("header"):
+        raise ValueError("Response is not an initialized GTFS Realtime feed")
+    timestamp = feed.header.timestamp if feed.header.HasField("timestamp") else None
+    age = time.time() - timestamp if timestamp is not None else None
+    counts = Counter()
+    for entity in feed.entity:
+        for field in ["trip_update", "vehicle", "alert"]:
+            if entity.HasField(field):
+                counts[field] += 1
+    return {
+        "feed_timestamp": timestamp,
+        "feed_age_seconds": round(age) if age is not None else None,
+        "freshness": "timestamp_missing"
+        if age is None
+        else ("future_timestamp" if age < -300 else "stale" if age > 300 else "recent"),
+        "incrementality": int(feed.header.incrementality),
+        "entity_count": len(feed.entity),
+        "entity_counts": dict(counts),
+    }, MessageToDict(feed, preserving_proto_field_name=True)
+
+
+def collect_feed(row, root, snapshot, timeout=90, refresh=False):
+    result = {
+        "id": row["id"],
+        "provider": row["provider"],
+        "data_type": row["data_type"],
+        "entity_type": row.get("entity_type", ""),
+        "static_reference": row.get("static_reference", "").split("|")
+        if row.get("static_reference")
+        else [],
+        "license_url": row.get("urls.license", ""),
+        "authentication_info": row.get("urls.authentication_info", ""),
+        "attempted_at": utc_now(),
+    }
+    if not re.fullmatch(r"[A-Za-z0-9_-]+", row["id"]):
+        return dict(result, status="failed", error="Unsafe catalog feed ID")
+    schedule = row["data_type"] == "gtfs"
+    relative = (
+        Path("schedule") / (row["id"] + ".zip")
+        if schedule
+        else (Path("realtime") / snapshot / (row["id"] + ".pb"))
+    )
+    target = root / relative
+    partial = target.with_suffix(target.suffix + ".part")
+    urls = feed_urls(row)
+    if not urls:
+        status = (
+            "requires_auth"
+            if row.get("urls.authentication_type", "") not in {"", "0"}
+            else "missing_url"
+        )
+        return dict(result, status=status)
+    if schedule and target.exists() and not refresh:
+        try:
+            details = inspect_schedule(target)
+            return dict(
+                result,
+                status="cached",
+                path=str(relative),
+                bytes=target.stat().st_size,
+                **details,
+            )
+        except (ValueError, OSError, zipfile.BadZipFile):
+            pass
+    attempts = []
+    for url in urls:
+        try:
+            fetch(
+                url,
+                partial,
+                timeout=timeout,
+                max_bytes=(512 if schedule else 32) * 1024 * 1024,
+            )
+            if schedule:
+                details = inspect_schedule(partial)
+            else:
+                details, decoded = inspect_realtime(partial)
+            digest = hashlib.sha256()
+            with partial.open("rb") as file:
+                for chunk in iter(lambda: file.read(1024 * 1024), b""):
+                    digest.update(chunk)
+            partial.replace(target)
+            if not schedule:
+                json_path = target.with_suffix(".json")
+                json_path.write_text(
+                    json.dumps(decoded, ensure_ascii=False), encoding="utf-8"
+                )
+                result["json_path"] = str(json_path.relative_to(root))
+            return dict(
+                result,
+                status="downloaded",
+                downloaded_at=utc_now(),
+                path=str(relative),
+                bytes=target.stat().st_size,
+                sha256=digest.hexdigest(),
+                source_url=url,
+                **details,
+            )
+        except Exception as error:
+            # A bad or unavailable feed should not cancel the national collection.
+            attempts.append({"source_url": url, "error": str(error)[:300]})
+        finally:
+            partial.unlink(missing_ok=True)
+    return dict(result, status="failed", attempts=attempts)
+
+
+def main(argv=None):
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--output", type=Path, default=Path("data/us"))
+    parser.add_argument(
+        "--catalog", type=Path, help="Use a previously downloaded catalog CSV"
+    )
+    parser.add_argument(
+        "--country", default="US", help="Catalog ISO country code (default: US)"
+    )
+    parser.add_argument(
+        "--kind", choices=["all", "schedule", "realtime"], default="all"
+    )
+    parser.add_argument(
+        "--feed-id", action="append", help="Collect only these catalog IDs; repeatable"
+    )
+    parser.add_argument("--workers", type=int, default=8)
+    parser.add_argument(
+        "--timeout", type=int, default=90, help="Maximum seconds per URL"
+    )
+    parser.add_argument(
+        "--refresh", action="store_true", help="Redownload existing schedules"
+    )
+    parser.add_argument("--catalog-only", action="store_true")
+    args = parser.parse_args(argv)
+    if args.workers < 1 or args.timeout < 1:
+        parser.error("workers and timeout must be positive")
+    if not shutil.which("curl"):
+        parser.error("curl is required")
+    if not args.catalog_only and args.kind in {"all", "realtime"}:
+        try:
+            from google.transit import gtfs_realtime_pb2  # noqa: F401
+        except ImportError:
+            parser.error("Install realtime support: pip install -e '.[data]'")
+    args.output.mkdir(parents=True, exist_ok=True)
+    snapshot = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%S%fZ")
+    run_dir = args.output / "runs" / snapshot
+    run_dir.mkdir(parents=True)
+    catalog = args.catalog
+    if catalog is None:
+        catalog = run_dir / "catalog-global.csv"
+        fetch(CATALOG_URL, catalog, max_bytes=32 * 1024 * 1024)
+    rows = [
+        row
+        for row in read_catalog(catalog)
+        if row["location.country_code"] == args.country.upper()
+    ]
+    write_catalog(run_dir / "catalog.csv", rows)
+    shutil.copyfile(run_dir / "catalog.csv", args.output / "catalog.csv")
+    kinds = (
+        {"gtfs", "gtfs_rt"}
+        if args.kind == "all"
+        else {"gtfs" if args.kind == "schedule" else "gtfs_rt"}
+    )
+    selected = [
+        row for row in rows if row["status"] == "active" and row["data_type"] in kinds
+    ]
+    if args.feed_id:
+        selected = [row for row in selected if row["id"] in args.feed_id]
+        missing = set(args.feed_id) - {row["id"] for row in selected}
+        if missing:
+            parser.error(
+                f"Requested feeds are not active in this country/kind: {sorted(missing)}"
+            )
+    summary = {
+        "started_at": utc_now(),
+        "catalog_source": CATALOG_URL,
+        "catalog_file": str(catalog),
+        "country": args.country.upper(),
+        "run": snapshot,
+        "catalog_entries": len(rows),
+        "selected_entries": len(selected),
+        "selected_by_type": dict(Counter(row["data_type"] for row in selected)),
+        "catalog_status_counts": dict(Counter(row["status"] for row in rows)),
+        "note": "Catalog coverage is not universal coverage. Realtime is a collection of snapshots taken at different times. Calendar bounds do not prove service on a specific day. Structural checks are not full GTFS validation.",
+    }
+    print(json.dumps(summary, indent=2), flush=True)
+    results = []
+    if not args.catalog_only:
+        with (run_dir / "manifest.jsonl").open("w", encoding="utf-8") as manifest:
+            with ThreadPoolExecutor(max_workers=args.workers) as pool:
+                futures = [
+                    pool.submit(
+                        collect_feed,
+                        row,
+                        args.output,
+                        snapshot,
+                        args.timeout,
+                        args.refresh,
+                    )
+                    for row in selected
+                ]
+                for future in as_completed(futures):
+                    result = future.result()
+                    results.append(result)
+                    manifest.write(json.dumps(result, ensure_ascii=False) + "\n")
+                    manifest.flush()
+                    if len(results) % 25 == 0 or len(results) == len(selected):
+                        print(
+                            f"{len(results)}/{len(selected)}: {dict(Counter(r['status'] for r in results))}",
+                            flush=True,
+                        )
+    summary.update(
+        {
+            "finished_at": utc_now(),
+            "result_counts": dict(Counter(row["status"] for row in results)),
+            "results_by_type": {
+                kind: dict(
+                    Counter(
+                        row["status"] for row in results if row["data_type"] == kind
+                    )
+                )
+                for kind in sorted(kinds)
+            },
+            "downloaded_bytes": sum(
+                row.get("bytes", 0) for row in results if row["status"] == "downloaded"
+            ),
+            "realtime_freshness": dict(
+                Counter(row["freshness"] for row in results if "freshness" in row)
+            ),
+            "expired_schedule_ranges": sum(
+                row.get("calendar_range_expired", False) for row in results
+            ),
+            "future_schedule_ranges": sum(
+                row.get("calendar_range_future", False) for row in results
+            ),
+            "realtime_entity_counts": dict(
+                sum(
+                    (Counter(row.get("entity_counts", {})) for row in results),
+                    Counter(),
+                )
+            ),
+        }
+    )
+    (run_dir / "summary.json").write_text(
+        json.dumps(summary, indent=2) + "\n", encoding="utf-8"
+    )
+    shutil.copyfile(run_dir / "summary.json", args.output / "summary.json")
+    print(json.dumps(summary, indent=2), flush=True)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

xtransit/config.py

@@ -0,0 +1,15 @@
+from dataclasses import dataclass, field
+import os
+from pathlib import Path
+
+
+@dataclass
+class Settings:
+    PKG_NAME: str = "xtransit"
+    XT_HOME: Path = field(default_factory=lambda: Path(os.environ.get("XT_HOME", Path.home() / ".xtransit")))
+
+    def __post_init__(self):
+        self.XT_HOME = Path(self.XT_HOME).expanduser()
+
+
+settings = Settings()

xtransit/core/service.py

@@ -0,0 +1,18 @@
+from dataclasses import dataclass
+
+from pydantic import BaseModel
+
+
+class ServiceConfig(BaseModel):
+    instance_id: str
+
+
+@dataclass
+class Service:
+    config: ServiceConfig
+
+    def start(self):
+        raise NotImplementedError
+
+    def stop(self):
+        raise NotImplementedError

xtransit/database.py

@@ -0,0 +1,88 @@
+"""The persistent PostgreSQL instance shared by this project's components."""
+
+import asyncio
+import json
+import os
+from pathlib import Path
+import secrets
+import tempfile
+import threading
+
+from sqlalchemy import text
+
+from xtransit.config import settings
+from xtransit.services.postgres import PostgreSQL, PostgreSQLConfig
+
+
+_database = None
+_database_lock = threading.Lock()
+
+
+def _project_config(path: Path) -> PostgreSQLConfig:
+    if not path.exists():
+        config = PostgreSQLConfig(
+            user="xtransit",
+            database="xtransit",
+            password=secrets.token_urlsafe(32),
+            instance_id="xtransit.postgres.default",
+            published_port=55432,
+            bind_host="127.0.0.1",
+        )
+        path.parent.mkdir(parents=True, exist_ok=True)
+        # Publish a complete, private file atomically, including across processes.
+        descriptor, temporary = tempfile.mkstemp(prefix=".postgres-", dir=path.parent)
+        try:
+            with os.fdopen(descriptor, "w", encoding="utf-8") as file:
+                file.write(config.model_dump_json(indent=2) + "\n")
+            try:
+                os.link(temporary, path)
+            except FileExistsError:
+                pass
+        finally:
+            Path(temporary).unlink(missing_ok=True)
+    return PostgreSQLConfig.model_validate_json(path.read_text(encoding="utf-8"))
+
+
+def get_database() -> PostgreSQL:
+    """Return the process-wide service; creation and connection happen lazily."""
+    global _database
+    with _database_lock:
+        if _database is None:
+            _database = PostgreSQL(_project_config(settings.XT_HOME / "postgres.json"))
+        return _database
+
+
+async def check_database():
+    database = get_database()
+    try:
+        async with database.aconnect() as connection:
+            result = (
+                (
+                    await connection.execute(
+                        text(
+                            "SELECT current_database() AS database, current_user AS username, "
+                            "current_setting('server_version') AS version"
+                        )
+                    )
+                )
+                .mappings()
+                .one()
+            )
+            return {
+                **dict(result),
+                "container": database.config.instance_id,
+                "host": database.service_host,
+                "port": database.port,
+                "config_file": str(settings.XT_HOME / "postgres.json"),
+            }
+    finally:
+        await database.aclose()
+
+
+def main():
+    """Create/reuse the instance and print a connection check without credentials."""
+    print(json.dumps(asyncio.run(check_database()), indent=2))
+
+
+if __name__ == "__main__":
+    main()