apographe 0.1.0__py3-none-any.whl

apographe/__init__.py

@@ -0,0 +1,37 @@
+"""Apographe: collect public data, source by source and day by day, into plain files.
+
+Quick start::
+
+    from apographe import run
+
+    report = run("cy.ekalathi", out="data", limit=5)
+    print(report.rows)  # rows written per table
+
+``help(run)`` shows every option and how to write your own source.
+"""
+
+from apographe._version import __version__
+from apographe.core.http import PoliteClient
+from apographe.core.oauth import ClientCredentials
+from apographe.core.source import (
+    LoginWall,
+    Options,
+    Report,
+    Source,
+    SourceError,
+)
+from apographe.core.storage import Store
+from apographe.runner import run
+
+__all__ = [
+    "ClientCredentials",
+    "LoginWall",
+    "Options",
+    "PoliteClient",
+    "Report",
+    "Source",
+    "SourceError",
+    "Store",
+    "__version__",
+    "run",
+]

apographe/__main__.py

@@ -0,0 +1,5 @@
+import sys
+
+from apographe.cli import main
+
+sys.exit(main())

apographe/_version.py

@@ -0,0 +1,8 @@
+"""The installed version, in its own module so core code can import it early."""
+
+from importlib.metadata import PackageNotFoundError, version
+
+try:
+    __version__ = version("apographe")
+except PackageNotFoundError:  # running from a source tree that is not installed
+    __version__ = "0.0.0"

apographe/cli.py

@@ -0,0 +1,208 @@
+"""Command line: ``apographe list`` and ``apographe run <source>``.
+
+Exit codes: 0 done, 2 the source now needs a login (nothing collected),
+1 anything else.
+"""
+
+from __future__ import annotations
+
+import argparse
+import os
+import sys
+from datetime import date
+from pathlib import Path
+
+from apographe._version import __version__
+from apographe.core import registry
+from apographe.core.source import LoginWall, Report, SourceError
+from apographe.runner import run
+
+DOCS = "https://github.com/aristidesz/apographe/blob/main/docs"
+EPILOG = f"""examples:
+  apographe list                                   what you can collect
+  apographe info cy.fuel                           a source's jobs and what they need
+  apographe run cy.ekalathi --limit 5 --out data   try a source
+
+docs: {DOCS}/usage.md"""
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(
+        prog="apographe",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog=EPILOG,
+        description=(
+            "Collect public data, source by source and day by day, into plain files."
+        ),
+    )
+    parser.add_argument("--version", action="version", version=__version__)
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    sub.add_parser("list", help="show the sources and their jobs")
+
+    info = sub.add_parser("info", help="show a source's jobs and what they need")
+    info.add_argument("source", help="source id, e.g. cy.fuel")
+
+    run = sub.add_parser("run", help="run a source's jobs")
+    run.add_argument("source", help="source id, e.g. cy.ekalathi")
+    run.add_argument(
+        "--job",
+        action="append",
+        dest="jobs",
+        help="job to run (repeatable); default: the source's usual daily jobs",
+    )
+    run.add_argument(
+        "--out", type=Path, default=Path("data"), help="folder to write into"
+    )
+    run.add_argument(
+        "--date",
+        type=date.fromisoformat,
+        help="the day to file the data under (default: today where the source is)",
+    )
+    run.add_argument("--limit", type=int, help="only the first N items")
+    run.add_argument(
+        "--only", help="only these item ids, comma separated (e.g. 2074,1906)"
+    )
+    run.add_argument("--delay", type=float, help="seconds between requests")
+    run.add_argument(
+        "--input",
+        type=Path,
+        action="append",
+        dest="inputs",
+        default=[],
+        help="file or folder for a job that imports (repeatable)",
+    )
+    run.add_argument("--force", action="store_true", help="redo existing work")
+    run.add_argument(
+        "--env-file",
+        type=Path,
+        help="file of KEY=VALUE lines to use as environment variables for this run",
+    )
+
+    args = parser.parse_args(argv)
+    if args.command == "list":
+        return _list()
+    if args.command == "info":
+        return _info(args.source)
+    return _run(args)
+
+
+def _list() -> int:
+    for source_id in sorted(registry.available()):
+        source = registry.load(source_id)
+        print(f"{source.id}  {source.title} ({source.country})")
+        for job, about in source.jobs.items():
+            mark = "*" if job in source.default_jobs else " "
+            needs = []
+            if source.job_env.get(job):
+                needs.append("[needs env]")
+            if job in source.extras:
+                needs.append(f"[needs extra {source.extras[job]}]")
+            print(f"   {mark} {job:<14} {about} {' '.join(needs)}".rstrip())
+    print("\n* runs by default")
+    return 0
+
+
+def _info(source_id: str) -> int:
+    try:
+        source = registry.load(source_id)
+    except KeyError as exc:
+        print(exc.args[0], file=sys.stderr)
+        return 1
+    print(f"{source.id}: {source.title}")
+    print(f"homepage:  {source.homepage}")
+    print(f"time zone: {source.timezone}")
+    print(f"output:    <out>/{source.folder}/")
+    print("jobs:")
+    for job, about in source.jobs.items():
+        default = " (default)" if job in source.default_jobs else ""
+        print(f"  {job}{default}: {about}")
+        if job in source.extras:
+            print(f"    install: pip install 'apographe[{source.extras[job]}]'")
+        for name in source.job_env.get(job, ()):
+            print(f"    needs {name}: {source.env.get(name, '')}")
+    optional = [
+        n for n in source.env if not any(n in v for v in source.job_env.values())
+    ]
+    for name in optional:
+        print(f"optional {name}: {source.env[name]}")
+    print(f"docs: {DOCS}/sources/{source.id.replace('.', '-')}.md")
+    return 0
+
+
+def _read_env_file(path: Path) -> dict[str, str]:
+    """KEY=VALUE lines; blank lines and # comments skipped, quotes stripped."""
+    values = {}
+    for line in path.read_text(encoding="utf-8").splitlines():
+        line = line.strip()
+        if not line or line.startswith("#") or "=" not in line:
+            continue
+        key, _, value = line.partition("=")
+        value = value.strip()
+        if len(value) >= 2 and value[0] == value[-1] and value[0] in "\"'":
+            value = value[1:-1]
+        values[key.strip()] = value
+    return values
+
+
+def _run(args: argparse.Namespace) -> int:
+    added: list[str] = []
+    if args.env_file:
+        try:
+            values = _read_env_file(args.env_file)
+        except OSError as exc:
+            print(f"cannot read {args.env_file}: {exc.strerror}", file=sys.stderr)
+            return 1
+        for key, value in values.items():
+            if key not in os.environ:  # a variable already set wins
+                os.environ[key] = value
+                added.append(key)
+    try:
+        return _run_source(args)
+    finally:
+        for key in added:  # the file only applies to this run
+            del os.environ[key]
+
+
+def _run_source(args: argparse.Namespace) -> int:
+    current = args.source
+
+    def progress(job: str, day: date, report: Report | None) -> None:
+        nonlocal current
+        current = f"{args.source} {job}"
+        if report is None:
+            print(f"{current}: started for {day}", flush=True)
+            return
+        for table, count in report.rows.items():
+            print(f"{current}: {count} rows -> {table}", flush=True)
+
+    try:
+        run(
+            args.source,
+            out=args.out,
+            jobs=args.jobs,
+            day=args.date,
+            limit=args.limit,
+            only=[x.strip() for x in args.only.split(",")] if args.only else None,
+            delay=args.delay,
+            inputs=args.inputs,
+            force=args.force,
+            progress=progress,
+        )
+    except LoginWall as exc:
+        print(f"::warning::{args.source} needs a login now, skipping: {exc}")
+        return 2
+    except SourceError as exc:
+        print(f"{current}: stopped: {exc}", file=sys.stderr)
+        return 1
+    except KeyError as exc:
+        print(exc.args[0], file=sys.stderr)
+        return 1
+    except ValueError as exc:
+        print(exc, file=sys.stderr)
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

apographe/core/__init__.py

@@ -0,0 +1 @@
+"""The framework every source plugs into."""

apographe/core/http.py

@@ -0,0 +1,94 @@
+"""A polite JSON client: one request at a time, spaced out, retried when the
+server stumbles, and clear about it when a site starts asking for a login."""
+
+from __future__ import annotations
+
+import time
+from collections.abc import Callable, Mapping
+from typing import Any
+from urllib.parse import urlsplit
+
+import requests
+
+from apographe._version import __version__
+from apographe.core.source import LoginWall, SourceError
+
+USER_AGENT = f"apographe/{__version__} (+https://github.com/aristidesz/apographe)"
+RETRY_STATUS = {429, 500, 502, 503, 504}
+
+
+class PoliteClient:
+    """Fetch JSON from one site, politely.
+
+    Waits ``min_interval`` seconds between requests, retries 429 and 5xx answers
+    with a growing pause, and raises ``LoginWall`` if the site asks for a login.
+    """
+
+    def __init__(
+        self,
+        base_url: str,
+        *,
+        min_interval: float = 3.0,
+        timeout: float = 60.0,
+        retries: int = 3,
+        backoff: float = 10.0,
+        session: requests.Session | None = None,
+        sleep: Callable[[float], None] = time.sleep,
+        clock: Callable[[], float] = time.monotonic,
+    ) -> None:
+        self.base_url = base_url.rstrip("/")
+        self.min_interval = min_interval
+        self.timeout = timeout
+        self.retries = retries
+        self.backoff = backoff
+        self.session = session or requests.Session()
+        self.session.headers.setdefault("User-Agent", USER_AGENT)
+        self.session.headers["Accept"] = "application/json"
+        self._sleep = sleep
+        self._clock = clock
+        self._last: float | None = None
+
+    def get_json(self, path: str, params: Mapping[str, Any] | None = None) -> Any:
+        """GET ``base_url + path`` and decode the JSON, retrying server errors.
+
+        Raises LoginWall when the site answers 401/403 or redirects elsewhere
+        (typically to a login page), and SourceError when retries run out.
+        """
+        url = f"{self.base_url}{path}"
+        last_error = ""
+        for attempt in range(self.retries + 1):
+            if attempt:
+                self._sleep(self.backoff * attempt)
+            self._wait()
+            try:
+                resp = self.session.get(url, params=params, timeout=self.timeout)
+            except (requests.ConnectionError, requests.Timeout) as exc:
+                last_error = type(exc).__name__
+                continue
+            if resp.status_code in (401, 403) or _left_site(url, resp.url):
+                raise LoginWall(f"{path} answered {resp.status_code} at {resp.url}")
+            if resp.status_code in RETRY_STATUS:
+                last_error = f"HTTP {resp.status_code}"
+                continue
+            if resp.status_code != 200:
+                raise SourceError(f"{path} answered HTTP {resp.status_code}")
+            try:
+                return resp.json()
+            except ValueError as exc:
+                kind = resp.headers.get("Content-Type", "unknown content")
+                raise SourceError(f"{path} did not return JSON ({kind})") from exc
+        raise SourceError(f"{path} failed {self.retries + 1} times: {last_error}")
+
+    def _wait(self) -> None:
+        now = self._clock()
+        if self._last is not None:
+            gap = self.min_interval - (now - self._last)
+            if gap > 0:
+                self._sleep(gap)
+        self._last = self._clock()
+
+
+def _left_site(requested: str, final: str | None) -> bool:
+    if not final:
+        return False
+    return urlsplit(requested).netloc != urlsplit(final).netloc

apographe/core/oauth.py

@@ -0,0 +1,83 @@
+"""OAuth 2.0 client-credentials sign-in, for sources that sit behind a gateway.
+
+The client id and secret identify a program, not a person. They are exchanged
+for a short-lived access token, which is kept and reused until it is about to
+expire (gateways ask clients not to fetch a new token for every call).
+"""
+
+from __future__ import annotations
+
+import time
+from collections.abc import Callable
+
+import requests
+
+from apographe._version import __version__
+from apographe.core.source import SourceError
+
+USER_AGENT = f"apographe/{__version__} (+https://github.com/aristidesz/apographe)"
+EARLY = 60.0  # seconds before expiry at which a token is replaced
+
+
+class ClientCredentials:
+    """Sign in with a client id and secret; ``token()`` gives the access token."""
+
+    def __init__(
+        self,
+        token_url: str,
+        client_id: str,
+        client_secret: str,
+        *,
+        session: requests.Session | None = None,
+        clock: Callable[[], float] = time.monotonic,
+        timeout: float = 30.0,
+    ) -> None:
+        self.token_url = token_url
+        self._id = client_id
+        self._secret = client_secret
+        self.session = session or requests.Session()
+        self._clock = clock
+        self.timeout = timeout
+        self._token: str | None = None
+        self._expires = 0.0
+
+    def __repr__(self) -> str:
+        # Never show the secret, even in a traceback or a log line.
+        return f"ClientCredentials(token_url={self.token_url!r}, client_id=...)"
+
+    def token(self) -> str:
+        """The current access token, fetched only when there is none or it expired."""
+        if self._token is None or self._clock() >= self._expires:
+            self._token = self._fetch()
+        return self._token
+
+    def forget(self) -> None:
+        """Drop the token, e.g. after the API refused it."""
+        self._token = None
+
+    def _fetch(self) -> str:
+        try:
+            resp = self.session.post(
+                self.token_url,
+                data={"grant_type": "client_credentials"},
+                auth=(self._id, self._secret),
+                headers={"Accept": "application/json", "User-Agent": USER_AGENT},
+                timeout=self.timeout,
+            )
+        except (requests.ConnectionError, requests.Timeout) as exc:
+            raise SourceError(f"the token server was unreachable: {exc!r}") from exc
+        if resp.status_code in (400, 401):
+            raise SourceError(
+                f"the token server refused the client id and secret "
+                f"(HTTP {resp.status_code}); check them and the environment"
+            )
+        if resp.status_code != 200:
+            raise SourceError(f"the token server answered HTTP {resp.status_code}")
+        try:
+            body = resp.json()
+            token = str(body["access_token"])
+            lifetime = float(body.get("expires_in", 3600))
+        except (ValueError, KeyError, TypeError) as exc:
+            raise SourceError("the token server's answer had no access_token") from exc
+        self._expires = self._clock() + max(lifetime - EARLY, 0.0)
+        return token

apographe/core/registry.py

@@ -0,0 +1,33 @@
+"""Find a source by its id.
+
+Sources are listed by "module:Class" text rather than imported up front, so
+running one source never imports another's dependencies (e-kalathi works
+without the browser that the fuel source needs).
+"""
+
+from __future__ import annotations
+
+from importlib import import_module
+
+from apographe.core.source import Source
+
+BUILT_IN = {
+    "cy.ekalathi": "apographe.sources.cy.ekalathi:EkalathiSource",
+    "cy.fuel": "apographe.sources.cy.fuel.source:FuelSource",
+}
+
+
+def available() -> dict[str, str]:
+    """Source id -> "module:Class"."""
+    return dict(BUILT_IN)
+
+
+def load(source_id: str) -> Source:
+    """Make the built-in source with this id; KeyError if there is none."""
+    target = available().get(source_id)
+    if target is None:
+        known = ", ".join(sorted(available()))
+        raise KeyError(f"unknown source {source_id!r}; known sources: {known}")
+    module, _, name = target.partition(":")
+    cls = getattr(import_module(module), name)
+    return cls()

apographe/core/source.py

@@ -0,0 +1,82 @@
+"""What every source provides, and what a run hands back."""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+from datetime import date
+from pathlib import Path
+from typing import ClassVar
+
+from apographe.core.storage import Store
+
+
+@dataclass
+class Options:
+    """Settings for one run, shared by every source.
+
+    A source reads the ones that make sense for it and ignores the rest.
+    """
+
+    day: date
+    limit: int | None = None  # only the first N items, for a quick trial run
+    only: list[str] = field(default_factory=list)  # only these item ids
+    delay: float | None = None  # seconds between requests; None = source default
+    inputs: list[Path] = field(default_factory=list)  # files a job imports
+    force: bool = False  # redo work that already exists
+
+
+@dataclass
+class Report:
+    """Rows written per table. Counts only, so it is safe to print in public logs."""
+
+    rows: dict[str, int] = field(default_factory=dict)
+    files: list[Path] = field(default_factory=list)
+
+    def add(self, table: str, count: int, path: Path | None = None) -> None:
+        """Record ``count`` rows written to ``table`` (and the file, if given)."""
+        self.rows[table] = self.rows.get(table, 0) + count
+        if path is not None:
+            self.files.append(path)
+
+    def merge(self, other: Report) -> None:
+        """Add another report's rows and files to this one."""
+        for table, count in other.rows.items():
+            self.add(table, count)
+        self.files.extend(other.files)
+
+
+class SourceError(RuntimeError):
+    """The source answered, but not with what we expected."""
+
+
+class LoginWall(SourceError):
+    """The source now asks for a login, so there is nothing to collect."""
+
+
+class Source(ABC):
+    """One place data comes from, such as a country's fuel price observatory.
+
+    Subclasses set the class attributes and implement :meth:`run`. Everything a
+    source writes goes through ``out``, under the source's own folder.
+    """
+
+    id: ClassVar[str]  # "cy.ekalathi": country code, then a short name
+    title: ClassVar[str]
+    country: ClassVar[str]  # ISO 3166-1 alpha-2, e.g. "CY"
+    homepage: ClassVar[str]
+    timezone: ClassVar[str] = "UTC"  # the day a run belongs to is taken here
+    jobs: ClassVar[dict[str, str]]  # job name -> one-line description
+    default_jobs: ClassVar[tuple[str, ...]]  # what `apographe run <id>` does
+    env: ClassVar[dict[str, str]] = {}  # variable -> what it is
+    job_env: ClassVar[dict[str, tuple[str, ...]]] = {}  # job -> variables it needs
+    extras: ClassVar[dict[str, str]] = {}  # job -> pip extra it needs
+
+    @property
+    def folder(self) -> str:
+        """Where the source writes inside a store: "cy.ekalathi" -> "cy/ekalathi"."""
+        return self.id.replace(".", "/")
+
+    @abstractmethod
+    def run(self, job: str, out: Store, options: Options) -> Report:
+        """Run one job, write its tables to ``out`` and report what was written."""

apographe/core/storage.py

@@ -0,0 +1,121 @@
+"""Write tables to a folder: CSV (optionally gzipped) and raw JSON responses.
+
+Every write goes to a temporary file first and is renamed into place, so a run
+that dies half way never leaves a truncated file behind.
+"""
+
+from __future__ import annotations
+
+import csv
+import gzip
+import io
+import json
+import os
+from collections.abc import Iterable, Mapping
+from pathlib import Path
+from typing import Any
+
+
+class Store:
+    """A folder that tables are written into, scoped to one source."""
+
+    def __init__(self, root: Path | str) -> None:
+        self.root = Path(root)
+
+    def scope(self, folder: str) -> Store:
+        """A store for the subfolder ``folder``, e.g. ``"cy/ekalathi"``."""
+        return Store(self.root / folder)
+
+    def path(self, rel: str) -> Path:
+        """The full path of ``rel`` inside this store."""
+        return self.root / rel
+
+    def exists(self, rel: str) -> bool:
+        """Whether ``rel`` has been written already."""
+        return self.path(rel).exists()
+
+    def write_csv(
+        self, rel: str, rows: Iterable[Mapping[str, Any]], fields: list[str]
+    ) -> tuple[Path, int]:
+        """Write rows as CSV (gzipped when ``rel`` ends in .gz). Returns path, count."""
+        path = self.path(rel)
+        count = 0
+        with _atomic(path) as fh:
+            writer = csv.DictWriter(
+                fh, fieldnames=fields, extrasaction="ignore", lineterminator="\n"
+            )
+            writer.writeheader()
+            for row in rows:
+                writer.writerow(row)
+                count += 1
+        return path, count
+
+    def read_csv(self, rel: str) -> list[dict[str, str]]:
+        """Read a CSV (gzipped or not) back as dicts; ``[]`` if it does not exist."""
+        path = self.path(rel)
+        if not path.exists():
+            return []
+        opener = gzip.open if path.suffix == ".gz" else open
+        with opener(path, "rt", encoding="utf-8", newline="") as fh:
+            return list(csv.DictReader(fh))
+
+    def append_csv(
+        self, rel: str, rows: Iterable[Mapping[str, Any]], fields: list[str]
+    ) -> int:
+        """Append rows to a plain CSV, writing the header if the file is new.
+
+        Used for progress files that a later run picks up from, so it is not
+        atomic: each call adds whole rows and flushes.
+        """
+        path = self.path(rel)
+        path.parent.mkdir(parents=True, exist_ok=True)
+        new = not path.exists()
+        count = 0
+        with open(path, "a", encoding="utf-8", newline="") as fh:
+            writer = csv.DictWriter(
+                fh, fieldnames=fields, extrasaction="ignore", lineterminator="\n"
+            )
+            if new:
+                writer.writeheader()
+            for row in rows:
+                writer.writerow(row)
+                count += 1
+        return count
+
+    def write_json(self, rel: str, payload: Any) -> Path:
+        """Save a response untouched (gzipped when ``rel`` ends in .gz)."""
+        path = self.path(rel)
+        with _atomic(path) as fh:
+            json.dump(payload, fh, ensure_ascii=False)
+        return path
+
+
+class _atomic:
+    """Open a text file for writing that only appears once it is complete."""
+
+    def __init__(self, path: Path) -> None:
+        self.path = path
+        self.tmp = path.with_name(f".{path.name}.tmp")
+        self.file: io.BufferedWriter | None = None
+        self.fh: io.TextIOWrapper | None = None
+
+    def __enter__(self) -> io.TextIOWrapper:
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        self.file = open(self.tmp, "wb")
+        stream: Any = self.file  # a plain or gzip binary stream
+        if self.path.suffix == ".gz":
+            # mtime=0 keeps the bytes identical for identical content, so
+            # re-running a day does not create a spurious git change.
+            stream = gzip.GzipFile(filename="", mode="wb", fileobj=self.file, mtime=0)
+        self.fh = io.TextIOWrapper(stream, encoding="utf-8", newline="")
+        return self.fh
+
+    def __exit__(self, exc_type, exc, tb) -> None:
+        if self.fh is None or self.file is None:
+            return
+        self.fh.close()  # also finishes the gzip stream, if any
+        self.file.close()
+        if exc_type is None:
+            os.replace(self.tmp, self.path)
+        else:
+            self.tmp.unlink(missing_ok=True)