sourceledger 0.1.0__py3-none-any.whl

sourceledger/__init__.py

@@ -0,0 +1,42 @@
+"""Fetch, verify and validate the public sources a dataset is built from.
+
+Every download keeps a provenance record, and every output row names the bytes it came from.
+
+The names below are the public API. Until 1.0, a breaking change to them, to the keys of
+sources.toml, or to the fetch-record format comes only in a minor version, listed in the
+changelog. `sourceledger.ledger` and `sourceledger.cli` may change in any release.
+"""
+
+from sourceledger.cli import main
+from sourceledger.ledger import (
+    USER_AGENT,
+    FetchError,
+    FetchRecord,
+    Parser,
+    Source,
+    build,
+    fetch,
+    load_manifest,
+    parse_csv,
+    read_output,
+    revisions,
+    validate,
+    verified,
+)
+
+__all__ = [
+    "USER_AGENT",
+    "FetchError",
+    "FetchRecord",
+    "Parser",
+    "Source",
+    "build",
+    "fetch",
+    "load_manifest",
+    "main",
+    "parse_csv",
+    "read_output",
+    "revisions",
+    "validate",
+    "verified",
+]

sourceledger/cli.py

@@ -0,0 +1,86 @@
+"""The command line: `<project> fetch|build|validate|revisions`, with the project's own parser."""
+
+from __future__ import annotations
+
+import argparse
+import csv
+import logging
+import sys
+from pathlib import Path
+
+from sourceledger.ledger import (
+    USER_AGENT,
+    FetchError,
+    FetchRecord,
+    Parser,
+    build,
+    fetch,
+    load_manifest,
+    one_line,
+    parse_csv,
+    read_output,
+    revisions,
+    validate,
+)
+
+
+def _fetch_line(record: FetchRecord) -> str:
+    return f"{record.source_id}  {record.bytes:>9,} B  {record.sha256[:12]}  {record.vintage}  {record.url}"
+
+
+def main(
+    argv: list[str] | None = None,
+    *,
+    parse: Parser = parse_csv,
+    prog: str = "sourceledger",
+    user_agent: str = USER_AGENT,
+) -> int:
+    """Run one stage: fetch, build, validate, or revisions against an earlier copy of the fetch log.
+
+    A project's own entry point calls this with its `parse`, its name as `prog`, and that name
+    as `user_agent`. Returns the exit code: 0, or 1 for any failure or validation problem.
+    """
+    parser = argparse.ArgumentParser(prog=prog, description="Fetch, build and validate the sources in sources.toml.")
+    parser.add_argument("stage", choices=["fetch", "build", "validate", "revisions"])
+    parser.add_argument("before", nargs="?", type=Path, help="revisions only: an earlier copy of data/fetches.jsonl")
+    args = parser.parse_args(argv)
+    # Retries and recovered fetch logs are reported as warnings, on stderr.
+    logging.basicConfig(format=f"{prog}: %(message)s", level=logging.WARNING)
+    try:
+        problems = _run(args.stage, args.before, parse, user_agent, parser)
+    except FetchError as error:
+        for record in error.records:
+            print(_fetch_line(record))
+        for failure in error.failures:
+            print(f"{args.stage}: {failure}", file=sys.stderr)
+        return 1
+    # OSError: a missing file or a failed download. ValueError: a bad manifest, a download
+    # that fails its checks, or a stale fetch. csv.Error: a source that is not readable CSV.
+    except (OSError, ValueError, csv.Error) as error:
+        print(f"{args.stage}: {one_line(error)}", file=sys.stderr)
+        return 1
+    for problem in problems:
+        print(problem, file=sys.stderr)
+    return 1 if problems else 0
+
+
+def _run(stage: str, before: Path | None, parse: Parser, user_agent: str, parser: argparse.ArgumentParser) -> list[str]:
+    """Run `stage`, print its report, and return its problems: only validate has any."""
+    match stage:
+        case "revisions":
+            if before is None:
+                parser.error("revisions needs an earlier copy of the fetch log")
+            # Not a failure: a revised source is news, and the tripwire test decides whether it breaks anything.
+            for line in revisions(before):
+                print(line)
+        case "fetch":
+            for record in fetch(load_manifest(), user_agent=user_agent):
+                print(_fetch_line(record))
+        case "build":
+            for path in build(load_manifest(), parse=parse):
+                print(f"{len(read_output(path))} rows -> {path}")
+        case _:
+            problems = validate(load_manifest())
+            print(f"{len(problems)} problems")
+            return problems
+    return []

sourceledger/ledger.py

@@ -0,0 +1,626 @@
+"""Fetch, build and validate the sources a dataset is built from, as `sources.toml` declares them.
+
+Raw files stay in data/raw/ and outputs in data/processed/. Each fetch leaves a
+`<file>.fetch.json` record beside its raw file: the URL, when it was retrieved, the SHA-256
+of the bytes, the vintage the manifest declares, and the server's Last-Modified and ETag
+headers. A later fetch sends those headers back, and when the server answers 304 Not
+Modified it keeps the raw file and its record as they are.
+
+Each fetch that brings bytes a source has not had before also appends its record to
+data/fetches.jsonl, the history of every version of every source the pipeline has seen.
+
+Paths are relative to the working directory, the project root.
+"""
+
+from __future__ import annotations
+
+import csv
+import dataclasses
+import errno
+import hashlib
+import http.client
+import io
+import ipaddress
+import json
+import logging
+import os
+import re
+import socket
+import tempfile
+import time
+import tomllib
+import urllib.error
+import urllib.request
+import urllib.response
+from collections.abc import Callable
+from datetime import UTC, datetime
+from email.utils import parsedate_to_datetime
+from http import HTTPStatus
+from importlib.metadata import version
+from pathlib import Path, PurePosixPath
+from typing import IO, TYPE_CHECKING, Any
+from urllib.parse import urlsplit
+
+if TYPE_CHECKING:
+    from email.message import Message
+
+MANIFEST = Path("sources.toml")
+RAW = Path("data/raw")
+PROCESSED = Path("data/processed")
+FETCH_LOG = Path("data/fetches.jsonl")
+
+IMPLEMENTED = "implemented"
+UNIMPLEMENTED = "unimplemented"
+# Bounds each connect and each read. A server that trickles bytes is stopped by _BUDGET_S.
+_SOCKET_TIMEOUT_S = 30
+# One source's whole fetch, every attempt and wait included, so a stalled publisher costs
+# at most 5 minutes of a CI job.
+_BUDGET_S = 300
+# Waits of 5, 10 and 20 s between four attempts ride out a brief publisher outage.
+_ATTEMPTS = 4
+_BACKOFF_S = 5
+# A server asking for hours would otherwise stall the run.
+_MAX_RETRY_AFTER_S = 120
+# Statuses a publisher recovers from. 501 and 505 never will.
+_RETRIED = frozenset({429, 500, 502, 503, 504})
+_CHUNK_BYTES = 1024 * 1024
+# A network error or timeout, or a connection dropped or garbled mid-response.
+_TRANSIENT = (OSError, http.client.HTTPException)
+# A raw file or fetch record that is missing, edited or unreadable.
+_UNUSABLE = (OSError, ValueError)
+# A runaway or wrong URL fails the fetch instead of filling the disk. A big source sets
+# its own `max_bytes` in sources.toml.
+_MAX_BYTES = 200 * 1024 * 1024
+_SHA256 = re.compile(r"[0-9a-f]{64}")
+
+# Sent with every request, so a publisher can tell who is fetching. `main` takes the project's own.
+USER_AGENT = f"sourceledger/{version('sourceledger')}"
+
+_logger = logging.getLogger("sourceledger")
+
+type Source = dict[str, Any]
+# What the opener returns: an HTTP response, or a file:// one.
+type _Response = http.client.HTTPResponse | urllib.response.addinfourl
+# Reads one raw file as its columns and rows. `parse_csv` is the default.
+type Parser = Callable[[Source, bytes], tuple[list[str], list[dict[str, str]]]]
+
+
+@dataclasses.dataclass(frozen=True, slots=True)
+class FetchRecord:
+    """What one download was, so every output row traces back to the bytes it came from."""
+
+    source_id: str
+    url: str
+    retrieved_at: str
+    sha256: str
+    bytes: int
+    vintage: str
+    # Empty when the server sends none. Defaults keep records written before these fields readable.
+    last_modified: str = ""
+    etag: str = ""
+    # Where the bytes came from after redirects. Empty when there were none.
+    final_url: str = ""
+
+
+class FetchError(OSError):
+    """One or more sources failed to fetch. The others were fetched and recorded as usual."""
+
+    def __init__(self, records: list[FetchRecord], failures: list[str]) -> None:
+        super().__init__("\n".join(failures))
+        self.records = records  # the sources that succeeded
+        self.failures = failures  # one line per source that did not
+
+
+def load_manifest(path: Path = MANIFEST) -> list[Source]:
+    """The `[[source]]` tables of sources.toml, after checking ids, URLs, caps and pins."""
+    with path.open("rb") as fh:
+        sources = tomllib.load(fh).get("source", [])
+    if not isinstance(sources, list) or not all(isinstance(s, dict) and isinstance(s.get("id"), str) for s in sources):
+        msg = "sources.toml: every [[source]] needs a string id"
+        raise ValueError(msg)
+    ids = [s["id"] for s in sources]
+    if bad := [i for i in ids if not re.fullmatch(r"[\w-]+", i)]:
+        msg = f"ids {bad} in sources.toml may hold only letters, digits, _ and -"
+        raise ValueError(msg)
+    if dupes := sorted({i for i in ids if ids.count(i) > 1}):
+        msg = f"ids {dupes} appear more than once in sources.toml"
+        raise ValueError(msg)
+    if no_url := [s["id"] for s in implemented(sources) if not isinstance(s.get("url"), str)]:
+        msg = f"implemented sources {no_url} in sources.toml have no url"
+        raise ValueError(msg)
+    # Not isinstance: bool is an int subclass, so `max_bytes = true` would pass as a 1-byte cap.
+    caps = {s["id"]: s.get("max_bytes", _MAX_BYTES) for s in sources}
+    if bad_cap := [i for i, cap in caps.items() if type(cap) is not int or cap < 1]:
+        msg = f"sources {bad_cap} in sources.toml need max_bytes to be a positive whole number"
+        raise ValueError(msg)
+    for source in sources:
+        _check_pin(source)
+        _check_vintage(source)
+    return sources
+
+
+def _check_pin(source: Source) -> None:
+    """Normalize a `sha256` pin to lower case, as `sha256sum` prints it, and reject one that is not a SHA-256."""
+    if (pin := source.get("sha256")) is None:
+        return
+    if not (isinstance(pin, str) and _SHA256.fullmatch(pin.lower())):
+        msg = f"{source['id']}: sha256 in sources.toml must be 64 hexadecimal characters"
+        raise ValueError(msg)
+    source["sha256"] = pin.lower()
+
+
+def _check_vintage(source: Source) -> None:
+    """Accept the old key `release` for `vintage`, with a warning."""
+    if (release := source.pop("release", None)) is None:
+        return
+    if source.get("vintage") is not None:
+        msg = f"{source['id']}: give vintage or release in sources.toml, not both"
+        raise ValueError(msg)
+    _logger.warning("%s: `release` in sources.toml is now `vintage`; rename it", source["id"])
+    source["vintage"] = release
+
+
+def _vintage(source: Source) -> str:
+    # str(): a TOML date (`vintage = 2024-06-01`) loads as a datetime.date.
+    return str(source.get("vintage", source.get("release", "not stated")))
+
+
+def implemented(sources: list[Source]) -> list[Source]:
+    """The sources the pipeline reads."""
+    return [s for s in sources if s.get("status") == IMPLEMENTED]
+
+
+def raw_name(source: Source) -> str:
+    """The raw file's name: the source id plus the URL's extension, e.g. `example-co2.csv`."""
+    return source["id"] + PurePosixPath(urlsplit(source["url"]).path).suffix
+
+
+def _write_atomic(path: Path, data: bytes) -> None:
+    """Write a new temp file beside `path`, flush it to disk, then rename it over `path`.
+
+    A reader never sees half a file. The temp file gets a fresh name, so two runs at once
+    don't share it, and it is created rather than opened, so a symlink in its place is
+    never followed.
+    """
+    fd, temp = tempfile.mkstemp(dir=path.parent, prefix=f".{path.name}.", suffix=".part")
+    try:
+        with os.fdopen(fd, "wb") as fh:
+            fh.write(data)
+            fh.flush()
+            os.fsync(fh.fileno())
+        Path(temp).replace(path)
+    except BaseException:
+        Path(temp).unlink(missing_ok=True)
+        raise
+
+
+def _is_public(host: str) -> bool:
+    """Whether every address `host` resolves to is on the public internet."""
+    try:
+        infos = socket.getaddrinfo(host, None)
+    except OSError:
+        return True  # unresolvable: the request itself will fail
+    return all(ipaddress.ip_address(info[4][0]).is_global for info in infos)
+
+
+class _NoRedirectInward(urllib.request.HTTPRedirectHandler):
+    """Refuses a redirect from a public host to a loopback, private or link-local one.
+
+    Otherwise a third-party server could have the fetch save an internal service's response
+    under the manifest's URL. A source that lives on an internal host can still redirect
+    within it.
+    """
+
+    def redirect_request(
+        self,
+        req: urllib.request.Request,
+        fp: IO[bytes],
+        code: int,
+        msg: str,
+        headers: http.client.HTTPMessage,
+        newurl: str,
+    ) -> urllib.request.Request | None:
+        """Raise HTTPError for an inward redirect; follow any other as urllib does."""
+        if _is_public(urlsplit(req.full_url).hostname or "") and not _is_public(urlsplit(newurl).hostname or ""):
+            reason = f"refused a redirect from a public host to {newurl}"
+            raise urllib.error.HTTPError(newurl, code, reason, headers, fp)
+        return super().redirect_request(req, fp, code, msg, headers, newurl)
+
+
+_OPENER = urllib.request.build_opener(_NoRedirectInward())
+
+
+def _open(request: urllib.request.Request, timeout: float) -> _Response:
+    # NOTE: the opener follows file:// too, which the tests use for fixtures. The URLs come
+    # from sources.toml, which is as trusted as the code.
+    return _OPENER.open(request, timeout=timeout)
+
+
+def _retry_after(headers: Message | None) -> float:
+    """Seconds a 429 or 503 asks to wait, from Retry-After as seconds or an HTTP date, capped."""
+    value = (headers.get("Retry-After", "") if headers else "").strip()
+    if value.isascii() and value.isdigit():
+        seconds = float(value)
+    else:
+        try:
+            seconds = (parsedate_to_datetime(value) - datetime.now(UTC)).total_seconds()
+        except (TypeError, ValueError):  # absent, malformed, or a date with no time zone
+            return 0.0
+    return min(max(seconds, 0.0), _MAX_RETRY_AFTER_S)
+
+
+def _content_length(headers: Message) -> int | None:
+    """The Content-Length header as a byte count, or None when it is absent or not a plain number."""
+    value = headers.get("Content-Length", "").strip()
+    return int(value) if value.isascii() and value.isdigit() else None
+
+
+def _read(response: _Response, source: Source, deadline: float) -> bytes:
+    """The whole body, refused past the source's `max_bytes` or past the fetch's time budget."""
+    cap = source.get("max_bytes", _MAX_BYTES)
+    encoding = response.headers.get("Content-Encoding", "identity").strip().lower()
+    if encoding not in {"", "identity"}:
+        msg = f"{source['id']}: served with Content-Encoding {encoding}, so its bytes are not the file itself"
+        raise ValueError(msg)
+    chunks: list[bytes] = []
+    size = 0
+    while chunk := response.read(_CHUNK_BYTES):
+        size += len(chunk)
+        if size > cap:
+            msg = f"{source['id']}: download exceeds {cap} bytes; raise max_bytes in sources.toml if expected"
+            raise ValueError(msg)
+        if time.monotonic() > deadline:
+            msg = f"download still unfinished after {_BUDGET_S} s"
+            raise TimeoutError(msg)
+        chunks.append(chunk)
+    return b"".join(chunks)
+
+
+def _get(source: Source, previous: FetchRecord | None, user_agent: str) -> tuple[bytes, Message, str] | FetchRecord:
+    """GET one source's URL: its body, headers and final URL, retrying failures a publisher recovers from.
+
+    The request is conditional on the ETag and Last-Modified of a `previous` record, which
+    is returned as it is when the server answers 304 Not Modified.
+
+    429, 500, 502, 503 and 504, network errors and a body shorter than its Content-Length
+    are retried with exponential backoff, waiting at least the server's Retry-After (up to
+    120 s). The whole fetch stops after 300 s. Any other error status is raised at once.
+
+    Raises ValueError for a body over the source's `max_bytes` (default 200 MiB), or one
+    served compressed.
+    """
+    conditional = _conditional(previous)
+    headers = {"User-Agent": user_agent, "Accept-Encoding": "identity", **conditional}
+    request = urllib.request.Request(source["url"], headers=headers)  # noqa: S310  # see _open
+    deadline = time.monotonic() + _BUDGET_S
+    for attempt in range(1, _ATTEMPTS + 1):
+        wait = _BACKOFF_S * 2 ** (attempt - 1)
+        try:
+            with _open(request, timeout=_SOCKET_TIMEOUT_S) as response:
+                payload = _read(response, source, deadline)
+                # A connection dropped mid-body can end the read early without an error.
+                expected = _content_length(response.headers)
+                if expected is not None and expected != len(payload):
+                    raise http.client.IncompleteRead(payload, expected - len(payload))
+                return payload, response.headers, response.url
+        except urllib.error.HTTPError as error:
+            # urllib raises a 304 like an error. Only a conditional request can get one.
+            if error.code == HTTPStatus.NOT_MODIFIED and previous and conditional:
+                return previous
+            if attempt == _ATTEMPTS or error.code not in _RETRIED:
+                raise
+            failure: Exception = error
+            wait = max(wait, _retry_after(error.headers))
+        except _TRANSIENT as error:
+            if attempt == _ATTEMPTS:
+                raise
+            failure = error
+        if time.monotonic() + wait > deadline:
+            raise failure
+        _logger.warning(
+            "%s: attempt %d of %d failed (%s); retrying in %.0f s", source["id"], attempt, _ATTEMPTS, failure, wait
+        )
+        time.sleep(wait)
+    raise AssertionError  # unreachable: the last attempt returns or raises
+
+
+def _previous(source: Source, raw: Path) -> FetchRecord | None:
+    """The last fetch record of `source`, if it and its raw file still match the manifest."""
+    try:
+        record, _ = verified(source, raw=raw)
+    except _UNUSABLE:
+        return None
+    return record
+
+
+def _conditional(previous: FetchRecord | None) -> dict[str, str]:
+    """The headers that ask the server to send the source only if it changed since `previous`."""
+    if previous is None:
+        return {}
+    headers = {"If-None-Match": previous.etag, "If-Modified-Since": previous.last_modified}
+    return {name: value for name, value in headers.items() if value}
+
+
+def _last_records(log: Path) -> dict[str, dict[str, Any]]:
+    """The record most recently logged for each source id, from the fetch log.
+
+    A last line cut off by an interrupted fetch is skipped with a warning. Any other line
+    that is not a record is an error that names the line.
+    """
+    if not log.exists():
+        return {}
+    text = log.read_text(encoding="utf-8")
+    lines = text.splitlines()
+    records: dict[str, dict[str, Any]] = {}
+    for number, line in enumerate(lines, start=1):
+        if not line.strip():
+            continue
+        try:
+            record = json.loads(line)
+        except json.JSONDecodeError as error:
+            if number == len(lines) and not text.endswith("\n"):
+                _logger.warning("%s: skipping a last line cut off by an interrupted fetch", log)
+                continue
+            msg = f"{log}:{number} is not a fetch record ({error})"
+            raise ValueError(msg) from error
+        records[record["source_id"]] = record
+    return records
+
+
+def _append(log: Path, record: FetchRecord) -> None:
+    """Add `record` as one line at the end of the log, in one write, flushed to disk.
+
+    A last line cut off by an interrupted fetch is dropped first, so the new line starts
+    on a line of its own.
+    """
+    if log.exists() and (text := log.read_bytes()) and not text.endswith(b"\n"):
+        _logger.warning("%s: dropping a last line cut off by an interrupted fetch", log)
+        os.truncate(log, text.rfind(b"\n") + 1)
+    with log.open("a", encoding="utf-8") as fh:
+        fh.write(json.dumps(dataclasses.asdict(record)) + "\n")
+        fh.flush()
+        os.fsync(fh.fileno())
+
+
+def revisions(before: Path, *, log: Path = FETCH_LOG) -> list[str]:
+    """One Markdown line per source whose bytes changed between `before`, an earlier copy of the log, and `log`.
+
+    A source that `before` never logged is new, not revised, and is left out. Raises
+    FileNotFoundError when `before` does not exist: an empty report must mean no
+    revisions, not a missing copy.
+    """
+    if not before.exists():
+        raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), str(before))
+    old, new = _last_records(before), _last_records(log)
+    return [
+        f"- `{sid}`: {old[sid]['sha256'][:12]} -> {r['sha256'][:12]}, {r['bytes']:,} bytes, "
+        f"fetched {r['retrieved_at']} from <{r['url']}>"
+        for sid, r in new.items()
+        if sid in old and old[sid]["sha256"] != r["sha256"]
+    ]
+
+
+def fetch(
+    sources: list[Source], *, raw: Path = RAW, log: Path = FETCH_LOG, user_agent: str = USER_AGENT
+) -> list[FetchRecord]:
+    """Download each implemented source into `raw`, with a fetch record beside it.
+
+    A source the server reports unchanged since its last intact fetch keeps that raw file
+    and record, and its old record is returned. A record whose bytes differ from the last
+    ones logged for its source is appended to `log`.
+
+    Every source is tried. Raises FetchError after the loop when any failed, listing each
+    with its reason; nothing is written for those. A source fails for a download over its
+    `max_bytes` (default 200 MiB), bytes that miss the `sha256` the manifest pins, or an
+    error that outlasts its retries.
+    """
+    raw.mkdir(parents=True, exist_ok=True)
+    log.parent.mkdir(parents=True, exist_ok=True)
+    logged = {sid: r["sha256"] for sid, r in _last_records(log).items()}
+    records: list[FetchRecord] = []
+    failures: list[str] = []
+    for source in implemented(sources):
+        try:
+            records.append(_fetch_one(source, raw, log, logged, user_agent))
+        except ValueError as error:  # messages already name the source
+            failures.append(str(error))
+        except _TRANSIENT as error:
+            failures.append(f"{source['id']}: fetching {source['url']} failed: {error}")
+    if failures:
+        raise FetchError(records, failures)
+    return records
+
+
+def _fetch_one(source: Source, raw: Path, log: Path, logged: dict[str, str], user_agent: str) -> FetchRecord:
+    """Fetch one source, write its raw file, log and record, and return the record."""
+    response = _get(source, _previous(source, raw), user_agent)
+    if isinstance(response, FetchRecord):
+        # A version written to disk by a fetch that died before logging it gets logged now.
+        _log_new_version(log, logged, response)
+        return response
+    payload, headers, final_url = response
+    digest = hashlib.sha256(payload).hexdigest()
+    if source.get("sha256", digest) != digest:
+        msg = f"{source['id']}: downloaded bytes hash to {digest}, not the sha256 in sources.toml"
+        raise ValueError(msg)
+    record = FetchRecord(
+        source_id=source["id"],
+        url=source["url"],
+        retrieved_at=datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ"),
+        sha256=digest,
+        bytes=len(payload),
+        vintage=_vintage(source),
+        # Kept for a conditional request (If-Modified-Since, If-None-Match) on a later run.
+        last_modified=headers.get("Last-Modified", ""),
+        etag=headers.get("ETag", ""),
+        final_url="" if final_url == source["url"] else final_url,
+    )
+    name = raw_name(source)
+    _write_atomic(raw / name, payload)
+    # Logged before the record is written: until the record matches the raw file, the next
+    # run fetches in full, so a crash in between loses nothing.
+    _log_new_version(log, logged, record)
+    _write_atomic(raw / f"{name}.fetch.json", (json.dumps(dataclasses.asdict(record), indent=2) + "\n").encode())
+    return record
+
+
+def _log_new_version(log: Path, logged: dict[str, str], record: FetchRecord) -> None:
+    # Appended, not rewritten: the log is history. A re-download of the same bytes adds nothing.
+    if logged.get(record.source_id) != record.sha256:
+        _append(log, record)
+        logged[record.source_id] = record.sha256
+
+
+def verified(source: Source, *, raw: Path = RAW) -> tuple[FetchRecord, bytes]:
+    """The fetch record and raw bytes for `source`, after checking that both are current.
+
+    Raises ValueError when the record was made for a different request than the manifest
+    now makes (another URL, vintage or `sha256` pin), or when the raw file no longer hashes
+    to the record.
+    """
+    name = raw_name(source)
+    try:
+        record = FetchRecord(**json.loads((raw / f"{name}.fetch.json").read_text(encoding="utf-8")))
+    except TypeError as error:
+        msg = f"{name}.fetch.json is not a fetch record; re-run fetch"
+        raise ValueError(msg) from error
+    if (record.source_id, record.url) != (source["id"], source["url"]):
+        msg = f"{name} was fetched from {record.url}, but sources.toml now requests {source['url']}; re-run fetch"
+        raise ValueError(msg)
+    if record.vintage != _vintage(source):
+        msg = f"{name} was fetched as vintage {record.vintage!r}, not {_vintage(source)!r}; re-run fetch"
+        raise ValueError(msg)
+    if source.get("sha256", record.sha256).lower() != record.sha256:
+        msg = f"{name} hashes to {record.sha256}, not the sha256 sources.toml now pins; re-run fetch"
+        raise ValueError(msg)
+    payload = (raw / name).read_bytes()
+    if hashlib.sha256(payload).hexdigest() != record.sha256:
+        msg = f"{name} changed since it was fetched; re-run fetch"
+        raise ValueError(msg)
+    return record, payload
+
+
+def parse_csv(source: Source, payload: bytes) -> tuple[list[str], list[dict[str, str]]]:
+    """One raw CSV file as its columns and rows: the default `Parser`.
+
+    Raises ValueError for a repeated column name, or a row with more or fewer fields than the header.
+    """
+    # utf-8-sig drops the byte-order mark that spreadsheet exports start with. Plain UTF-8 reads the same.
+    text = payload.decode("utf-8-sig")
+    reader = csv.DictReader(io.StringIO(text, newline=""))
+    columns = list(reader.fieldnames or [])
+    # DictReader keeps only the last of two same-named columns, silently dropping the other.
+    if dupes := sorted({c for c in columns if columns.count(c) > 1}):
+        msg = f"{source['id']}: columns {dupes} appear more than once in the header"
+        raise ValueError(msg)
+    rows = []
+    for row in reader:
+        # DictReader files the surplus fields of a long row under the key None.
+        if None in row:
+            msg = f"{source['id']}: line {reader.line_num} has more fields than the header"
+            raise ValueError(msg)
+        # It fills the missing fields of a short row with None, which would publish as "".
+        if None in row.values():
+            msg = f"{source['id']}: line {reader.line_num} has fewer fields than the header"
+            raise ValueError(msg)
+        rows.append(row)
+    return columns, rows
+
+
+def _stamp(record: FetchRecord) -> dict[str, str]:
+    """The provenance columns `build` adds to every output row."""
+    return {
+        "source_id": record.source_id,
+        "source_url": record.url,
+        "retrieved_at": record.retrieved_at,
+        "vintage": record.vintage,
+        "raw_sha256": record.sha256,
+    }
+
+
+def build(
+    sources: list[Source], *, raw: Path = RAW, processed: Path = PROCESSED, parse: Parser = parse_csv
+) -> list[Path]:
+    """Write each implemented source to `processed/<id>.csv`, each row stamped with its fetch record.
+
+    `parse` reads each raw file. Give it a function that branches on `source["id"]` for sources that are not CSV.
+    """
+    processed.mkdir(parents=True, exist_ok=True)
+    written = []
+    for source in implemented(sources):
+        record, payload = verified(source, raw=raw)
+        stamp = _stamp(record)
+        columns, rows = parse(source, payload)
+        if clash := stamp.keys() & set(columns):
+            msg = f"{source['id']}: source columns {sorted(clash)} collide with the provenance columns"
+            raise ValueError(msg)
+        buffer = io.StringIO(newline="")
+        writer = csv.DictWriter(buffer, fieldnames=[*stamp, *columns])
+        writer.writeheader()
+        # The stamp last: a parser's row cannot overwrite a provenance column.
+        writer.writerows({**row, **stamp} for row in rows)
+        out = processed / f"{source['id']}.csv"
+        _write_atomic(out, buffer.getvalue().encode())
+        written.append(out)
+    return written
+
+
+def read_output(path: Path) -> list[dict[str, str]]:
+    """A published output file, as written."""
+    with path.open(newline="", encoding="utf-8") as fh:
+        return list(csv.DictReader(fh))
+
+
+def _terms_problems(source: Source) -> list[str]:
+    """What an implemented source is missing of its reuse terms: in the publisher's words, and machine-readable."""
+    sid, problems = source["id"], []
+    if not (source.get("terms") and source.get("terms_url")):
+        problems.append(f"{sid}: no terms or terms_url; record the publisher's reuse terms")
+    license_ = source.get("license")
+    if not (isinstance(license_, str) and license_.strip()):
+        problems.append(f"{sid}: no license; give the SPDX id of the terms, or LicenseRef-<name> for custom ones")
+    if not isinstance(source.get("redistribute"), bool):
+        problems.append(f"{sid}: redistribute must be true or false: may the raw file be republished?")
+    return problems
+
+
+def validate(sources: list[Source], *, raw: Path = RAW, processed: Path = PROCESSED) -> list[str]:
+    """Check the published output against the manifest and the fetch records, and return one string per problem."""
+    problems = []
+    ids = {source["id"] for source in sources}
+    # Output of a source since removed or renamed would otherwise pass as published data.
+    problems += [
+        f"{out}: no source {out.stem!r} in sources.toml; delete it"
+        for out in sorted(processed.glob("*.csv"))
+        if out.stem not in ids
+    ]
+    for source in sources:
+        sid, status = source["id"], source.get("status")
+        out = processed / f"{sid}.csv"
+        if status == UNIMPLEMENTED:
+            if out.exists():
+                problems.append(f"{sid}: status is unimplemented, but {out} exists")
+            continue
+        if status != IMPLEMENTED:
+            problems.append(f"{sid}: status must be {IMPLEMENTED!r} or {UNIMPLEMENTED!r}, not {status!r}")
+            continue
+        problems += _terms_problems(source)
+        try:
+            record, _ = verified(source, raw=raw)
+            rows = read_output(out)
+        except (OSError, ValueError, csv.Error) as error:
+            problems.append(one_line(error))
+            continue
+        if not rows:
+            problems.append(f"{out}: no rows")
+        stamp = _stamp(record)
+        if any({column: row.get(column) for column in stamp} != stamp for row in rows):
+            problems.append(f"{out}: rows come from an earlier fetch than {raw_name(source)}; re-run build")
+    return problems
+
+
+def one_line(error: Exception) -> str:
+    """One line for an expected failure: a missing file by name, anything else as raised."""
+    if isinstance(error, FileNotFoundError) and error.filename:
+        return f"{error.filename} is missing"
+    return str(error)

sourceledger-0.1.0.dist-info/METADATA

@@ -0,0 +1,182 @@
+Metadata-Version: 2.4
+Name: sourceledger
+Version: 0.1.0
+Summary: Fetch, verify and validate the public sources a dataset is built from, with a provenance record for every download
+Keywords: provenance,data pipeline,reproducibility,open data,research software,FAIR
+Author: Simon van Lierde
+Author-email: Simon van Lierde <90462640+simonvanlierde@users.noreply.github.com>
+License-Expression: MIT
+License-File: LICENSE
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Science/Research
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Scientific/Engineering
+Classifier: Typing :: Typed
+Requires-Python: >=3.12
+Project-URL: Homepage, https://github.com/simonvanlierde/sourceledger
+Project-URL: Issues, https://github.com/simonvanlierde/sourceledger/issues
+Description-Content-Type: text/markdown
+
+# sourceledger
+
+[![python](https://github.com/simonvanlierde/sourceledger/actions/workflows/python.yml/badge.svg)](https://github.com/simonvanlierde/sourceledger/actions/workflows/python.yml)
+[![license](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE)
+
+Build a dataset from public sources where every row says which download it came from.
+
+A project lists its inputs in `sources.toml`: each source's URL, the publisher's reuse
+terms, and whether the raw file may be republished. sourceledger runs four stages over
+that manifest:
+
+- `fetch` downloads each source into `data/raw/`, with a record beside it: the URL, when
+  it was fetched, the SHA-256 and size of the bytes, the vintage the manifest declares,
+  and the server's ETag and Last-Modified. The next fetch sends the ETag back, so an
+  unchanged source is not downloaded again. Each new version of a source is also
+  appended to `data/fetches.jsonl`.
+- `build` re-hashes each download against its record, reads it, and writes
+  `data/processed/<id>.csv` with five provenance columns in front of the source's own:
+  `source_id`, `source_url`, `retrieved_at`, `vintage` and `raw_sha256`.
+- `validate` re-reads the output and reports every problem it finds: output built from
+  an older fetch, a raw file changed since its fetch, a record made for another URL,
+  vintage or pin, a source with no reuse terms, an output with no rows, and output left
+  by a source that is gone or marked unimplemented.
+- `revisions OLD_LOG` lists the sources whose bytes changed since an earlier copy of the
+  fetch log. A weekly CI run can use it to say when a publisher revised its data.
+
+sourceledger has no dependencies: the standard library does all of it.
+
+## Install
+
+```sh
+uv add sourceledger
+```
+
+Until the first release is on PyPI, install it from GitHub:
+`uv add git+https://github.com/simonvanlierde/sourceledger`.
+
+## Quick start
+
+Write a `sources.toml` in the project root:
+
+```toml
+[[source]]
+id = "co2"
+status = "implemented"
+url = "https://raw.githubusercontent.com/datasets/co2-ppm/main/data/co2-annmean-mlo.csv"
+vintage = "main branch, unversioned"
+terms = "ODC-PDDL-1.0 (datasets/co2-ppm package; original data from NOAA GML)"
+terms_url = "https://opendatacommons.org/licenses/pddl/"
+license = "PDDL-1.0"
+redistribute = true
+```
+
+Then run the stages from that directory:
+
+```sh
+sourceledger fetch
+sourceledger build
+sourceledger validate
+```
+
+`sourceledger` reads every source as CSV. For other formats, see
+[Your own readers](#your-own-readers).
+
+## What to commit
+
+Commit `sources.toml` and `data/fetches.jsonl`. The log holds checksums and URLs, not
+data, and it is the history of every version of every source you have fetched.
+
+Keep `data/raw/` and `data/processed/` out of git, so the repository never
+redistributes a publisher's data. The per-file `.fetch.json` records live in
+`data/raw/` too. A fresh clone runs `fetch` before `build` and `validate`.
+
+## sources.toml
+
+Each `[[source]]` table takes these keys:
+
+| Key | Required | Meaning |
+| --- | --- | --- |
+| `id` | yes | Letters, digits, `_` and `-`. Names the raw file (`id` plus the URL's extension) and the output. |
+| `status` | yes | `"implemented"`, read by every stage, or `"unimplemented"`: a source you know about but don't read yet. `validate` fails if one of those has output. |
+| `url` | for implemented sources | What `fetch` downloads. Change it and `build` refuses the old download until you fetch again. |
+| `terms`, `terms_url` | for implemented sources | The publisher's reuse terms in their words, and where they are stated. |
+| `license` | for implemented sources | The same terms as an [SPDX id](https://spdx.org/licenses/), such as `"CC-BY-4.0"`, or `"LicenseRef-<name>"` for terms with none. |
+| `redistribute` | for implemented sources | `true` if the terms let you republish the raw file, `false` if not. |
+| `vintage` | no | The edition you expect, such as `"2024 release"` or a TOML date. Recorded with each fetch and stamped on each row. `"not stated"` when absent. The old key `release` still works, with a warning. |
+| `sha256` | no | The SHA-256 the download must have. Pin it for a fixed release, and `fetch` refuses bytes that changed upstream. |
+| `max_bytes` | no | The largest download `fetch` accepts. 200 MiB unless set. |
+| `reason` | no | A note for people, such as why a source is unimplemented. sourceledger doesn't read it. |
+
+`validate` reports a missing `terms`, `terms_url`, `license` or `redistribute`. The other
+keys are checked when the manifest loads.
+
+## How fetch behaves
+
+- It tries four times, waiting 5, 10 and 20 s between attempts, on 429, 500, 502, 503,
+  504, network errors and a body cut short. A `Retry-After` header, in seconds or as a
+  date, raises the wait to at most 120 s. Any other error status fails at once.
+- Each connect and read times out after 30 s, and one source's whole fetch after 300 s.
+- It asks for the file uncompressed, and refuses a compressed reply. The checksum is
+  always of the file as published.
+- It refuses a redirect from a public host to a loopback, private or link-local
+  address. When a redirect is followed, the record keeps the final URL too.
+- Every source is tried. When some fail, the others are still written, and the run
+  exits 1 with one line per failed source.
+- Each file is written to a new temporary file and renamed into place, so a crash never
+  leaves half a file. A fetch log line cut off by a crash is dropped on the next run.
+
+## Your own readers
+
+`sourceledger` reads every source with `parse_csv`. A project with other formats gives
+it its own reader, a function from a source's manifest table and raw bytes to columns
+and rows, and its own command:
+
+```python
+# src/my_dataset/pipeline.py
+import sourceledger
+from sourceledger import Source
+
+
+def parse(source: Source, payload: bytes) -> tuple[list[str], list[dict[str, str]]]:
+    """Every source is CSV here. Branch on source["id"] for one that is not."""
+    return sourceledger.parse_csv(source, payload)
+
+
+def main(argv: list[str] | None = None) -> int:
+    return sourceledger.main(argv, parse=parse, prog="my-dataset", user_agent="my-dataset")
+```
+
+```toml
+# pyproject.toml
+[project.scripts]
+my-dataset = "my_dataset.pipeline:main"
+```
+
+`my-dataset fetch`, `my-dataset build` and `my-dataset validate` then run with that
+reader, and publishers see `my-dataset` as the User-Agent. The
+[project-templates](https://github.com/simonvanlierde/project-templates) `data-pipeline`
+kind sets all of this up, with CI and a weekly run that reports revised sources.
+
+The same stages are functions too: `load_manifest`, `fetch`, `build`, `validate` and
+`revisions`.
+
+## Stability
+
+Until 1.0, the public API is what `sourceledger` exports in `__all__`, the keys of
+`sources.toml`, and the format of fetch records and the fetch log. A breaking change to
+any of them comes only in a minor version, listed in [CHANGELOG.md](CHANGELOG.md), with
+a warning first where that is cheap. Fetch logs written by an earlier version stay
+readable.
+
+## Development
+
+```sh
+uv sync
+uv tool install prek   # once per machine
+prek install           # git hooks
+just check             # lint, types and tests, as CI runs them
+```
+
+## License
+
+MIT. See [LICENSE](LICENSE).

sourceledger-0.1.0.dist-info/RECORD

@@ -0,0 +1,9 @@
+sourceledger/__init__.py,sha256=Kcn3BS_q1lkS6_kxuQMAuJpWO42SguMYU5gNm9OSUH8,936
+sourceledger/cli.py,sha256=Le7dOIeazrIEjedrsZ0ouD_Oee_9isjg5lS0FClr-iA,3279
+sourceledger/ledger.py,sha256=nEcR5TX3D1H6VhaeKLDNZ2o_FNNbnLzvDu01ksaHm_w,27491
+sourceledger/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+sourceledger-0.1.0.dist-info/licenses/LICENSE,sha256=UyQB0-mQ8EWLdthWsp_QwjcaKj1mz4BgZVTwwBeednA,1073
+sourceledger-0.1.0.dist-info/WHEEL,sha256=cmC5s21ojypbVslldL7IJq3hjZH-tINy4rziKePFsG0,81
+sourceledger-0.1.0.dist-info/entry_points.txt,sha256=cAJLbTo4_3LGZQTGDPYy4yXWCmwXytV0YVojTnV-LUM,56
+sourceledger-0.1.0.dist-info/METADATA,sha256=5u8PMcUEtE2pUQsYBGrFrdSr_4HuEankCU4rSRnCdBo,8090
+sourceledger-0.1.0.dist-info/RECORD,,

sourceledger-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: uv 0.12.23
+Root-Is-Purelib: true
+Tag: py3-none-any

sourceledger-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,3 @@
+[console_scripts]
+sourceledger = sourceledger.cli:main
+

sourceledger-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Simon van Lierde
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.