tods-validate 0.1.0__py3-none-any.whl

tods_validate/__init__.py

@@ -0,0 +1,3 @@
+"""tods-validate: a validator for Transit Operational Data Standard (TODS) feeds."""
+
+__version__ = "0.1.0"

tods_validate/cli.py

@@ -0,0 +1,71 @@
+"""Command-line interface.
+
+Exit codes: 0 when no errors were found (warnings and infos do not fail the
+run unless --fail-on warning is set), 1 when errors were found, 2 when the
+package could not be read at all.
+"""
+
+from __future__ import annotations
+
+import sys
+
+import click
+
+from . import __version__
+from .findings import Severity
+from .loader import PackageNotFoundError
+from .report import RENDERERS, summarize
+from .runner import run
+from .schema import SPEC_VERSION
+
+
+@click.command(name="tods-validate")
+@click.version_option(__version__, message=f"%(prog)s %(version)s (TODS v{SPEC_VERSION})")
+@click.argument("path", type=click.Path(exists=False))
+@click.option(
+    "--gtfs",
+    "gtfs_path",
+    type=click.Path(exists=False),
+    default=None,
+    help=(
+        "Companion GTFS feed (directory or .zip) to resolve trip, stop, service, and "
+        "block references against. If omitted and GTFS files sit next to the TODS "
+        "files, those are used."
+    ),
+)
+@click.option(
+    "--format",
+    "output_format",
+    type=click.Choice(sorted(RENDERERS)),
+    default="text",
+    show_default=True,
+    help="Report format: human-readable text, JSON, or GitHub Actions annotations.",
+)
+@click.option(
+    "--fail-on",
+    type=click.Choice(["error", "warning"]),
+    default="error",
+    show_default=True,
+    help="Exit non-zero if findings at or above this severity exist.",
+)
+def main(path: str, gtfs_path: str | None, output_format: str, fail_on: str) -> None:
+    """Validate a TODS (Transit Operational Data Standard) feed at PATH.
+
+    PATH is a directory or .zip file containing the TODS .txt files, with or
+    without the GTFS feed alongside them.
+    """
+    try:
+        package, findings = run(path, gtfs_path)
+    except PackageNotFoundError as exc:
+        click.echo(f"tods-validate: error: {exc}", err=True)
+        sys.exit(2)
+
+    click.echo(RENDERERS[output_format](findings, package.source))
+
+    counts = summarize(findings)
+    failed = counts[Severity.ERROR] > 0 or (fail_on == "warning" and counts[Severity.WARNING] > 0)
+    sys.exit(1 if failed else 0)
+
+
+if __name__ == "__main__":
+    main()

tods_validate/findings.py

@@ -0,0 +1,56 @@
+"""Finding and severity types shared by rules and reporting."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from enum import IntEnum
+
+
+class Severity(IntEnum):
+    """Ordered so that max() picks the worst severity."""
+
+    INFO = 0
+    WARNING = 1
+    ERROR = 2
+
+    def __str__(self) -> str:
+        return self.name
+
+
+@dataclass(frozen=True)
+class Finding:
+    """One validation result.
+
+    Messages are written for transit schedulers, not programmers: say what is
+    wrong, where, and what good looks like.
+    """
+
+    rule_id: str
+    severity: Severity
+    message: str
+    file: str | None = None
+    # 1-based line number in the CSV file, counting the header as line 1.
+    row: int | None = None
+    field: str | None = None
+    suggestion: str | None = None
+
+    def location(self) -> str:
+        parts = []
+        if self.file:
+            parts.append(self.file)
+        if self.row is not None:
+            parts.append(f"row {self.row}")
+        if self.field:
+            parts.append(f"field {self.field!r}")
+        return ", ".join(parts)
+
+    def to_dict(self) -> dict[str, object]:
+        return {
+            "rule_id": self.rule_id,
+            "severity": str(self.severity),
+            "file": self.file,
+            "row": self.row,
+            "field": self.field,
+            "message": self.message,
+            "suggestion": self.suggestion,
+        }

tods_validate/gtfs_companion.py

@@ -0,0 +1,170 @@
+"""Load the companion GTFS feed and apply TODS supplement files.
+
+TODS IDs resolve against the GTFS feed *after* supplements are applied (the
+spec calls this "TODS-Supplemented GTFS"). Supplement evaluation follows the
+spec's "Supplement Files > Evaluation" section:
+
+1. PK matches and TODS_delete == "1": remove the GTFS row.
+2. PK matches otherwise: non-empty supplement values overwrite GTFS values.
+3. PK does not match: add the whole row.
+
+Only the slices of GTFS that TODS references are modeled here (trips, stops,
+calendars). This is not a GTFS validator; for that, use MobilityData's
+gtfs-validator on the supplemented feed.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from datetime import date, timedelta
+
+from .loader import FeedFile, Package
+from .schema import GTFS_PRIMARY_KEYS
+
+_WEEKDAY_FIELDS = ("monday", "tuesday", "wednesday", "thursday", "friday", "saturday", "sunday")
+
+
+def parse_gtfs_date(value: str) -> date | None:
+    """Parse a GTFS YYYYMMDD date, returning None if malformed."""
+    if len(value) != 8 or not value.isdigit():
+        return None
+    try:
+        return date(int(value[0:4]), int(value[4:6]), int(value[6:8]))
+    except ValueError:
+        return None
+
+
+def merge_supplement(
+    base: FeedFile | None,
+    supplement: FeedFile | None,
+    primary_key: tuple[str, ...],
+) -> dict[tuple[str, ...], dict[str, str]]:
+    """Compute effective rows keyed by primary key.
+
+    Rows whose primary-key fields are blank or missing are skipped here; the
+    field rules report those problems on the supplement file itself.
+    """
+    effective: dict[tuple[str, ...], dict[str, str]] = {}
+    if base is not None:
+        for row in base.rows:
+            key = tuple(row.values.get(f, "") for f in primary_key)
+            if any(v == "" for v in key):
+                continue
+            effective[key] = dict(row.values)
+    if supplement is not None:
+        for row in supplement.rows:
+            key = tuple(row.values.get(f, "") for f in primary_key)
+            if any(v == "" for v in key):
+                continue
+            if row.values.get("TODS_delete", "") == "1":
+                effective.pop(key, None)
+                continue
+            if key in effective:
+                target = effective[key]
+                for name, value in row.values.items():
+                    if name == "TODS_delete" or value == "":
+                        continue
+                    target[name] = value
+            else:
+                effective[key] = {k: v for k, v in row.values.items() if k != "TODS_delete"}
+    return effective
+
+
+@dataclass
+class CompanionGTFS:
+    """The supplemented GTFS slices that TODS references resolve against."""
+
+    source: str
+    # Which GTFS base files were actually present (affects what can be checked).
+    present: set[str] = field(default_factory=set)
+    trip_service: dict[str, str] = field(default_factory=dict)
+    trip_block: dict[str, str] = field(default_factory=dict)
+    stop_ids: set[str] = field(default_factory=set)
+    route_ids: set[str] = field(default_factory=set)
+    service_ids: set[str] = field(default_factory=set)
+    block_services: dict[str, set[str]] = field(default_factory=dict)
+    # Operating dates per service_id, from calendar + calendar_dates after
+    # supplements. Only populated for services whose calendar rows parse.
+    service_dates: dict[str, frozenset[date]] = field(default_factory=dict)
+    # Primary keys present in each GTFS base file *before* supplements, used
+    # to check that TODS_delete rows target something that exists.
+    base_keys: dict[str, set[tuple[str, ...]]] = field(default_factory=dict)
+
+    @property
+    def block_ids(self) -> set[str]:
+        return set(self.block_services)
+
+
+def _calendar_dates_for(
+    calendar: dict[tuple[str, ...], dict[str, str]],
+    calendar_dates: dict[tuple[str, ...], dict[str, str]],
+) -> dict[str, frozenset[date]]:
+    dates: dict[str, set[date]] = {}
+    for row in calendar.values():
+        service_id = row.get("service_id", "")
+        start = parse_gtfs_date(row.get("start_date", ""))
+        end = parse_gtfs_date(row.get("end_date", ""))
+        if not service_id or start is None or end is None or end < start:
+            continue
+        active = {i for i, name in enumerate(_WEEKDAY_FIELDS) if row.get(name, "") == "1"}
+        days = dates.setdefault(service_id, set())
+        current = start
+        while current <= end:
+            if current.weekday() in active:
+                days.add(current)
+            current += timedelta(days=1)
+    for row in calendar_dates.values():
+        service_id = row.get("service_id", "")
+        day = parse_gtfs_date(row.get("date", ""))
+        if not service_id or day is None:
+            continue
+        exception = row.get("exception_type", "")
+        if exception == "1":
+            dates.setdefault(service_id, set()).add(day)
+        elif exception == "2":
+            dates.setdefault(service_id, set()).discard(day)
+    return {k: frozenset(v) for k, v in dates.items()}
+
+
+def build_companion(gtfs: Package | None, tods: Package, source: str) -> CompanionGTFS:
+    """Build the supplemented GTFS view.
+
+    ``gtfs`` is the package holding the GTFS base files (may be the same
+    package as ``tods`` when the feed ships both together); supplements always
+    come from the TODS package.
+    """
+    companion = CompanionGTFS(source=source)
+
+    def effective(base_name: str) -> dict[tuple[str, ...], dict[str, str]]:
+        base = gtfs.get(base_name) if gtfs is not None else None
+        supplement = tods.get(base_name.removesuffix(".txt") + "_supplement.txt")
+        pk = GTFS_PRIMARY_KEYS[base_name]
+        if base is not None:
+            companion.present.add(base_name)
+            keys = companion.base_keys.setdefault(base_name, set())
+            for row in base.rows:
+                key = tuple(row.values.get(f, "") for f in pk)
+                if all(key):
+                    keys.add(key)
+        return merge_supplement(base, supplement, pk)
+
+    trips = effective("trips.txt")
+    for (trip_id,), row in trips.items():
+        companion.trip_service[trip_id] = row.get("service_id", "")
+        block_id = row.get("block_id", "")
+        companion.trip_block[trip_id] = block_id
+        if block_id:
+            companion.block_services.setdefault(block_id, set()).add(row.get("service_id", ""))
+
+    companion.stop_ids = {key[0] for key in effective("stops.txt")}
+    companion.route_ids = {key[0] for key in effective("routes.txt")}
+
+    calendar = effective("calendar.txt")
+    calendar_dates = effective("calendar_dates.txt")
+    companion.service_ids = {row.get("service_id", "") for row in calendar.values()} | {
+        row.get("service_id", "") for row in calendar_dates.values()
+    }
+    companion.service_ids.discard("")
+    companion.service_dates = _calendar_dates_for(calendar, calendar_dates)
+
+    return companion

tods_validate/loader.py

@@ -0,0 +1,183 @@
+"""Read a TODS package from a directory or .zip file.
+
+Loading is deliberately forgiving: structural problems (bad encoding, ragged
+rows, duplicate headers) are recorded as load problems for the structure rules
+to report, rather than raised, so one bad file does not hide findings in the
+rest of the package.
+"""
+
+from __future__ import annotations
+
+import csv
+import io
+import zipfile
+from dataclasses import dataclass, field
+from pathlib import Path
+
+
+class PackageNotFoundError(Exception):
+    """The path does not exist or is not a directory or .zip file."""
+
+
+@dataclass
+class Row:
+    """One data row of a CSV file.
+
+    ``line`` is the 1-based line number in the file, counting the header as
+    line 1, so the first data row is line 2. ``values`` maps header name to
+    the raw cell value ('' for cells missing from a short row).
+    """
+
+    line: int
+    values: dict[str, str]
+    # Cells beyond the header width, kept so rules can report them.
+    extra_cells: tuple[str, ...] = ()
+
+
+@dataclass
+class LoadProblem:
+    """A structural defect found while reading a file."""
+
+    code: str  # "encoding" | "empty" | "ragged" | "duplicate_header" | "csv_error"
+    message: str
+    line: int | None = None
+
+
+@dataclass
+class FeedFile:
+    name: str
+    headers: tuple[str, ...] = ()
+    rows: list[Row] = field(default_factory=list)
+    problems: list[LoadProblem] = field(default_factory=list)
+
+    def column(self, name: str) -> bool:
+        return name in self.headers
+
+
+@dataclass
+class Package:
+    """All files found in the package, parsed where possible."""
+
+    source: str
+    files: dict[str, FeedFile] = field(default_factory=dict)
+    # Names of entries that are present but were not parsed (non-CSV, nested
+    # directories inside a zip, etc.).
+    unparsed: list[str] = field(default_factory=list)
+
+    def get(self, name: str) -> FeedFile | None:
+        return self.files.get(name)
+
+
+def _parse_csv(name: str, data: bytes) -> FeedFile:
+    feed = FeedFile(name=name)
+    try:
+        # utf-8-sig transparently strips a BOM if present.
+        text = data.decode("utf-8-sig")
+    except UnicodeDecodeError as exc:
+        feed.problems.append(
+            LoadProblem(
+                code="encoding",
+                message=(
+                    f"{name} is not valid UTF-8 (byte {exc.start}). TODS files must be "
+                    "UTF-8 encoded, like GTFS. Re-export the file as UTF-8."
+                ),
+            )
+        )
+        return feed
+
+    if text.strip() == "":
+        feed.problems.append(LoadProblem(code="empty", message=f"{name} is empty (no header row)."))
+        return feed
+
+    try:
+        reader = csv.reader(io.StringIO(text))
+        raw_rows = list(reader)
+    except csv.Error as exc:
+        feed.problems.append(
+            LoadProblem(code="csv_error", message=f"{name} could not be parsed as CSV: {exc}.")
+        )
+        return feed
+
+    header = [h.strip() for h in raw_rows[0]]
+    feed.headers = tuple(header)
+
+    seen: set[str] = set()
+    for h in header:
+        if h in seen:
+            feed.problems.append(
+                LoadProblem(
+                    code="duplicate_header",
+                    message=(
+                        f"{name} declares the column {h!r} more than once. Each column "
+                        "may appear only once; values in the duplicate column are ignored."
+                    ),
+                    line=1,
+                )
+            )
+        seen.add(h)
+
+    width = len(header)
+    for i, raw in enumerate(raw_rows[1:], start=2):
+        if raw == [] or all(cell == "" for cell in raw):
+            continue  # skip blank lines
+        if len(raw) != width:
+            feed.problems.append(
+                LoadProblem(
+                    code="ragged",
+                    message=(
+                        f"{name} row {i} has {len(raw)} values but the header declares "
+                        f"{width} columns. Every row must have one value per column "
+                        "(use empty values for blanks)."
+                    ),
+                    line=i,
+                )
+            )
+        values = {h: (raw[j] if j < len(raw) else "") for j, h in enumerate(header)}
+        extra = tuple(raw[width:])
+        feed.rows.append(Row(line=i, values=values, extra_cells=extra))
+
+    return feed
+
+
+def load_package(path: str | Path) -> Package:
+    """Load all top-level files from a directory or .zip file."""
+    p = Path(path)
+    if p.is_dir():
+        pkg = Package(source=str(p))
+        for entry in sorted(p.iterdir()):
+            if entry.name.startswith("."):
+                continue
+            if entry.is_dir():
+                pkg.unparsed.append(entry.name + "/")
+                continue
+            if entry.suffix.lower() in (".txt", ".csv"):
+                pkg.files[entry.name] = _parse_csv(entry.name, entry.read_bytes())
+            else:
+                pkg.unparsed.append(entry.name)
+        return pkg
+
+    if p.is_file() and zipfile.is_zipfile(p):
+        pkg = Package(source=str(p))
+        with zipfile.ZipFile(p) as zf:
+            for info in sorted(zf.infolist(), key=lambda i: i.filename):
+                if info.is_dir():
+                    continue
+                name = info.filename
+                base = Path(name).name
+                if "/" in name.strip("/"):
+                    # Files nested in subdirectories are outside the spec's
+                    # flat-package shape; surface them rather than guessing.
+                    pkg.unparsed.append(name)
+                    continue
+                if base.startswith("."):
+                    continue
+                if Path(base).suffix.lower() in (".txt", ".csv"):
+                    pkg.files[base] = _parse_csv(base, zf.read(info))
+                else:
+                    pkg.unparsed.append(name)
+        return pkg
+
+    raise PackageNotFoundError(
+        f"{p} is not a directory or a .zip file. Pass the folder or zip that "
+        "contains the TODS .txt files."
+    )

tods_validate/report.py

@@ -0,0 +1,107 @@
+"""Render findings as human text, JSON, or GitHub Actions annotations.
+
+Accessibility note: severity is always carried by a word (ERROR, WARNING,
+INFO), never by color alone, so reports stay readable when piped to a file.
+"""
+
+from __future__ import annotations
+
+import json
+from collections import Counter
+
+from .findings import Finding, Severity
+from .schema import SPEC_VERSION
+
+
+def summarize(findings: list[Finding]) -> Counter[Severity]:
+    return Counter(f.severity for f in findings)
+
+
+def render_text(findings: list[Finding], source: str) -> str:
+    lines: list[str] = [f"tods-validate: {source} (TODS v{SPEC_VERSION})", ""]
+    if not findings:
+        lines.append("No problems found.")
+        return "\n".join(lines)
+
+    for severity in (Severity.ERROR, Severity.WARNING, Severity.INFO):
+        group = [f for f in findings if f.severity == severity]
+        if not group:
+            continue
+        plural = "s" if len(group) != 1 else ""
+        lines.append(f"{len(group)} {severity.name.lower()}{plural}:")
+        for f in group:
+            location = f.location()
+            prefix = f"  {severity.name} {f.rule_id}"
+            lines.append(f"{prefix} [{location}]" if location else prefix)
+            lines.append(f"    {f.message}")
+            if f.suggestion:
+                lines.append(f"    Fix: {f.suggestion}")
+        lines.append("")
+
+    counts = summarize(findings)
+    lines.append(
+        "Summary: "
+        f"{counts[Severity.ERROR]} error(s), "
+        f"{counts[Severity.WARNING]} warning(s), "
+        f"{counts[Severity.INFO]} info."
+    )
+    return "\n".join(lines)
+
+
+def render_json(findings: list[Finding], source: str) -> str:
+    counts = summarize(findings)
+    payload = {
+        "validator": "tods-validate",
+        "specVersion": SPEC_VERSION,
+        "source": source,
+        "summary": {
+            "errors": counts[Severity.ERROR],
+            "warnings": counts[Severity.WARNING],
+            "infos": counts[Severity.INFO],
+        },
+        "findings": [f.to_dict() for f in findings],
+    }
+    return json.dumps(payload, indent=2)
+
+
+_GITHUB_COMMANDS = {
+    Severity.ERROR: "error",
+    Severity.WARNING: "warning",
+    Severity.INFO: "notice",
+}
+
+
+def _escape_annotation(text: str) -> str:
+    return text.replace("%", "%25").replace("\r", "%0D").replace("\n", "%0A")
+
+
+def render_github(findings: list[Finding], source: str) -> str:
+    """Workflow command format; one annotation per finding.
+
+    See https://docs.github.com/actions/reference/workflow-commands-for-github-actions
+    """
+    lines = []
+    for f in findings:
+        command = _GITHUB_COMMANDS[f.severity]
+        properties = []
+        if f.file:
+            properties.append(f"file={_escape_annotation(f.file)}")
+        if f.row is not None:
+            properties.append(f"line={f.row}")
+        properties.append(f"title={f.rule_id}")
+        message = f.message if not f.suggestion else f"{f.message} Fix: {f.suggestion}"
+        lines.append(f"::{command} {','.join(properties)}::{_escape_annotation(message)}")
+    counts = summarize(findings)
+    lines.append(
+        f"tods-validate: {counts[Severity.ERROR]} error(s), "
+        f"{counts[Severity.WARNING]} warning(s), {counts[Severity.INFO]} info "
+        f"in {source}."
+    )
+    return "\n".join(lines)
+
+
+RENDERERS = {
+    "text": render_text,
+    "json": render_json,
+    "github": render_github,
+}