gtfslint-cli 0.1.0__py3-none-any.whl

gtfslint/notices.py

@@ -0,0 +1,682 @@
+"""Rule catalogue, severities and the :class:`Notice` record.
+
+Every check gtfslint performs is a *rule* with a stable code (``GL101``), a
+snake_case name, a default severity, a one-line description and a link to the
+part of the GTFS Schedule reference it enforces. Where a rule mirrors a notice
+of the canonical MobilityData validator, the MobilityData notice name is kept
+in :attr:`Rule.mobilitydata` (it is usually identical to :attr:`Rule.name`).
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from dataclasses import field as _dc_field
+from enum import IntEnum
+from typing import Any, Dict, Iterable, List, Optional, Tuple
+
+__all__ = [
+    "RULES",
+    "Notice",
+    "Rule",
+    "Severity",
+    "resolve_selection",
+]
+
+_REF = "https://gtfs.org/documentation/schedule/reference/"
+
+
+class Severity(IntEnum):
+    """Notice severity. Higher values are more severe."""
+
+    INFO = 1
+    WARNING = 2
+    ERROR = 3
+
+    @property
+    def label(self) -> str:
+        """Lower-case name used in every output format (``"error"``)."""
+        return self.name.lower()
+
+    @classmethod
+    def parse(cls, value: str) -> Severity:
+        """Parse ``"error"``, ``"warning"`` or ``"info"`` (case-insensitive)."""
+        try:
+            return cls[value.strip().upper()]
+        except KeyError:
+            raise ValueError(
+                f"unknown severity {value!r}; expected error, warning or info"
+            ) from None
+
+
+@dataclass(frozen=True)
+class Rule:
+    """Static description of one validation rule."""
+
+    code: str
+    name: str
+    severity: Severity
+    description: str
+    reference: str
+    mobilitydata: Optional[str] = None
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "code": self.code,
+            "name": self.name,
+            "severity": self.severity.label,
+            "description": self.description,
+            "reference": self.reference,
+            "mobilitydata": self.mobilitydata,
+        }
+
+
+_E, _W, _I = Severity.ERROR, Severity.WARNING, Severity.INFO
+
+# (code, name, severity, description, reference anchor, mirrors MobilityData notice?)
+_CATALOGUE: Tuple[Tuple[str, str, Severity, str, str, bool], ...] = (
+    # --- GL0xx: files, columns and rows -------------------------------------------------
+    (
+        "GL001",
+        "missing_required_file",
+        _E,
+        "A file required by the specification (agency, routes, trips, stop_times, stops) is "
+        "missing.",
+        "#dataset-files",
+        True,
+    ),
+    (
+        "GL002",
+        "missing_calendar_and_calendar_date_files",
+        _E,
+        "Neither calendar.txt nor calendar_dates.txt is present; at least one is required.",
+        "#dataset-files",
+        True,
+    ),
+    (
+        "GL003",
+        "empty_file",
+        _E,
+        "A required file is empty or has a header but no records.",
+        "#file-requirements",
+        True,
+    ),
+    (
+        "GL004",
+        "missing_required_column",
+        _E,
+        "A required column is absent from the header of a file.",
+        "#file-requirements",
+        True,
+    ),
+    (
+        "GL005",
+        "duplicated_column",
+        _E,
+        "The same column name appears more than once in a header.",
+        "#file-requirements",
+        True,
+    ),
+    (
+        "GL006",
+        "empty_column_name",
+        _E,
+        "A header contains an empty column name.",
+        "#file-requirements",
+        True,
+    ),
+    (
+        "GL007",
+        "invalid_row_length",
+        _E,
+        "A row has a different number of fields than the header.",
+        "#file-requirements",
+        True,
+    ),
+    (
+        "GL008",
+        "csv_parsing_failed",
+        _E,
+        "The file could not be parsed as UTF-8 CSV; the rest of the file was skipped.",
+        "#file-requirements",
+        True,
+    ),
+    (
+        "GL009",
+        "unknown_file",
+        _I,
+        "The feed contains a file that is not part of the GTFS Schedule specification.",
+        "#dataset-files",
+        True,
+    ),
+    (
+        "GL010",
+        "unknown_column",
+        _I,
+        "A file has a column that is not defined by the specification.",
+        "#file-requirements",
+        True,
+    ),
+    (
+        "GL011",
+        "leading_or_trailing_whitespaces",
+        _W,
+        "A value or column name has leading or trailing whitespace (it is trimmed before the "
+        "other checks).",
+        "#file-requirements",
+        True,
+    ),
+    (
+        "GL012",
+        "duplicate_key",
+        _E,
+        "Two records share the same primary key (e.g. stop_id, or trip_id + stop_sequence).",
+        "#dataset-attributes",
+        True,
+    ),
+    (
+        "GL013",
+        "missing_required_field",
+        _E,
+        "A required or conditionally required field is empty.",
+        "#field-definitions",
+        True,
+    ),
+    (
+        "GL014",
+        "route_both_short_and_long_name_missing",
+        _E,
+        "A route has neither route_short_name nor route_long_name.",
+        "#routestxt",
+        True,
+    ),
+    ("GL015", "empty_row", _W, "A row contains only empty fields.", "#file-requirements", True),
+    (
+        "GL016",
+        "missing_recommended_file",
+        _W,
+        "feed_info.txt is missing; it is recommended so consumers know the publisher and "
+        "validity window.",
+        "#feed_infotxt",
+        True,
+    ),
+    # --- GL1xx: field values -----------------------------------------------------------
+    (
+        "GL101",
+        "invalid_color",
+        _E,
+        "A color is not a six-digit hexadecimal value such as FFFFFF.",
+        "#field-types",
+        True,
+    ),
+    (
+        "GL102",
+        "invalid_url",
+        _E,
+        "A URL is not a fully qualified http:// or https:// URL.",
+        "#field-types",
+        True,
+    ),
+    (
+        "GL103",
+        "invalid_timezone",
+        _E,
+        "A timezone is not a valid IANA TZ database name such as Europe/Istanbul.",
+        "#field-types",
+        True,
+    ),
+    (
+        "GL104",
+        "invalid_date",
+        _E,
+        "A date is not a valid calendar date in YYYYMMDD format.",
+        "#field-types",
+        True,
+    ),
+    (
+        "GL105",
+        "invalid_time",
+        _E,
+        "A time is not in H:MM:SS or HH:MM:SS format (hours above 24 are allowed).",
+        "#field-types",
+        True,
+    ),
+    (
+        "GL106",
+        "invalid_integer",
+        _E,
+        "A value that must be an integer (or enumeration) could not be parsed.",
+        "#field-types",
+        True,
+    ),
+    (
+        "GL107",
+        "invalid_float",
+        _E,
+        "A value that must be a number could not be parsed.",
+        "#field-types",
+        True,
+    ),
+    (
+        "GL108",
+        "unexpected_enum_value",
+        _W,
+        "An enumeration field has a value not listed by the specification.",
+        "#field-types",
+        True,
+    ),
+    (
+        "GL109",
+        "number_out_of_range",
+        _E,
+        "A number is outside its allowed range (latitude, longitude, non-negative or positive "
+        "fields).",
+        "#field-types",
+        True,
+    ),
+    ("GL110", "invalid_email", _E, "An e-mail address is malformed.", "#field-types", True),
+    (
+        "GL111",
+        "invalid_language_code",
+        _E,
+        "A language code is not a well-formed IETF BCP 47 tag.",
+        "#field-types",
+        True,
+    ),
+    (
+        "GL112",
+        "invalid_currency",
+        _E,
+        "A currency code is not an ISO 4217 alphabetical code.",
+        "#field-types",
+        True,
+    ),
+    (
+        "GL113",
+        "point_near_origin",
+        _E,
+        "A stop or shape point lies at (0, 0) ('null island'), usually a missing coordinate.",
+        "#stopstxt",
+        True,
+    ),
+    (
+        "GL114",
+        "inconsistent_agency_timezone",
+        _E,
+        "Agencies in agency.txt use different agency_timezone values.",
+        "#agencytxt",
+        True,
+    ),
+    (
+        "GL115",
+        "route_color_contrast",
+        _W,
+        "route_color and route_text_color have a WCAG contrast ratio below 3:1.",
+        "#routestxt",
+        True,
+    ),
+    (
+        "GL116",
+        "non_ascii_or_non_printable_char",
+        _W,
+        "An ID contains non-ASCII or non-printable characters.",
+        "#field-types",
+        True,
+    ),
+    # --- GL2xx: references -------------------------------------------------------------
+    (
+        "GL201",
+        "foreign_key_violation",
+        _E,
+        "A reference points to a record that does not exist (e.g. trips.route_id, "
+        "stop_times.stop_id, stops.parent_station, transfers, fare tables).",
+        "#field-definitions",
+        True,
+    ),
+    (
+        "GL202",
+        "wrong_parent_location_type",
+        _E,
+        "parent_station references a location of the wrong type (a stop's parent must be a "
+        "station, a boarding area's parent must be a stop).",
+        "#stopstxt",
+        True,
+    ),
+    (
+        "GL203",
+        "station_with_parent_station",
+        _E,
+        "A station (location_type 1) has a parent_station.",
+        "#stopstxt",
+        True,
+    ),
+    (
+        "GL204",
+        "location_with_unexpected_stop_time",
+        _E,
+        "stop_times references a station, entrance or generic node instead of a stop or "
+        "boarding area.",
+        "#stop_timestxt",
+        True,
+    ),
+    # --- GL3xx: calendars --------------------------------------------------------------
+    (
+        "GL301",
+        "start_and_end_range_out_of_order",
+        _E,
+        "A start date/time is after its end (calendar, feed_info, frequencies).",
+        "#calendartxt",
+        True,
+    ),
+    (
+        "GL302",
+        "service_never_active",
+        _W,
+        "A service_id is never active once calendar and calendar_dates are combined.",
+        "#calendartxt",
+        False,
+    ),
+    (
+        "GL303",
+        "redundant_calendar_date",
+        _I,
+        "A calendar_dates exception adds a date that is already active or removes one that is not.",
+        "#calendar_datestxt",
+        False,
+    ),
+    (
+        "GL304",
+        "expired_calendar",
+        _W,
+        "A service has no active dates on or after the reference date.",
+        "#calendartxt",
+        True,
+    ),
+    # --- GL4xx: stop_times and trips ---------------------------------------------------
+    (
+        "GL401",
+        "stop_times_not_sorted",
+        _I,
+        "The rows of a trip are not listed in increasing stop_sequence order (allowed, but "
+        "slows down consumers).",
+        "#stop_timestxt",
+        False,
+    ),
+    (
+        "GL402",
+        "stop_time_with_arrival_before_previous_departure_time",
+        _E,
+        "A stop's arrival_time is earlier than the departure_time at the previous stop.",
+        "#stop_timestxt",
+        True,
+    ),
+    (
+        "GL403",
+        "stop_time_arrival_after_departure",
+        _E,
+        "A stop's arrival_time is later than its own departure_time.",
+        "#stop_timestxt",
+        False,
+    ),
+    (
+        "GL404",
+        "stop_time_with_only_arrival_or_departure_time",
+        _E,
+        "Only one of arrival_time and departure_time is set.",
+        "#stop_timestxt",
+        True,
+    ),
+    (
+        "GL405",
+        "missing_trip_edge",
+        _E,
+        "The first or last stop of a trip has no arrival_time/departure_time.",
+        "#stop_timestxt",
+        True,
+    ),
+    (
+        "GL406",
+        "decreasing_or_equal_stop_time_distance",
+        _E,
+        "shape_dist_traveled does not increase along a trip.",
+        "#stop_timestxt",
+        True,
+    ),
+    (
+        "GL407",
+        "stop_time_timepoint_without_times",
+        _E,
+        "timepoint=1 but the stop has no arrival/departure time.",
+        "#stop_timestxt",
+        True,
+    ),
+    (
+        "GL408",
+        "fast_travel_between_consecutive_stops",
+        _W,
+        "The speed between two timed stops exceeds the plausible maximum for the route type.",
+        "#stop_timestxt",
+        True,
+    ),
+    ("GL409", "unusable_trip", _W, "A trip visits fewer than two stops.", "#stop_timestxt", True),
+    # --- GL5xx: geometry ---------------------------------------------------------------
+    (
+        "GL501",
+        "decreasing_shape_distance",
+        _E,
+        "shape_dist_traveled decreases along a shape.",
+        "#shapestxt",
+        True,
+    ),
+    (
+        "GL502",
+        "stops_too_close",
+        _W,
+        "Two different stops (not in the same station) are closer than min-stop-distance-m; "
+        "often an accidental duplicate.",
+        "#stopstxt",
+        False,
+    ),
+    (
+        "GL503",
+        "stops_too_far",
+        _W,
+        "Two consecutive stops of a trip are farther apart than plausible for the route type; "
+        "often a wrong coordinate.",
+        "#stopstxt",
+        False,
+    ),
+    (
+        "GL504",
+        "stop_too_far_from_parent_station",
+        _W,
+        "A stop, entrance or node is farther than max-parent-distance-m from its parent station.",
+        "#stopstxt",
+        False,
+    ),
+    # --- GL6xx: unused entities --------------------------------------------------------
+    (
+        "GL601",
+        "stop_without_stop_time",
+        _W,
+        "A stop (location_type 0) is never served by stop_times.",
+        "#stopstxt",
+        True,
+    ),
+    ("GL602", "unused_station", _I, "A station has no child stops.", "#stopstxt", True),
+    ("GL603", "route_without_trips", _W, "A route has no trips.", "#routestxt", False),
+    ("GL604", "unused_shape", _W, "A shape is not referenced by any trip.", "#shapestxt", True),
+    ("GL605", "unused_trip", _W, "A trip has no stop_times.", "#tripstxt", True),
+    (
+        "GL606",
+        "unused_service",
+        _I,
+        "A service_id from calendar/calendar_dates is not used by any trip.",
+        "#calendartxt",
+        False,
+    ),
+    # --- GL7xx: blocks and feed_info ---------------------------------------------------
+    (
+        "GL701",
+        "block_trips_with_overlapping_stop_times",
+        _E,
+        "Two trips with the same block_id run at overlapping times on a shared service date.",
+        "#tripstxt",
+        True,
+    ),
+    (
+        "GL702",
+        "missing_feed_info_date",
+        _W,
+        "feed_info.txt has no feed_start_date or no feed_end_date.",
+        "#feed_infotxt",
+        True,
+    ),
+    (
+        "GL703",
+        "feed_expiration_date7_days",
+        _W,
+        "feed_end_date is less than 7 days after the reference date (or already passed).",
+        "#feed_infotxt",
+        True,
+    ),
+    (
+        "GL704",
+        "feed_expiration_date30_days",
+        _W,
+        "feed_end_date is between 7 and 30 days after the reference date.",
+        "#feed_infotxt",
+        True,
+    ),
+)
+
+RULES: Dict[str, Rule] = {
+    code: Rule(code, name, sev, desc, _REF + anchor, name if md else None)
+    for code, name, sev, desc, anchor, md in _CATALOGUE
+}
+"""All rules, keyed by code, in catalogue order."""
+
+_BY_NAME: Dict[str, str] = {rule.name: code for code, rule in RULES.items()}
+
+
+def _matches(code: str, selector: str) -> bool:
+    sel = selector.strip()
+    if not sel:
+        return False
+    if sel.upper() == "ALL":
+        return True
+    if _BY_NAME.get(sel) == code:
+        return True
+    return code.startswith(sel.upper())
+
+
+def resolve_selection(
+    select: Optional[Iterable[str]] = None, ignore: Optional[Iterable[str]] = None
+) -> frozenset:
+    """Return the set of enabled rule codes.
+
+    ``select`` and ``ignore`` accept full codes (``GL101``), code prefixes (``GL1``,
+    ``GL``), rule names (``invalid_color``) or ``ALL``. ``select=None`` enables every
+    rule; ``ignore`` is applied afterwards. Unknown selectors raise :class:`ValueError`.
+    """
+    sel = list(select) if select is not None else ["ALL"]
+    ign = list(ignore) if ignore is not None else []
+    for item in sel + ign:
+        if item.strip() and not any(_matches(code, item) for code in RULES):
+            raise ValueError(f"unknown rule selector {item!r}")
+    enabled = {c for c in RULES if any(_matches(c, s) for s in sel)}
+    enabled -= {c for c in RULES if any(_matches(c, s) for s in ign)}
+    return frozenset(enabled)
+
+
+@dataclass
+class Notice:
+    """One finding: a rule violated at a location in the feed.
+
+    ``row`` is the 1-based physical line number in the file (the header is line 1).
+    """
+
+    code: str
+    severity: Severity
+    message: str
+    file: Optional[str] = None
+    row: Optional[int] = None
+    field: Optional[str] = None
+    value: Optional[str] = None
+    context: Dict[str, str] = _dc_field(default_factory=dict)
+
+    @property
+    def name(self) -> str:
+        return RULES[self.code].name
+
+    @property
+    def rule(self) -> Rule:
+        return RULES[self.code]
+
+    @property
+    def location(self) -> str:
+        """Human-readable location such as ``stops.txt:12 [stop_lat]``."""
+        out = self.file or "<feed>"
+        if self.row is not None:
+            out += f":{self.row}"
+        if self.field:
+            out += f" [{self.field}]"
+        return out
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "code": self.code,
+            "name": self.name,
+            "severity": self.severity.label,
+            "message": self.message,
+            "file": self.file,
+            "row": self.row,
+            "field": self.field,
+            "value": self.value,
+            "context": dict(self.context),
+        }
+
+
+class Collector:
+    """Stores notices, honouring the rule selection and a per-rule cap.
+
+    Counts always include every occurrence; only the first ``max_per_rule`` notices of
+    each rule are kept in memory.
+    """
+
+    def __init__(self, enabled: frozenset, max_per_rule: int = 1000) -> None:
+        self.enabled = enabled
+        self.max_per_rule = max_per_rule
+        self.notices: List[Notice] = []
+        self.counts: Dict[str, int] = {}
+
+    def emit(
+        self,
+        code: str,
+        message: str,
+        file: Optional[str] = None,
+        row: Optional[int] = None,
+        field: Optional[str] = None,
+        value: Optional[str] = None,
+        **context: str,
+    ) -> None:
+        if code not in self.enabled:
+            return
+        n = self.counts.get(code, 0) + 1
+        self.counts[code] = n
+        if self.max_per_rule <= 0 or n <= self.max_per_rule:
+            self.notices.append(
+                Notice(code, RULES[code].severity, message, file, row, field, value, context)
+            )
+
+    def discard(self, codes: Iterable[str], key: str, values: Iterable[str]) -> None:
+        """Drop stored notices of ``codes`` whose ``context[key]`` is in ``values``."""
+        codes_set, vals = set(codes), set(values)
+        kept: List[Notice] = []
+        for n in self.notices:
+            if n.code in codes_set and n.context.get(key) in vals:
+                self.counts[n.code] -= 1
+                if not self.counts[n.code]:
+                    del self.counts[n.code]
+            else:
+                kept.append(n)
+        self.notices = kept