civic-data 0.1.0__py3-none-any.whl

civic_data/__init__.py

@@ -0,0 +1,14 @@
+"""civic-data — the extraction / normalization / documentation spine.
+
+One canonical home for the data-layer core: source adapters, medallion ingest
+(bronze → silver), and a generated data dictionary with a transformation-lineage
+layer. The product's value is not the data (the city gives that away raw) — it is
+the *documented, reproducible transformation lineage*: what we did, why, and what
+is lost. That lineage is generated from the live DB + code, never hand-maintained,
+so it cannot drift from reality.
+
+civic-graph (the private flagship) was the origin of this code and is now a
+downstream consumer / worked example built on top.
+"""
+
+__version__ = "0.1.0"

civic_data/adapters/__init__.py

@@ -0,0 +1,7 @@
+"""Source adapters — the extraction spine.
+
+`arcgis` is the working portal adapter (Louisville / LOJIC, and any ArcGIS Hub
+portal). `socrata` is a dormant placeholder that activates at city #2
+(Cincinnati / Chicago). `validation` is the ZIP/geography gatekeeper shared by
+every ingest path. `base` is the adapter contract.
+"""

civic_data/adapters/arcgis.py

@@ -0,0 +1,128 @@
+"""ArcGIS Feature Service adapter.
+
+Louisville's open data portal runs on ArcGIS Hub. Most datasets are
+exposed as ArcGIS Feature Services with a REST query endpoint.
+
+Feature Service query pattern:
+  GET {service_url}/query?where=1=1&outFields=*&f=json&resultOffset=0&resultRecordCount=2000
+
+Pagination: resultOffset increments by resultRecordCount until
+exceededTransferLimit is false.
+"""
+
+import time
+import httpx
+from typing import Any
+
+
+def feature_count(service_url: str, timeout: float = 30.0) -> int | None:
+    """Source row count via ArcGIS returnCountOnly (one tiny call). None on error.
+
+    Captured at ingest as source-pull metadata (the reproducibility substrate): the
+    source_rowcount at pull time, to compare against what we actually fetched/kept.
+    """
+    try:
+        resp = httpx.get(f"{service_url}/query",
+                         params={"where": "1=1", "returnCountOnly": "true", "f": "json"},
+                         timeout=timeout)
+        resp.raise_for_status()
+        return resp.json().get("count")
+    except Exception:
+        return None
+
+
+def _query_page(client, url, params, max_retries: int = 4):
+    """Fetch one page, retrying transient failures. Returns parsed JSON with a 'features' key.
+
+    Raises RuntimeError if the server never returns a valid page — so a transient error
+    (HTTP 5xx, an ArcGIS error payload, a truncated body) can NEVER be mistaken for
+    end-of-data and silently truncate a large pull. Only a *valid* response with an empty
+    'features' array or exceededTransferLimit=False ends pagination (handled by the caller).
+    """
+    last_err = None
+    for attempt in range(max_retries):
+        try:
+            resp = client.get(url, params=params)
+            resp.raise_for_status()
+            data = resp.json()
+            if "error" in data:
+                last_err = f"ArcGIS error payload: {data['error']}"
+            elif "features" not in data:
+                last_err = f"response missing 'features' (keys={list(data)[:6]})"
+            else:
+                return data
+        except (httpx.HTTPError, ValueError) as e:  # ValueError also covers JSON decode errors
+            last_err = repr(e)
+        time.sleep(1.0 * (attempt + 1))  # linear backoff between retries
+    raise RuntimeError(
+        f"ArcGIS page failed after {max_retries} attempts "
+        f"(resultOffset={params.get('resultOffset')}): {last_err}")
+
+
+def fetch_feature_service(
+    service_url: str,
+    where: str = "1=1",
+    out_fields: str = "*",
+    page_size: int = 2000,
+    max_records: int | None = None,
+    delay: float = 0.5,
+) -> list[dict[str, Any]]:
+    """Fetch all records from an ArcGIS Feature Service.
+
+    Args:
+        service_url: The Feature Service layer URL
+            (e.g., https://services1.arcgis.com/.../FeatureServer/0)
+        where: SQL WHERE clause for filtering
+        out_fields: Comma-separated field names or "*" for all
+        page_size: Records per request (max varies by service, 2000 is safe)
+        max_records: Optional cap on total records fetched
+        delay: Seconds between paginated requests (polite scraping)
+
+    Returns:
+        List of feature attribute dicts (geometry stripped to lat/lng if present)
+
+    Pagination terminates only on a *valid* empty page or exceededTransferLimit=False;
+    transient per-page failures are retried (see _query_page), never treated as done.
+    """
+    all_records = []
+    offset = 0
+    url = f"{service_url}/query"
+
+    with httpx.Client(timeout=60.0) as client:
+        while True:
+            params = {
+                "where": where,
+                "outFields": out_fields,
+                "f": "json",
+                "resultOffset": offset,
+                "resultRecordCount": page_size,
+                "outSR": 4326,  # WGS84
+            }
+
+            data = _query_page(client, url, params)
+            features = data.get("features", [])
+            if not features:
+                break
+
+            for f in features:
+                record = f.get("attributes", {})
+                # Extract geometry as flat lat/lng if present
+                geom = f.get("geometry")
+                if geom:
+                    record["_longitude"] = geom.get("x")
+                    record["_latitude"] = geom.get("y")
+                all_records.append(record)
+
+            if max_records and len(all_records) >= max_records:
+                all_records = all_records[:max_records]
+                break
+
+            if not data.get("exceededTransferLimit", False):
+                break
+
+            # Increment by actual records returned, not page_size.
+            # Some services cap below our page_size (e.g., 1000 max).
+            offset += len(features)
+            time.sleep(delay)
+
+    return all_records

civic_data/adapters/base.py

@@ -0,0 +1,71 @@
+"""Base adapter interface for civic data sources."""
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from typing import Any
+
+
+@dataclass
+class IngestResult:
+    """Result of a data ingestion run."""
+    adapter: str
+    source_dataset: str
+    record_count: int
+    status: str  # "ok" or "error"
+    error_message: str | None = None
+
+
+class CivicDataAdapter(ABC):
+    """Abstract base class for all civic data adapters.
+
+    Each adapter handles one data domain (crime, 311, property, etc.)
+    and knows how to:
+    1. Fetch raw data from the source API
+    2. Store raw records in bronze layer (JSONB, append-only)
+    3. Transform raw records into silver layer (typed, geocoded)
+    """
+
+    @property
+    @abstractmethod
+    def adapter_name(self) -> str:
+        """Unique identifier for this adapter (e.g., 'crime', 'service_requests')."""
+        ...
+
+    @abstractmethod
+    def fetch_raw(self, dataset_id: str, **kwargs) -> list[dict[str, Any]]:
+        """Fetch raw records from the source API.
+
+        Returns a list of raw JSON-serializable dicts.
+        """
+        ...
+
+    @abstractmethod
+    def load_bronze(self, records: list[dict[str, Any]], dataset_id: str) -> int:
+        """Store raw records in the bronze layer. Returns count of rows inserted."""
+        ...
+
+    @abstractmethod
+    def promote_to_silver(self, dataset_id: str) -> int:
+        """Transform bronze records into silver layer. Returns count of rows promoted."""
+        ...
+
+    def ingest(self, dataset_id: str, **kwargs) -> IngestResult:
+        """Full ingestion pipeline: fetch → bronze → silver."""
+        try:
+            raw = self.fetch_raw(dataset_id, **kwargs)
+            bronze_count = self.load_bronze(raw, dataset_id)
+            silver_count = self.promote_to_silver(dataset_id)
+            return IngestResult(
+                adapter=self.adapter_name,
+                source_dataset=dataset_id,
+                record_count=silver_count,
+                status="ok",
+            )
+        except Exception as e:
+            return IngestResult(
+                adapter=self.adapter_name,
+                source_dataset=dataset_id,
+                record_count=0,
+                status="error",
+                error_message=str(e),
+            )

civic_data/adapters/socrata.py

@@ -0,0 +1,28 @@
+"""Socrata Open Data adapter — DORMANT (activates at city #2).
+
+STATUS: placeholder, not implemented. v1 is Louisville-only (crime + 311) on
+ArcGIS Hub; see `adapters/arcgis.py` for the working pattern. Socrata is the
+multi-city unlock — Cincinnati, Chicago, and ~100 other US cities publish on the
+Socrata platform (SODA API). When city #2 lands, implement `fetch_dataset`
+against the SODA endpoint, mirroring the shape `arcgis.fetch_feature_service`
+returns (a list of flat attribute dicts) so the downstream ingest/validation
+spine is unchanged.
+
+Design note (from the kickoff): this file was listed in the seed manifest as
+inherited from civic-graph, but civic-graph never actually authored a Socrata
+adapter — only the ArcGIS one exists there. So this is a fresh, honest stub, not
+a lift. Multi-city is the thesis, not the v1 scope; do not build it out until a
+real second city is in scope (guardrail: no future-proofing).
+
+SODA query pattern (for reference when implemented):
+  GET https://{domain}/resource/{dataset_id}.json?$limit=50000&$offset=0
+  Pagination via $limit/$offset; optional $$app_token header to raise rate limits.
+"""
+
+
+def fetch_dataset(*args, **kwargs):
+    raise NotImplementedError(
+        "The Socrata adapter is dormant. v1 is Louisville-only (ArcGIS). "
+        "Implement this against the SODA API when city #2 (Cincinnati/Chicago) is in scope, "
+        "returning the same flat-dict shape as adapters.arcgis.fetch_feature_service."
+    )

civic_data/adapters/validation.py

@@ -0,0 +1,135 @@
+"""ZIP code validation for Louisville data quality.
+
+Validates ZIP codes against silver.zip_codes (41 Jefferson County ZIPs).
+Records with non-Louisville ZIPs are quarantined, not dropped.
+
+Return value contract for validate_zip():
+  - valid ZIP string  → insert with this normalized 5-digit ZIP
+  - None              → unknown/missing ZIP, insert with SQL NULL
+  - QUARANTINED       → record was quarantined, caller should skip
+
+The canonical ZIP set (silver.zip_codes) is this project's master/reference data
+(MDM). Every event table validates against it; it is the one canonical geography.
+"""
+
+import json
+import re
+
+# Louisville / Jefferson County has 41 unique ZIP code boundaries
+# sourced from Louisville's ArcGIS portal (government data, not derived).
+# ArcGIS feature count reports 42 but one is a duplicate geometry —
+# 41 distinct ZIPCODEs confirmed against source.
+EXPECTED_ZIP_COUNT = 41
+
+# Sentinel returned when a record has been quarantined (caller should skip)
+QUARANTINED = object()
+
+
+def load_valid_zips(conn) -> frozenset:
+    """Load and verify the Louisville ZIP whitelist from silver.zip_codes.
+
+    Returns an immutable set of valid ZIP strings.
+    Raises RuntimeError if the whitelist is missing or incomplete.
+    """
+    with conn.cursor() as cur:
+        cur.execute("SELECT zip FROM silver.zip_codes ORDER BY zip")
+        zips = frozenset(row[0] for row in cur.fetchall())
+    if len(zips) < EXPECTED_ZIP_COUNT:
+        raise RuntimeError(
+            f"ZIP whitelist integrity check failed: expected {EXPECTED_ZIP_COUNT}, "
+            f"got {len(zips)}. Run `civic-data ingest-zips` first."
+        )
+    return zips
+
+
+def normalize_zip(zip_code):
+    """Normalize a ZIP code to 5 digits.
+
+    Handles ZIP+4 variants: "40211-0000", "40211 0000", "402110000".
+    Returns None if input is empty/missing (unknown ZIP).
+    """
+    if not zip_code:
+        return None
+    raw = str(zip_code).strip()
+    if not raw:
+        return None
+
+    # Extract leading 5 digits from common formats:
+    #   "40211"         -> "40211"
+    #   "40211-0000"    -> "40211"
+    #   "40211 0000"    -> "40211"
+    #   "402110000"     -> "40211"
+    #   "40211-1769"    -> "40211"
+    m = re.match(r'^(\d{5})[\s\-]?\d*$', raw)
+    if m:
+        return m.group(1)
+
+    # Didn't match — return as-is for quarantine (e.g., "UNKNOWN", "4020")
+    return raw
+
+
+def validate_zip(cur, zip_code, raw_record, source_table, valid_zips):
+    """Validate a ZIP code against the Louisville whitelist.
+
+    Returns:
+      - normalized ZIP string if valid (insert with this ZIP)
+      - None if ZIP is missing/empty (insert with NULL — unknown, not invalid)
+      - QUARANTINED sentinel if the record was quarantined (caller should skip)
+    """
+    if not zip_code or not str(zip_code).strip():
+        return None  # unknown ZIP — not a validation failure
+
+    normalized = normalize_zip(zip_code)
+
+    if normalized in valid_zips:
+        return normalized
+
+    # Quarantine: ZIP exists but is not in Louisville
+    _quarantine_record(cur, source_table, str(zip_code).strip(), raw_record,
+                       "ZIP not in Louisville boundary set")
+    return QUARANTINED
+
+
+def strip_pii_keys(record, keys):
+    """Return a shallow copy of a raw source record with the named keys removed.
+
+    Enforces the no-PII-in-DB policy at the ingest boundary: person-name source fields are
+    stripped BEFORE the record is written to bronze raw or stored in the quarantine JSONB, so
+    individual names never persist anywhere. Addresses are NOT stripped (kept by policy).
+    Removing an absent key is a no-op (safe across source schema variants).
+    """
+    if not record:
+        return record
+    return {k: v for k, v in record.items() if k not in keys}
+
+
+def derive_zip_from_point(cur, longitude, latitude):
+    """Derive a Louisville ZIP from a point by spatial join against silver.zip_codes.
+
+    For point datasets that carry NO ZIP field (e.g. STR permits) — the geometry is the
+    only locator. Returns the containing ZIP string, or None when the point is missing or
+    falls outside the 41-ZIP boundary set.
+
+    Pairs with validate_zip()'s contract: a derived ZIP is always in the valid set (it came
+    FROM the set), so callers insert it directly; None means "real record, no usable
+    location" → insert with NULL zip (kept, not quarantined).
+    """
+    if longitude is None or latitude is None:
+        return None
+    cur.execute(
+        """SELECT zip FROM silver.zip_codes
+           WHERE ST_Contains(geom, ST_SetSRID(ST_MakePoint(%s, %s), 4326))
+           LIMIT 1""",
+        (longitude, latitude),
+    )
+    row = cur.fetchone()
+    return row[0] if row else None
+
+
+def _quarantine_record(cur, source_table, zip_code, raw_record, reason):
+    """Insert a rejected record into the quarantine table."""
+    cur.execute(
+        "INSERT INTO silver.quarantine (source_table, zip_code, record_data, reason) "
+        "VALUES (%s, %s, %s::jsonb, %s)",
+        (source_table, str(zip_code).strip(), json.dumps(raw_record, default=str), reason),
+    )

civic_data/bulk.py

@@ -0,0 +1,55 @@
+"""Bulk-insert helper — batched multi-row INSERT via psycopg2 execute_values.
+
+Introduced for the LOJIC reference tables (~450K address points): the row-by-row
+promote path is fine for typed event promotion, but far too slow at reference-table
+scale. Reusable for any large silver load. Kept out of db.py so that module stays a
+pure connection factory.
+
+`template` describes ONE row's placeholder layout, so a caller can build PostGIS
+geometry inline, e.g. "(%s, %s, ST_SetSRID(ST_MakePoint(%s, %s), 4326), %s)" — the
+per-row tuple then supplies args in the order the %s appear.
+"""
+
+from psycopg2.extras import execute_values
+
+
+def log_source_pull(conn, adapter, source_dataset, source_url, source_rowcount, record_count):
+    """Record one completed source pull in bronze.sync_log (Constitution art. 1: every dataset
+    carries per-pull provenance). Domain ingests write their own running→ok ledger rows inline;
+    MDM/reference loads are quick single-shot pulls, so they log one 'ok' row after success.
+
+    Convention: MDM tables with no per-row source_dataset column ledger under their SILVER TABLE
+    NAME (e.g. 'zip_codes', 'council_districts') — the exporter's provenance fallback looks pulls
+    up by that name, keeping the mapping introspectable instead of hand-maintained."""
+    with conn.cursor() as cur:
+        cur.execute(
+            """INSERT INTO bronze.sync_log
+                   (adapter, source_dataset, source_url, source_rowcount, status,
+                    completed_at, record_count)
+               VALUES (%s, %s, %s, %s, 'ok', NOW(), %s)""",
+            (adapter, source_dataset, source_url, source_rowcount, record_count),
+        )
+    conn.commit()
+
+
+def bulk_insert(conn, table, columns, rows, template=None, page_size=5000) -> int:
+    """Insert `rows` into `table` (a fully-qualified name) over `columns`, committing once.
+
+    `table`/`columns` are trusted, code-supplied identifiers (never user input) — the
+    only interpolation here. Row values are always parameterized by execute_values.
+    Returns the number of rows inserted.
+    """
+    rows = list(rows)
+    if not rows:
+        return 0
+    col_list = ", ".join(columns)
+    with conn.cursor() as cur:
+        execute_values(
+            cur,
+            f"INSERT INTO {table} ({col_list}) VALUES %s",
+            rows,
+            template=template,
+            page_size=page_size,
+        )
+    conn.commit()
+    return len(rows)