civic-data 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. civic_data-0.1.0/LICENSE +21 -0
  2. civic_data-0.1.0/LICENSE-data +37 -0
  3. civic_data-0.1.0/PKG-INFO +129 -0
  4. civic_data-0.1.0/README.md +100 -0
  5. civic_data-0.1.0/civic_data/__init__.py +14 -0
  6. civic_data-0.1.0/civic_data/adapters/__init__.py +7 -0
  7. civic_data-0.1.0/civic_data/adapters/arcgis.py +128 -0
  8. civic_data-0.1.0/civic_data/adapters/base.py +71 -0
  9. civic_data-0.1.0/civic_data/adapters/socrata.py +28 -0
  10. civic_data-0.1.0/civic_data/adapters/validation.py +135 -0
  11. civic_data-0.1.0/civic_data/bulk.py +55 -0
  12. civic_data-0.1.0/civic_data/cli.py +533 -0
  13. civic_data-0.1.0/civic_data/consumer/__init__.py +6 -0
  14. civic_data-0.1.0/civic_data/consumer/bootstrap.py +178 -0
  15. civic_data-0.1.0/civic_data/consumer/bundle.py +213 -0
  16. civic_data-0.1.0/civic_data/consumer/lake.py +87 -0
  17. civic_data-0.1.0/civic_data/consumer/registry.py +193 -0
  18. civic_data-0.1.0/civic_data/consumer/verbs.py +401 -0
  19. civic_data-0.1.0/civic_data/datadict/__init__.py +12 -0
  20. civic_data-0.1.0/civic_data/datadict/audit.py +105 -0
  21. civic_data-0.1.0/civic_data/datadict/config.py +538 -0
  22. civic_data-0.1.0/civic_data/datadict/generator.py +504 -0
  23. civic_data-0.1.0/civic_data/datadict/inventory.py +46 -0
  24. civic_data-0.1.0/civic_data/datadict/lineage.py +142 -0
  25. civic_data-0.1.0/civic_data/datadict/lineage_config.py +756 -0
  26. civic_data-0.1.0/civic_data/db.py +40 -0
  27. civic_data-0.1.0/civic_data/handoff/__init__.py +7 -0
  28. civic_data-0.1.0/civic_data/handoff/dump_local.py +126 -0
  29. civic_data-0.1.0/civic_data/handoff/export_parquet.py +360 -0
  30. civic_data-0.1.0/civic_data/ingest/__init__.py +13 -0
  31. civic_data-0.1.0/civic_data/ingest/abc.py +225 -0
  32. civic_data-0.1.0/civic_data/ingest/crime.py +340 -0
  33. civic_data-0.1.0/civic_data/ingest/crime_geocode.py +178 -0
  34. civic_data-0.1.0/civic_data/ingest/food_service.py +123 -0
  35. civic_data-0.1.0/civic_data/ingest/foreclosures.py +153 -0
  36. civic_data-0.1.0/civic_data/ingest/geography.py +232 -0
  37. civic_data-0.1.0/civic_data/ingest/inspections.py +126 -0
  38. civic_data-0.1.0/civic_data/ingest/landbank.py +146 -0
  39. civic_data-0.1.0/civic_data/ingest/lien_orders.py +160 -0
  40. civic_data-0.1.0/civic_data/ingest/lojic_reference.py +137 -0
  41. civic_data-0.1.0/civic_data/ingest/permits.py +144 -0
  42. civic_data-0.1.0/civic_data/ingest/population.py +125 -0
  43. civic_data-0.1.0/civic_data/ingest/requests_311.py +292 -0
  44. civic_data-0.1.0/civic_data/ingest/str_licenses.py +165 -0
  45. civic_data-0.1.0/civic_data/ingest/zip_boundaries.py +101 -0
  46. civic_data-0.1.0/civic_data/quality.py +212 -0
  47. civic_data-0.1.0/civic_data.egg-info/PKG-INFO +129 -0
  48. civic_data-0.1.0/civic_data.egg-info/SOURCES.txt +60 -0
  49. civic_data-0.1.0/civic_data.egg-info/dependency_links.txt +1 -0
  50. civic_data-0.1.0/civic_data.egg-info/entry_points.txt +2 -0
  51. civic_data-0.1.0/civic_data.egg-info/requires.txt +10 -0
  52. civic_data-0.1.0/civic_data.egg-info/top_level.txt +1 -0
  53. civic_data-0.1.0/pyproject.toml +52 -0
  54. civic_data-0.1.0/setup.cfg +4 -0
  55. civic_data-0.1.0/tests/test_catalog_contract.py +173 -0
  56. civic_data-0.1.0/tests/test_citability.py +43 -0
  57. civic_data-0.1.0/tests/test_consumer_verbs.py +385 -0
  58. civic_data-0.1.0/tests/test_packaging.py +82 -0
  59. civic_data-0.1.0/tests/test_pii_absence.py +115 -0
  60. civic_data-0.1.0/tests/test_producer_invariants.py +87 -0
  61. civic_data-0.1.0/tests/test_producer_smoke.py +227 -0
  62. civic_data-0.1.0/tests/test_registry.py +135 -0
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 NewGuyAI
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,37 @@
1
+ Data & Documentation License
2
+ ============================
3
+
4
+ The DATA and DOCUMENTATION produced by this project — the cleaned/normalized
5
+ tables, the generated data dictionary, the transformation-lineage records, and
6
+ the prose in this repository — are licensed under the
7
+
8
+ Creative Commons Attribution 4.0 International License (CC BY 4.0).
9
+
10
+ Full legal text: https://creativecommons.org/licenses/by/4.0/legalcode
11
+ Human-readable summary: https://creativecommons.org/licenses/by/4.0/
12
+
13
+ You are free to share and adapt this material for any purpose, including
14
+ commercially, provided you give appropriate credit ("civic-data"), link to the
15
+ license, and indicate if changes were made.
16
+
17
+ (The project's SOURCE CODE is licensed separately under the MIT License; see
18
+ the LICENSE file.)
19
+
20
+ ------------------------------------------------------------------------------
21
+ What this license does and does NOT cover
22
+ ------------------------------------------------------------------------------
23
+
24
+ This project adds value to public open data by cleaning, normalizing, and
25
+ documenting it. The CC BY 4.0 license above covers ONLY that added value.
26
+
27
+ The UNDERLYING RAW DATA is public record obtained from third-party open-data
28
+ portals (Louisville Metro / LOJIC Open Data on ArcGIS Hub, and others as cities
29
+ are added). That raw data is provided by its sources "as is," and its own terms
30
+ of use are set by those sources, not by this project. Per-layer source terms are
31
+ UNVERIFIED in this repository (see each domain's "License / terms" line in the
32
+ generated data dictionary).
33
+
34
+ Before republishing a specific source dataset externally, confirm that layer's
35
+ terms of use on its portal item page and attribute the original source
36
+ (e.g. "Louisville Metro / LOJIC Open Data"). Treat commercial redistribution
37
+ rights on the underlying raw data as unconfirmed until checked.
@@ -0,0 +1,129 @@
1
+ Metadata-Version: 2.4
2
+ Name: civic-data
3
+ Version: 0.1.0
4
+ Summary: Cleaned, documented civic open data — a DuckDB/GeoParquet consumer CLI plus the extraction/normalization spine that builds it, with a generated data dictionary and transformation-lineage layer.
5
+ Author: NewGuyAI
6
+ License: MIT
7
+ Project-URL: Homepage, https://newguyai.com
8
+ Keywords: civic-data,open-data,duckdb,geoparquet,louisville,data-dictionary
9
+ Classifier: Development Status :: 4 - Beta
10
+ Classifier: Intended Audience :: Developers
11
+ Classifier: Intended Audience :: Science/Research
12
+ Classifier: License :: OSI Approved :: MIT License
13
+ Classifier: Programming Language :: Python :: 3
14
+ Classifier: Operating System :: OS Independent
15
+ Classifier: Topic :: Scientific/Engineering :: GIS
16
+ Requires-Python: >=3.11
17
+ Description-Content-Type: text/markdown
18
+ License-File: LICENSE
19
+ License-File: LICENSE-data
20
+ Requires-Dist: duckdb>=1.1
21
+ Requires-Dist: pytz>=2024.1
22
+ Provides-Extra: producer
23
+ Requires-Dist: psycopg2-binary>=2.9; extra == "producer"
24
+ Requires-Dist: httpx>=0.27; extra == "producer"
25
+ Requires-Dist: python-dotenv>=1.0; extra == "producer"
26
+ Provides-Extra: dev
27
+ Requires-Dist: pytest>=8.0; extra == "dev"
28
+ Dynamic: license-file
29
+
30
+ # civic-data
31
+
32
+ **Cleaned, normalized, and documented public open data — with a generated, code-coupled data
33
+ dictionary and transformation-lineage layer, consumable as a GeoParquet lake with a self-describing
34
+ CLI.**
35
+
36
+ The city gives away raw open data. civic-data's value is not the data — it's the honest,
37
+ reproducible record of **what we did to it, why, and what's lost**: schema, grain, NULL rates,
38
+ per-year coverage, quarantine loss, bias caveats, and a step-by-step transformation lineage that
39
+ doubles as the reproduction recipe. Because that record is *generated from the live database and the
40
+ ingest code*, it cannot drift from reality.
41
+
42
+ v1 covers **Louisville, KY**: ten domain datasets (crime, 311 service requests, landbank sales,
43
+ lien orders, foreclosures, food-service establishments, ABC licenses, restaurant inspections,
44
+ construction permits, short-term-rental licenses) plus the geography spine (ZIP boundaries, council
45
+ districts, neighborhoods, HUD zones, crosswalks, Census population) — ~3.5M rows, all ingested from
46
+ source, all lineage-documented.
47
+
48
+ ## Quick start (60 seconds, no database required)
49
+
50
+ ```bash
51
+ pip install civic-data # consumer install — DuckDB only, no Postgres
52
+ civic-data cities # list published city lakes
53
+ civic-data get louisville # download the lake (sha256-verified) into ./export
54
+ civic-data query -d crime --zip 40202 --limit 5
55
+ ```
56
+
57
+ ## Consuming the data
58
+
59
+ The consumer product is a **GeoParquet lake** described by a machine-readable `catalog.json`, read
60
+ with DuckDB — no server, no Docker, no Postgres. Every verb takes `--json` (for agents) and
61
+ `--lake <dir>` (default: `./export`):
62
+
63
+ ```bash
64
+ civic-data catalog --json # the discoverability contract — start here
65
+ civic-data datasets # list datasets by catalog key
66
+ civic-data schema crime # one dataset's columns + shape
67
+ civic-data query -d crime --zip 40202 --year 2024 --limit 5
68
+ civic-data query -d crime --district 6 # point-in-polygon; geocoded rows only (it says so)
69
+ civic-data place 40202 # one place across every dataset: identity + counts
70
+ civic-data sql "SELECT zip_code, count(*) FROM crime GROUP BY 1" # raw DuckDB; keys are views
71
+ civic-data query -d crime --sources # provenance: source pulls, license, how-to-cite
72
+ ```
73
+
74
+ **Hand someone a place as a folder:** `civic-data place 6 --export` writes a self-contained
75
+ mini-lake (filtered Parquet + the district's boundary/crosswalks/population + a scoped catalog +
76
+ generated `llms.txt`/`AGENTS.md`). Every verb above works against that folder via `--lake`.
77
+
78
+ The lake root also carries generated **`llms.txt`** and **`AGENTS.md`** so an AI agent can bootstrap
79
+ itself from the artifact alone. Both are regenerated from the catalog on every export — never
80
+ hand-edited (a test enforces it).
81
+
82
+ ## Producing the data (the reproduction path)
83
+
84
+ ```bash
85
+ docker compose up -d # PostgreSQL 16 + PostGIS on host port 5434
86
+ python -m venv .venv && . .venv/Scripts/activate # (Windows: .venv\Scripts\activate)
87
+ pip install -e .[producer] # the pipeline extras (Postgres driver, HTTP client)
88
+
89
+ civic-data ingest-zips # canonical ZIP boundary set (MDM) — run first
90
+ civic-data ingest-refs # LOJIC geocoding references (address points + intersections)
91
+ civic-data ingest-geo && civic-data ingest-population # boundaries, crosswalks, Census ACS
92
+ civic-data ingest crime --all --promote # source -> bronze -> silver (any of the 10 domains)
93
+ civic-data geocode-crime --write --apply # place ~1.2M crime records (geom + derived ZIP)
94
+ civic-data datadict --with-source-audit # generate the data dictionary + lineage
95
+ civic-data export # silver -> GeoParquet lake + catalog + agent docs
96
+ ```
97
+
98
+ The generated dictionary lands in [`docs/data-dictionary/`](docs/data-dictionary/) — a human-readable
99
+ `README.md` and a machine-readable `data-reality.json` (diffing successive runs is drift detection).
100
+
101
+ Ingest order matters: `ingest-zips` first. The ZIP set is the canonical geography (MDM) every record
102
+ validates against; crime/311 ingest refuse to run without it.
103
+
104
+ ## Honesty, by construction
105
+
106
+ - **Trust tiers** on every dataset (`trendable` / `viewable_caveated` / `snapshot_only`) — a current
107
+ roster is not a time series, and the catalog says which is which.
108
+ - **Documented loss, not silent loss:** quarantine counts, geocode coverage (spatial filters match
109
+ geocoded rows only, and the CLI says so), source-shape divergences recorded as findings.
110
+ - **Generated, never hand-maintained:** the dictionary, the catalog, and the agent docs are all
111
+ derived from the live database + code; drift guards raise when authored prose falls out of sync.
112
+
113
+ ## Licensing
114
+
115
+ - **Code:** MIT — see [`LICENSE`](LICENSE).
116
+ - **Data & documentation:** CC BY 4.0 — see [`LICENSE-data`](LICENSE-data).
117
+
118
+ The underlying raw data is **public record, provided as-is**, and its terms were verified per origin
119
+ (2026-07-11): Louisville Metro publishes under the **PDDL** (public domain; when republishing
120
+ modified data, identify the source, version, and modifications — which the generated lineage layer
121
+ does per dataset); LOJIC layers are public domain (*Mapping Data Source: LOJIC*); Census ACS is a
122
+ U.S. government work — *this product uses the Census Bureau Data API but is not endorsed or
123
+ certified by the Census Bureau.* Every dataset's exact license text ships in its catalog provenance:
124
+ `civic-data query -d <dataset> --sources`.
125
+
126
+ ## Provenance
127
+
128
+ Born out of civic-graph (the private flagship built on top of this layer, and the origin of this
129
+ spine). See `wiki.md` for the source of truth and `ai-working/plans/` for the phase plans.
@@ -0,0 +1,100 @@
1
+ # civic-data
2
+
3
+ **Cleaned, normalized, and documented public open data — with a generated, code-coupled data
4
+ dictionary and transformation-lineage layer, consumable as a GeoParquet lake with a self-describing
5
+ CLI.**
6
+
7
+ The city gives away raw open data. civic-data's value is not the data — it's the honest,
8
+ reproducible record of **what we did to it, why, and what's lost**: schema, grain, NULL rates,
9
+ per-year coverage, quarantine loss, bias caveats, and a step-by-step transformation lineage that
10
+ doubles as the reproduction recipe. Because that record is *generated from the live database and the
11
+ ingest code*, it cannot drift from reality.
12
+
13
+ v1 covers **Louisville, KY**: ten domain datasets (crime, 311 service requests, landbank sales,
14
+ lien orders, foreclosures, food-service establishments, ABC licenses, restaurant inspections,
15
+ construction permits, short-term-rental licenses) plus the geography spine (ZIP boundaries, council
16
+ districts, neighborhoods, HUD zones, crosswalks, Census population) — ~3.5M rows, all ingested from
17
+ source, all lineage-documented.
18
+
19
+ ## Quick start (60 seconds, no database required)
20
+
21
+ ```bash
22
+ pip install civic-data # consumer install — DuckDB only, no Postgres
23
+ civic-data cities # list published city lakes
24
+ civic-data get louisville # download the lake (sha256-verified) into ./export
25
+ civic-data query -d crime --zip 40202 --limit 5
26
+ ```
27
+
28
+ ## Consuming the data
29
+
30
+ The consumer product is a **GeoParquet lake** described by a machine-readable `catalog.json`, read
31
+ with DuckDB — no server, no Docker, no Postgres. Every verb takes `--json` (for agents) and
32
+ `--lake <dir>` (default: `./export`):
33
+
34
+ ```bash
35
+ civic-data catalog --json # the discoverability contract — start here
36
+ civic-data datasets # list datasets by catalog key
37
+ civic-data schema crime # one dataset's columns + shape
38
+ civic-data query -d crime --zip 40202 --year 2024 --limit 5
39
+ civic-data query -d crime --district 6 # point-in-polygon; geocoded rows only (it says so)
40
+ civic-data place 40202 # one place across every dataset: identity + counts
41
+ civic-data sql "SELECT zip_code, count(*) FROM crime GROUP BY 1" # raw DuckDB; keys are views
42
+ civic-data query -d crime --sources # provenance: source pulls, license, how-to-cite
43
+ ```
44
+
45
+ **Hand someone a place as a folder:** `civic-data place 6 --export` writes a self-contained
46
+ mini-lake (filtered Parquet + the district's boundary/crosswalks/population + a scoped catalog +
47
+ generated `llms.txt`/`AGENTS.md`). Every verb above works against that folder via `--lake`.
48
+
49
+ The lake root also carries generated **`llms.txt`** and **`AGENTS.md`** so an AI agent can bootstrap
50
+ itself from the artifact alone. Both are regenerated from the catalog on every export — never
51
+ hand-edited (a test enforces it).
52
+
53
+ ## Producing the data (the reproduction path)
54
+
55
+ ```bash
56
+ docker compose up -d # PostgreSQL 16 + PostGIS on host port 5434
57
+ python -m venv .venv && . .venv/Scripts/activate # (Windows: .venv\Scripts\activate)
58
+ pip install -e .[producer] # the pipeline extras (Postgres driver, HTTP client)
59
+
60
+ civic-data ingest-zips # canonical ZIP boundary set (MDM) — run first
61
+ civic-data ingest-refs # LOJIC geocoding references (address points + intersections)
62
+ civic-data ingest-geo && civic-data ingest-population # boundaries, crosswalks, Census ACS
63
+ civic-data ingest crime --all --promote # source -> bronze -> silver (any of the 10 domains)
64
+ civic-data geocode-crime --write --apply # place ~1.2M crime records (geom + derived ZIP)
65
+ civic-data datadict --with-source-audit # generate the data dictionary + lineage
66
+ civic-data export # silver -> GeoParquet lake + catalog + agent docs
67
+ ```
68
+
69
+ The generated dictionary lands in [`docs/data-dictionary/`](docs/data-dictionary/) — a human-readable
70
+ `README.md` and a machine-readable `data-reality.json` (diffing successive runs is drift detection).
71
+
72
+ Ingest order matters: `ingest-zips` first. The ZIP set is the canonical geography (MDM) every record
73
+ validates against; crime/311 ingest refuse to run without it.
74
+
75
+ ## Honesty, by construction
76
+
77
+ - **Trust tiers** on every dataset (`trendable` / `viewable_caveated` / `snapshot_only`) — a current
78
+ roster is not a time series, and the catalog says which is which.
79
+ - **Documented loss, not silent loss:** quarantine counts, geocode coverage (spatial filters match
80
+ geocoded rows only, and the CLI says so), source-shape divergences recorded as findings.
81
+ - **Generated, never hand-maintained:** the dictionary, the catalog, and the agent docs are all
82
+ derived from the live database + code; drift guards raise when authored prose falls out of sync.
83
+
84
+ ## Licensing
85
+
86
+ - **Code:** MIT — see [`LICENSE`](LICENSE).
87
+ - **Data & documentation:** CC BY 4.0 — see [`LICENSE-data`](LICENSE-data).
88
+
89
+ The underlying raw data is **public record, provided as-is**, and its terms were verified per origin
90
+ (2026-07-11): Louisville Metro publishes under the **PDDL** (public domain; when republishing
91
+ modified data, identify the source, version, and modifications — which the generated lineage layer
92
+ does per dataset); LOJIC layers are public domain (*Mapping Data Source: LOJIC*); Census ACS is a
93
+ U.S. government work — *this product uses the Census Bureau Data API but is not endorsed or
94
+ certified by the Census Bureau.* Every dataset's exact license text ships in its catalog provenance:
95
+ `civic-data query -d <dataset> --sources`.
96
+
97
+ ## Provenance
98
+
99
+ Born out of civic-graph (the private flagship built on top of this layer, and the origin of this
100
+ spine). See `wiki.md` for the source of truth and `ai-working/plans/` for the phase plans.
@@ -0,0 +1,14 @@
1
+ """civic-data — the extraction / normalization / documentation spine.
2
+
3
+ One canonical home for the data-layer core: source adapters, medallion ingest
4
+ (bronze → silver), and a generated data dictionary with a transformation-lineage
5
+ layer. The product's value is not the data (the city gives that away raw) — it is
6
+ the *documented, reproducible transformation lineage*: what we did, why, and what
7
+ is lost. That lineage is generated from the live DB + code, never hand-maintained,
8
+ so it cannot drift from reality.
9
+
10
+ civic-graph (the private flagship) was the origin of this code and is now a
11
+ downstream consumer / worked example built on top.
12
+ """
13
+
14
+ __version__ = "0.1.0"
@@ -0,0 +1,7 @@
1
+ """Source adapters — the extraction spine.
2
+
3
+ `arcgis` is the working portal adapter (Louisville / LOJIC, and any ArcGIS Hub
4
+ portal). `socrata` is a dormant placeholder that activates at city #2
5
+ (Cincinnati / Chicago). `validation` is the ZIP/geography gatekeeper shared by
6
+ every ingest path. `base` is the adapter contract.
7
+ """
@@ -0,0 +1,128 @@
1
+ """ArcGIS Feature Service adapter.
2
+
3
+ Louisville's open data portal runs on ArcGIS Hub. Most datasets are
4
+ exposed as ArcGIS Feature Services with a REST query endpoint.
5
+
6
+ Feature Service query pattern:
7
+ GET {service_url}/query?where=1=1&outFields=*&f=json&resultOffset=0&resultRecordCount=2000
8
+
9
+ Pagination: resultOffset increments by resultRecordCount until
10
+ exceededTransferLimit is false.
11
+ """
12
+
13
+ import time
14
+ import httpx
15
+ from typing import Any
16
+
17
+
18
+ def feature_count(service_url: str, timeout: float = 30.0) -> int | None:
19
+ """Source row count via ArcGIS returnCountOnly (one tiny call). None on error.
20
+
21
+ Captured at ingest as source-pull metadata (the reproducibility substrate): the
22
+ source_rowcount at pull time, to compare against what we actually fetched/kept.
23
+ """
24
+ try:
25
+ resp = httpx.get(f"{service_url}/query",
26
+ params={"where": "1=1", "returnCountOnly": "true", "f": "json"},
27
+ timeout=timeout)
28
+ resp.raise_for_status()
29
+ return resp.json().get("count")
30
+ except Exception:
31
+ return None
32
+
33
+
34
+ def _query_page(client, url, params, max_retries: int = 4):
35
+ """Fetch one page, retrying transient failures. Returns parsed JSON with a 'features' key.
36
+
37
+ Raises RuntimeError if the server never returns a valid page — so a transient error
38
+ (HTTP 5xx, an ArcGIS error payload, a truncated body) can NEVER be mistaken for
39
+ end-of-data and silently truncate a large pull. Only a *valid* response with an empty
40
+ 'features' array or exceededTransferLimit=False ends pagination (handled by the caller).
41
+ """
42
+ last_err = None
43
+ for attempt in range(max_retries):
44
+ try:
45
+ resp = client.get(url, params=params)
46
+ resp.raise_for_status()
47
+ data = resp.json()
48
+ if "error" in data:
49
+ last_err = f"ArcGIS error payload: {data['error']}"
50
+ elif "features" not in data:
51
+ last_err = f"response missing 'features' (keys={list(data)[:6]})"
52
+ else:
53
+ return data
54
+ except (httpx.HTTPError, ValueError) as e: # ValueError also covers JSON decode errors
55
+ last_err = repr(e)
56
+ time.sleep(1.0 * (attempt + 1)) # linear backoff between retries
57
+ raise RuntimeError(
58
+ f"ArcGIS page failed after {max_retries} attempts "
59
+ f"(resultOffset={params.get('resultOffset')}): {last_err}")
60
+
61
+
62
+ def fetch_feature_service(
63
+ service_url: str,
64
+ where: str = "1=1",
65
+ out_fields: str = "*",
66
+ page_size: int = 2000,
67
+ max_records: int | None = None,
68
+ delay: float = 0.5,
69
+ ) -> list[dict[str, Any]]:
70
+ """Fetch all records from an ArcGIS Feature Service.
71
+
72
+ Args:
73
+ service_url: The Feature Service layer URL
74
+ (e.g., https://services1.arcgis.com/.../FeatureServer/0)
75
+ where: SQL WHERE clause for filtering
76
+ out_fields: Comma-separated field names or "*" for all
77
+ page_size: Records per request (max varies by service, 2000 is safe)
78
+ max_records: Optional cap on total records fetched
79
+ delay: Seconds between paginated requests (polite scraping)
80
+
81
+ Returns:
82
+ List of feature attribute dicts (geometry stripped to lat/lng if present)
83
+
84
+ Pagination terminates only on a *valid* empty page or exceededTransferLimit=False;
85
+ transient per-page failures are retried (see _query_page), never treated as done.
86
+ """
87
+ all_records = []
88
+ offset = 0
89
+ url = f"{service_url}/query"
90
+
91
+ with httpx.Client(timeout=60.0) as client:
92
+ while True:
93
+ params = {
94
+ "where": where,
95
+ "outFields": out_fields,
96
+ "f": "json",
97
+ "resultOffset": offset,
98
+ "resultRecordCount": page_size,
99
+ "outSR": 4326, # WGS84
100
+ }
101
+
102
+ data = _query_page(client, url, params)
103
+ features = data.get("features", [])
104
+ if not features:
105
+ break
106
+
107
+ for f in features:
108
+ record = f.get("attributes", {})
109
+ # Extract geometry as flat lat/lng if present
110
+ geom = f.get("geometry")
111
+ if geom:
112
+ record["_longitude"] = geom.get("x")
113
+ record["_latitude"] = geom.get("y")
114
+ all_records.append(record)
115
+
116
+ if max_records and len(all_records) >= max_records:
117
+ all_records = all_records[:max_records]
118
+ break
119
+
120
+ if not data.get("exceededTransferLimit", False):
121
+ break
122
+
123
+ # Increment by actual records returned, not page_size.
124
+ # Some services cap below our page_size (e.g., 1000 max).
125
+ offset += len(features)
126
+ time.sleep(delay)
127
+
128
+ return all_records
@@ -0,0 +1,71 @@
1
+ """Base adapter interface for civic data sources."""
2
+
3
+ from abc import ABC, abstractmethod
4
+ from dataclasses import dataclass
5
+ from typing import Any
6
+
7
+
8
+ @dataclass
9
+ class IngestResult:
10
+ """Result of a data ingestion run."""
11
+ adapter: str
12
+ source_dataset: str
13
+ record_count: int
14
+ status: str # "ok" or "error"
15
+ error_message: str | None = None
16
+
17
+
18
+ class CivicDataAdapter(ABC):
19
+ """Abstract base class for all civic data adapters.
20
+
21
+ Each adapter handles one data domain (crime, 311, property, etc.)
22
+ and knows how to:
23
+ 1. Fetch raw data from the source API
24
+ 2. Store raw records in bronze layer (JSONB, append-only)
25
+ 3. Transform raw records into silver layer (typed, geocoded)
26
+ """
27
+
28
+ @property
29
+ @abstractmethod
30
+ def adapter_name(self) -> str:
31
+ """Unique identifier for this adapter (e.g., 'crime', 'service_requests')."""
32
+ ...
33
+
34
+ @abstractmethod
35
+ def fetch_raw(self, dataset_id: str, **kwargs) -> list[dict[str, Any]]:
36
+ """Fetch raw records from the source API.
37
+
38
+ Returns a list of raw JSON-serializable dicts.
39
+ """
40
+ ...
41
+
42
+ @abstractmethod
43
+ def load_bronze(self, records: list[dict[str, Any]], dataset_id: str) -> int:
44
+ """Store raw records in the bronze layer. Returns count of rows inserted."""
45
+ ...
46
+
47
+ @abstractmethod
48
+ def promote_to_silver(self, dataset_id: str) -> int:
49
+ """Transform bronze records into silver layer. Returns count of rows promoted."""
50
+ ...
51
+
52
+ def ingest(self, dataset_id: str, **kwargs) -> IngestResult:
53
+ """Full ingestion pipeline: fetch → bronze → silver."""
54
+ try:
55
+ raw = self.fetch_raw(dataset_id, **kwargs)
56
+ bronze_count = self.load_bronze(raw, dataset_id)
57
+ silver_count = self.promote_to_silver(dataset_id)
58
+ return IngestResult(
59
+ adapter=self.adapter_name,
60
+ source_dataset=dataset_id,
61
+ record_count=silver_count,
62
+ status="ok",
63
+ )
64
+ except Exception as e:
65
+ return IngestResult(
66
+ adapter=self.adapter_name,
67
+ source_dataset=dataset_id,
68
+ record_count=0,
69
+ status="error",
70
+ error_message=str(e),
71
+ )
@@ -0,0 +1,28 @@
1
+ """Socrata Open Data adapter — DORMANT (activates at city #2).
2
+
3
+ STATUS: placeholder, not implemented. v1 is Louisville-only (crime + 311) on
4
+ ArcGIS Hub; see `adapters/arcgis.py` for the working pattern. Socrata is the
5
+ multi-city unlock — Cincinnati, Chicago, and ~100 other US cities publish on the
6
+ Socrata platform (SODA API). When city #2 lands, implement `fetch_dataset`
7
+ against the SODA endpoint, mirroring the shape `arcgis.fetch_feature_service`
8
+ returns (a list of flat attribute dicts) so the downstream ingest/validation
9
+ spine is unchanged.
10
+
11
+ Design note (from the kickoff): this file was listed in the seed manifest as
12
+ inherited from civic-graph, but civic-graph never actually authored a Socrata
13
+ adapter — only the ArcGIS one exists there. So this is a fresh, honest stub, not
14
+ a lift. Multi-city is the thesis, not the v1 scope; do not build it out until a
15
+ real second city is in scope (guardrail: no future-proofing).
16
+
17
+ SODA query pattern (for reference when implemented):
18
+ GET https://{domain}/resource/{dataset_id}.json?$limit=50000&$offset=0
19
+ Pagination via $limit/$offset; optional $$app_token header to raise rate limits.
20
+ """
21
+
22
+
23
+ def fetch_dataset(*args, **kwargs):
24
+ raise NotImplementedError(
25
+ "The Socrata adapter is dormant. v1 is Louisville-only (ArcGIS). "
26
+ "Implement this against the SODA API when city #2 (Cincinnati/Chicago) is in scope, "
27
+ "returning the same flat-dict shape as adapters.arcgis.fetch_feature_service."
28
+ )