civic-data 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- civic_data-0.1.0/LICENSE +21 -0
- civic_data-0.1.0/LICENSE-data +37 -0
- civic_data-0.1.0/PKG-INFO +129 -0
- civic_data-0.1.0/README.md +100 -0
- civic_data-0.1.0/civic_data/__init__.py +14 -0
- civic_data-0.1.0/civic_data/adapters/__init__.py +7 -0
- civic_data-0.1.0/civic_data/adapters/arcgis.py +128 -0
- civic_data-0.1.0/civic_data/adapters/base.py +71 -0
- civic_data-0.1.0/civic_data/adapters/socrata.py +28 -0
- civic_data-0.1.0/civic_data/adapters/validation.py +135 -0
- civic_data-0.1.0/civic_data/bulk.py +55 -0
- civic_data-0.1.0/civic_data/cli.py +533 -0
- civic_data-0.1.0/civic_data/consumer/__init__.py +6 -0
- civic_data-0.1.0/civic_data/consumer/bootstrap.py +178 -0
- civic_data-0.1.0/civic_data/consumer/bundle.py +213 -0
- civic_data-0.1.0/civic_data/consumer/lake.py +87 -0
- civic_data-0.1.0/civic_data/consumer/registry.py +193 -0
- civic_data-0.1.0/civic_data/consumer/verbs.py +401 -0
- civic_data-0.1.0/civic_data/datadict/__init__.py +12 -0
- civic_data-0.1.0/civic_data/datadict/audit.py +105 -0
- civic_data-0.1.0/civic_data/datadict/config.py +538 -0
- civic_data-0.1.0/civic_data/datadict/generator.py +504 -0
- civic_data-0.1.0/civic_data/datadict/inventory.py +46 -0
- civic_data-0.1.0/civic_data/datadict/lineage.py +142 -0
- civic_data-0.1.0/civic_data/datadict/lineage_config.py +756 -0
- civic_data-0.1.0/civic_data/db.py +40 -0
- civic_data-0.1.0/civic_data/handoff/__init__.py +7 -0
- civic_data-0.1.0/civic_data/handoff/dump_local.py +126 -0
- civic_data-0.1.0/civic_data/handoff/export_parquet.py +360 -0
- civic_data-0.1.0/civic_data/ingest/__init__.py +13 -0
- civic_data-0.1.0/civic_data/ingest/abc.py +225 -0
- civic_data-0.1.0/civic_data/ingest/crime.py +340 -0
- civic_data-0.1.0/civic_data/ingest/crime_geocode.py +178 -0
- civic_data-0.1.0/civic_data/ingest/food_service.py +123 -0
- civic_data-0.1.0/civic_data/ingest/foreclosures.py +153 -0
- civic_data-0.1.0/civic_data/ingest/geography.py +232 -0
- civic_data-0.1.0/civic_data/ingest/inspections.py +126 -0
- civic_data-0.1.0/civic_data/ingest/landbank.py +146 -0
- civic_data-0.1.0/civic_data/ingest/lien_orders.py +160 -0
- civic_data-0.1.0/civic_data/ingest/lojic_reference.py +137 -0
- civic_data-0.1.0/civic_data/ingest/permits.py +144 -0
- civic_data-0.1.0/civic_data/ingest/population.py +125 -0
- civic_data-0.1.0/civic_data/ingest/requests_311.py +292 -0
- civic_data-0.1.0/civic_data/ingest/str_licenses.py +165 -0
- civic_data-0.1.0/civic_data/ingest/zip_boundaries.py +101 -0
- civic_data-0.1.0/civic_data/quality.py +212 -0
- civic_data-0.1.0/civic_data.egg-info/PKG-INFO +129 -0
- civic_data-0.1.0/civic_data.egg-info/SOURCES.txt +60 -0
- civic_data-0.1.0/civic_data.egg-info/dependency_links.txt +1 -0
- civic_data-0.1.0/civic_data.egg-info/entry_points.txt +2 -0
- civic_data-0.1.0/civic_data.egg-info/requires.txt +10 -0
- civic_data-0.1.0/civic_data.egg-info/top_level.txt +1 -0
- civic_data-0.1.0/pyproject.toml +52 -0
- civic_data-0.1.0/setup.cfg +4 -0
- civic_data-0.1.0/tests/test_catalog_contract.py +173 -0
- civic_data-0.1.0/tests/test_citability.py +43 -0
- civic_data-0.1.0/tests/test_consumer_verbs.py +385 -0
- civic_data-0.1.0/tests/test_packaging.py +82 -0
- civic_data-0.1.0/tests/test_pii_absence.py +115 -0
- civic_data-0.1.0/tests/test_producer_invariants.py +87 -0
- civic_data-0.1.0/tests/test_producer_smoke.py +227 -0
- civic_data-0.1.0/tests/test_registry.py +135 -0
civic_data-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 NewGuyAI
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
Data & Documentation License
|
|
2
|
+
============================
|
|
3
|
+
|
|
4
|
+
The DATA and DOCUMENTATION produced by this project — the cleaned/normalized
|
|
5
|
+
tables, the generated data dictionary, the transformation-lineage records, and
|
|
6
|
+
the prose in this repository — are licensed under the
|
|
7
|
+
|
|
8
|
+
Creative Commons Attribution 4.0 International License (CC BY 4.0).
|
|
9
|
+
|
|
10
|
+
Full legal text: https://creativecommons.org/licenses/by/4.0/legalcode
|
|
11
|
+
Human-readable summary: https://creativecommons.org/licenses/by/4.0/
|
|
12
|
+
|
|
13
|
+
You are free to share and adapt this material for any purpose, including
|
|
14
|
+
commercially, provided you give appropriate credit ("civic-data"), link to the
|
|
15
|
+
license, and indicate if changes were made.
|
|
16
|
+
|
|
17
|
+
(The project's SOURCE CODE is licensed separately under the MIT License; see
|
|
18
|
+
the LICENSE file.)
|
|
19
|
+
|
|
20
|
+
------------------------------------------------------------------------------
|
|
21
|
+
What this license does and does NOT cover
|
|
22
|
+
------------------------------------------------------------------------------
|
|
23
|
+
|
|
24
|
+
This project adds value to public open data by cleaning, normalizing, and
|
|
25
|
+
documenting it. The CC BY 4.0 license above covers ONLY that added value.
|
|
26
|
+
|
|
27
|
+
The UNDERLYING RAW DATA is public record obtained from third-party open-data
|
|
28
|
+
portals (Louisville Metro / LOJIC Open Data on ArcGIS Hub, and others as cities
|
|
29
|
+
are added). That raw data is provided by its sources "as is," and its own terms
|
|
30
|
+
of use are set by those sources, not by this project. Per-layer source terms are
|
|
31
|
+
UNVERIFIED in this repository (see each domain's "License / terms" line in the
|
|
32
|
+
generated data dictionary).
|
|
33
|
+
|
|
34
|
+
Before republishing a specific source dataset externally, confirm that layer's
|
|
35
|
+
terms of use on its portal item page and attribute the original source
|
|
36
|
+
(e.g. "Louisville Metro / LOJIC Open Data"). Treat commercial redistribution
|
|
37
|
+
rights on the underlying raw data as unconfirmed until checked.
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: civic-data
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Cleaned, documented civic open data — a DuckDB/GeoParquet consumer CLI plus the extraction/normalization spine that builds it, with a generated data dictionary and transformation-lineage layer.
|
|
5
|
+
Author: NewGuyAI
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://newguyai.com
|
|
8
|
+
Keywords: civic-data,open-data,duckdb,geoparquet,louisville,data-dictionary
|
|
9
|
+
Classifier: Development Status :: 4 - Beta
|
|
10
|
+
Classifier: Intended Audience :: Developers
|
|
11
|
+
Classifier: Intended Audience :: Science/Research
|
|
12
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Operating System :: OS Independent
|
|
15
|
+
Classifier: Topic :: Scientific/Engineering :: GIS
|
|
16
|
+
Requires-Python: >=3.11
|
|
17
|
+
Description-Content-Type: text/markdown
|
|
18
|
+
License-File: LICENSE
|
|
19
|
+
License-File: LICENSE-data
|
|
20
|
+
Requires-Dist: duckdb>=1.1
|
|
21
|
+
Requires-Dist: pytz>=2024.1
|
|
22
|
+
Provides-Extra: producer
|
|
23
|
+
Requires-Dist: psycopg2-binary>=2.9; extra == "producer"
|
|
24
|
+
Requires-Dist: httpx>=0.27; extra == "producer"
|
|
25
|
+
Requires-Dist: python-dotenv>=1.0; extra == "producer"
|
|
26
|
+
Provides-Extra: dev
|
|
27
|
+
Requires-Dist: pytest>=8.0; extra == "dev"
|
|
28
|
+
Dynamic: license-file
|
|
29
|
+
|
|
30
|
+
# civic-data
|
|
31
|
+
|
|
32
|
+
**Cleaned, normalized, and documented public open data — with a generated, code-coupled data
|
|
33
|
+
dictionary and transformation-lineage layer, consumable as a GeoParquet lake with a self-describing
|
|
34
|
+
CLI.**
|
|
35
|
+
|
|
36
|
+
The city gives away raw open data. civic-data's value is not the data — it's the honest,
|
|
37
|
+
reproducible record of **what we did to it, why, and what's lost**: schema, grain, NULL rates,
|
|
38
|
+
per-year coverage, quarantine loss, bias caveats, and a step-by-step transformation lineage that
|
|
39
|
+
doubles as the reproduction recipe. Because that record is *generated from the live database and the
|
|
40
|
+
ingest code*, it cannot drift from reality.
|
|
41
|
+
|
|
42
|
+
v1 covers **Louisville, KY**: ten domain datasets (crime, 311 service requests, landbank sales,
|
|
43
|
+
lien orders, foreclosures, food-service establishments, ABC licenses, restaurant inspections,
|
|
44
|
+
construction permits, short-term-rental licenses) plus the geography spine (ZIP boundaries, council
|
|
45
|
+
districts, neighborhoods, HUD zones, crosswalks, Census population) — ~3.5M rows, all ingested from
|
|
46
|
+
source, all lineage-documented.
|
|
47
|
+
|
|
48
|
+
## Quick start (60 seconds, no database required)
|
|
49
|
+
|
|
50
|
+
```bash
|
|
51
|
+
pip install civic-data # consumer install — DuckDB only, no Postgres
|
|
52
|
+
civic-data cities # list published city lakes
|
|
53
|
+
civic-data get louisville # download the lake (sha256-verified) into ./export
|
|
54
|
+
civic-data query -d crime --zip 40202 --limit 5
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
## Consuming the data
|
|
58
|
+
|
|
59
|
+
The consumer product is a **GeoParquet lake** described by a machine-readable `catalog.json`, read
|
|
60
|
+
with DuckDB — no server, no Docker, no Postgres. Every verb takes `--json` (for agents) and
|
|
61
|
+
`--lake <dir>` (default: `./export`):
|
|
62
|
+
|
|
63
|
+
```bash
|
|
64
|
+
civic-data catalog --json # the discoverability contract — start here
|
|
65
|
+
civic-data datasets # list datasets by catalog key
|
|
66
|
+
civic-data schema crime # one dataset's columns + shape
|
|
67
|
+
civic-data query -d crime --zip 40202 --year 2024 --limit 5
|
|
68
|
+
civic-data query -d crime --district 6 # point-in-polygon; geocoded rows only (it says so)
|
|
69
|
+
civic-data place 40202 # one place across every dataset: identity + counts
|
|
70
|
+
civic-data sql "SELECT zip_code, count(*) FROM crime GROUP BY 1" # raw DuckDB; keys are views
|
|
71
|
+
civic-data query -d crime --sources # provenance: source pulls, license, how-to-cite
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
**Hand someone a place as a folder:** `civic-data place 6 --export` writes a self-contained
|
|
75
|
+
mini-lake (filtered Parquet + the district's boundary/crosswalks/population + a scoped catalog +
|
|
76
|
+
generated `llms.txt`/`AGENTS.md`). Every verb above works against that folder via `--lake`.
|
|
77
|
+
|
|
78
|
+
The lake root also carries generated **`llms.txt`** and **`AGENTS.md`** so an AI agent can bootstrap
|
|
79
|
+
itself from the artifact alone. Both are regenerated from the catalog on every export — never
|
|
80
|
+
hand-edited (a test enforces it).
|
|
81
|
+
|
|
82
|
+
## Producing the data (the reproduction path)
|
|
83
|
+
|
|
84
|
+
```bash
|
|
85
|
+
docker compose up -d # PostgreSQL 16 + PostGIS on host port 5434
|
|
86
|
+
python -m venv .venv && . .venv/Scripts/activate # (Windows: .venv\Scripts\activate)
|
|
87
|
+
pip install -e .[producer] # the pipeline extras (Postgres driver, HTTP client)
|
|
88
|
+
|
|
89
|
+
civic-data ingest-zips # canonical ZIP boundary set (MDM) — run first
|
|
90
|
+
civic-data ingest-refs # LOJIC geocoding references (address points + intersections)
|
|
91
|
+
civic-data ingest-geo && civic-data ingest-population # boundaries, crosswalks, Census ACS
|
|
92
|
+
civic-data ingest crime --all --promote # source -> bronze -> silver (any of the 10 domains)
|
|
93
|
+
civic-data geocode-crime --write --apply # place ~1.2M crime records (geom + derived ZIP)
|
|
94
|
+
civic-data datadict --with-source-audit # generate the data dictionary + lineage
|
|
95
|
+
civic-data export # silver -> GeoParquet lake + catalog + agent docs
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
The generated dictionary lands in [`docs/data-dictionary/`](docs/data-dictionary/) — a human-readable
|
|
99
|
+
`README.md` and a machine-readable `data-reality.json` (diffing successive runs is drift detection).
|
|
100
|
+
|
|
101
|
+
Ingest order matters: `ingest-zips` first. The ZIP set is the canonical geography (MDM) every record
|
|
102
|
+
validates against; crime/311 ingest refuse to run without it.
|
|
103
|
+
|
|
104
|
+
## Honesty, by construction
|
|
105
|
+
|
|
106
|
+
- **Trust tiers** on every dataset (`trendable` / `viewable_caveated` / `snapshot_only`) — a current
|
|
107
|
+
roster is not a time series, and the catalog says which is which.
|
|
108
|
+
- **Documented loss, not silent loss:** quarantine counts, geocode coverage (spatial filters match
|
|
109
|
+
geocoded rows only, and the CLI says so), source-shape divergences recorded as findings.
|
|
110
|
+
- **Generated, never hand-maintained:** the dictionary, the catalog, and the agent docs are all
|
|
111
|
+
derived from the live database + code; drift guards raise when authored prose falls out of sync.
|
|
112
|
+
|
|
113
|
+
## Licensing
|
|
114
|
+
|
|
115
|
+
- **Code:** MIT — see [`LICENSE`](LICENSE).
|
|
116
|
+
- **Data & documentation:** CC BY 4.0 — see [`LICENSE-data`](LICENSE-data).
|
|
117
|
+
|
|
118
|
+
The underlying raw data is **public record, provided as-is**, and its terms were verified per origin
|
|
119
|
+
(2026-07-11): Louisville Metro publishes under the **PDDL** (public domain; when republishing
|
|
120
|
+
modified data, identify the source, version, and modifications — which the generated lineage layer
|
|
121
|
+
does per dataset); LOJIC layers are public domain (*Mapping Data Source: LOJIC*); Census ACS is a
|
|
122
|
+
U.S. government work — *this product uses the Census Bureau Data API but is not endorsed or
|
|
123
|
+
certified by the Census Bureau.* Every dataset's exact license text ships in its catalog provenance:
|
|
124
|
+
`civic-data query -d <dataset> --sources`.
|
|
125
|
+
|
|
126
|
+
## Provenance
|
|
127
|
+
|
|
128
|
+
Born out of civic-graph (the private flagship built on top of this layer, and the origin of this
|
|
129
|
+
spine). See `wiki.md` for the source of truth and `ai-working/plans/` for the phase plans.
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
# civic-data
|
|
2
|
+
|
|
3
|
+
**Cleaned, normalized, and documented public open data — with a generated, code-coupled data
|
|
4
|
+
dictionary and transformation-lineage layer, consumable as a GeoParquet lake with a self-describing
|
|
5
|
+
CLI.**
|
|
6
|
+
|
|
7
|
+
The city gives away raw open data. civic-data's value is not the data — it's the honest,
|
|
8
|
+
reproducible record of **what we did to it, why, and what's lost**: schema, grain, NULL rates,
|
|
9
|
+
per-year coverage, quarantine loss, bias caveats, and a step-by-step transformation lineage that
|
|
10
|
+
doubles as the reproduction recipe. Because that record is *generated from the live database and the
|
|
11
|
+
ingest code*, it cannot drift from reality.
|
|
12
|
+
|
|
13
|
+
v1 covers **Louisville, KY**: ten domain datasets (crime, 311 service requests, landbank sales,
|
|
14
|
+
lien orders, foreclosures, food-service establishments, ABC licenses, restaurant inspections,
|
|
15
|
+
construction permits, short-term-rental licenses) plus the geography spine (ZIP boundaries, council
|
|
16
|
+
districts, neighborhoods, HUD zones, crosswalks, Census population) — ~3.5M rows, all ingested from
|
|
17
|
+
source, all lineage-documented.
|
|
18
|
+
|
|
19
|
+
## Quick start (60 seconds, no database required)
|
|
20
|
+
|
|
21
|
+
```bash
|
|
22
|
+
pip install civic-data # consumer install — DuckDB only, no Postgres
|
|
23
|
+
civic-data cities # list published city lakes
|
|
24
|
+
civic-data get louisville # download the lake (sha256-verified) into ./export
|
|
25
|
+
civic-data query -d crime --zip 40202 --limit 5
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
## Consuming the data
|
|
29
|
+
|
|
30
|
+
The consumer product is a **GeoParquet lake** described by a machine-readable `catalog.json`, read
|
|
31
|
+
with DuckDB — no server, no Docker, no Postgres. Every verb takes `--json` (for agents) and
|
|
32
|
+
`--lake <dir>` (default: `./export`):
|
|
33
|
+
|
|
34
|
+
```bash
|
|
35
|
+
civic-data catalog --json # the discoverability contract — start here
|
|
36
|
+
civic-data datasets # list datasets by catalog key
|
|
37
|
+
civic-data schema crime # one dataset's columns + shape
|
|
38
|
+
civic-data query -d crime --zip 40202 --year 2024 --limit 5
|
|
39
|
+
civic-data query -d crime --district 6 # point-in-polygon; geocoded rows only (it says so)
|
|
40
|
+
civic-data place 40202 # one place across every dataset: identity + counts
|
|
41
|
+
civic-data sql "SELECT zip_code, count(*) FROM crime GROUP BY 1" # raw DuckDB; keys are views
|
|
42
|
+
civic-data query -d crime --sources # provenance: source pulls, license, how-to-cite
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
**Hand someone a place as a folder:** `civic-data place 6 --export` writes a self-contained
|
|
46
|
+
mini-lake (filtered Parquet + the district's boundary/crosswalks/population + a scoped catalog +
|
|
47
|
+
generated `llms.txt`/`AGENTS.md`). Every verb above works against that folder via `--lake`.
|
|
48
|
+
|
|
49
|
+
The lake root also carries generated **`llms.txt`** and **`AGENTS.md`** so an AI agent can bootstrap
|
|
50
|
+
itself from the artifact alone. Both are regenerated from the catalog on every export — never
|
|
51
|
+
hand-edited (a test enforces it).
|
|
52
|
+
|
|
53
|
+
## Producing the data (the reproduction path)
|
|
54
|
+
|
|
55
|
+
```bash
|
|
56
|
+
docker compose up -d # PostgreSQL 16 + PostGIS on host port 5434
|
|
57
|
+
python -m venv .venv && . .venv/Scripts/activate # (Windows: .venv\Scripts\activate)
|
|
58
|
+
pip install -e .[producer] # the pipeline extras (Postgres driver, HTTP client)
|
|
59
|
+
|
|
60
|
+
civic-data ingest-zips # canonical ZIP boundary set (MDM) — run first
|
|
61
|
+
civic-data ingest-refs # LOJIC geocoding references (address points + intersections)
|
|
62
|
+
civic-data ingest-geo && civic-data ingest-population # boundaries, crosswalks, Census ACS
|
|
63
|
+
civic-data ingest crime --all --promote # source -> bronze -> silver (any of the 10 domains)
|
|
64
|
+
civic-data geocode-crime --write --apply # place ~1.2M crime records (geom + derived ZIP)
|
|
65
|
+
civic-data datadict --with-source-audit # generate the data dictionary + lineage
|
|
66
|
+
civic-data export # silver -> GeoParquet lake + catalog + agent docs
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
The generated dictionary lands in [`docs/data-dictionary/`](docs/data-dictionary/) — a human-readable
|
|
70
|
+
`README.md` and a machine-readable `data-reality.json` (diffing successive runs is drift detection).
|
|
71
|
+
|
|
72
|
+
Ingest order matters: `ingest-zips` first. The ZIP set is the canonical geography (MDM) every record
|
|
73
|
+
validates against; crime/311 ingest refuse to run without it.
|
|
74
|
+
|
|
75
|
+
## Honesty, by construction
|
|
76
|
+
|
|
77
|
+
- **Trust tiers** on every dataset (`trendable` / `viewable_caveated` / `snapshot_only`) — a current
|
|
78
|
+
roster is not a time series, and the catalog says which is which.
|
|
79
|
+
- **Documented loss, not silent loss:** quarantine counts, geocode coverage (spatial filters match
|
|
80
|
+
geocoded rows only, and the CLI says so), source-shape divergences recorded as findings.
|
|
81
|
+
- **Generated, never hand-maintained:** the dictionary, the catalog, and the agent docs are all
|
|
82
|
+
derived from the live database + code; drift guards raise when authored prose falls out of sync.
|
|
83
|
+
|
|
84
|
+
## Licensing
|
|
85
|
+
|
|
86
|
+
- **Code:** MIT — see [`LICENSE`](LICENSE).
|
|
87
|
+
- **Data & documentation:** CC BY 4.0 — see [`LICENSE-data`](LICENSE-data).
|
|
88
|
+
|
|
89
|
+
The underlying raw data is **public record, provided as-is**, and its terms were verified per origin
|
|
90
|
+
(2026-07-11): Louisville Metro publishes under the **PDDL** (public domain; when republishing
|
|
91
|
+
modified data, identify the source, version, and modifications — which the generated lineage layer
|
|
92
|
+
does per dataset); LOJIC layers are public domain (*Mapping Data Source: LOJIC*); Census ACS is a
|
|
93
|
+
U.S. government work — *this product uses the Census Bureau Data API but is not endorsed or
|
|
94
|
+
certified by the Census Bureau.* Every dataset's exact license text ships in its catalog provenance:
|
|
95
|
+
`civic-data query -d <dataset> --sources`.
|
|
96
|
+
|
|
97
|
+
## Provenance
|
|
98
|
+
|
|
99
|
+
Born out of civic-graph (the private flagship built on top of this layer, and the origin of this
|
|
100
|
+
spine). See `wiki.md` for the source of truth and `ai-working/plans/` for the phase plans.
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
"""civic-data — the extraction / normalization / documentation spine.
|
|
2
|
+
|
|
3
|
+
One canonical home for the data-layer core: source adapters, medallion ingest
|
|
4
|
+
(bronze → silver), and a generated data dictionary with a transformation-lineage
|
|
5
|
+
layer. The product's value is not the data (the city gives that away raw) — it is
|
|
6
|
+
the *documented, reproducible transformation lineage*: what we did, why, and what
|
|
7
|
+
is lost. That lineage is generated from the live DB + code, never hand-maintained,
|
|
8
|
+
so it cannot drift from reality.
|
|
9
|
+
|
|
10
|
+
civic-graph (the private flagship) was the origin of this code and is now a
|
|
11
|
+
downstream consumer / worked example built on top.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
__version__ = "0.1.0"
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
"""Source adapters — the extraction spine.
|
|
2
|
+
|
|
3
|
+
`arcgis` is the working portal adapter (Louisville / LOJIC, and any ArcGIS Hub
|
|
4
|
+
portal). `socrata` is a dormant placeholder that activates at city #2
|
|
5
|
+
(Cincinnati / Chicago). `validation` is the ZIP/geography gatekeeper shared by
|
|
6
|
+
every ingest path. `base` is the adapter contract.
|
|
7
|
+
"""
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
"""ArcGIS Feature Service adapter.
|
|
2
|
+
|
|
3
|
+
Louisville's open data portal runs on ArcGIS Hub. Most datasets are
|
|
4
|
+
exposed as ArcGIS Feature Services with a REST query endpoint.
|
|
5
|
+
|
|
6
|
+
Feature Service query pattern:
|
|
7
|
+
GET {service_url}/query?where=1=1&outFields=*&f=json&resultOffset=0&resultRecordCount=2000
|
|
8
|
+
|
|
9
|
+
Pagination: resultOffset increments by resultRecordCount until
|
|
10
|
+
exceededTransferLimit is false.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
import time
|
|
14
|
+
import httpx
|
|
15
|
+
from typing import Any
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def feature_count(service_url: str, timeout: float = 30.0) -> int | None:
|
|
19
|
+
"""Source row count via ArcGIS returnCountOnly (one tiny call). None on error.
|
|
20
|
+
|
|
21
|
+
Captured at ingest as source-pull metadata (the reproducibility substrate): the
|
|
22
|
+
source_rowcount at pull time, to compare against what we actually fetched/kept.
|
|
23
|
+
"""
|
|
24
|
+
try:
|
|
25
|
+
resp = httpx.get(f"{service_url}/query",
|
|
26
|
+
params={"where": "1=1", "returnCountOnly": "true", "f": "json"},
|
|
27
|
+
timeout=timeout)
|
|
28
|
+
resp.raise_for_status()
|
|
29
|
+
return resp.json().get("count")
|
|
30
|
+
except Exception:
|
|
31
|
+
return None
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _query_page(client, url, params, max_retries: int = 4):
|
|
35
|
+
"""Fetch one page, retrying transient failures. Returns parsed JSON with a 'features' key.
|
|
36
|
+
|
|
37
|
+
Raises RuntimeError if the server never returns a valid page — so a transient error
|
|
38
|
+
(HTTP 5xx, an ArcGIS error payload, a truncated body) can NEVER be mistaken for
|
|
39
|
+
end-of-data and silently truncate a large pull. Only a *valid* response with an empty
|
|
40
|
+
'features' array or exceededTransferLimit=False ends pagination (handled by the caller).
|
|
41
|
+
"""
|
|
42
|
+
last_err = None
|
|
43
|
+
for attempt in range(max_retries):
|
|
44
|
+
try:
|
|
45
|
+
resp = client.get(url, params=params)
|
|
46
|
+
resp.raise_for_status()
|
|
47
|
+
data = resp.json()
|
|
48
|
+
if "error" in data:
|
|
49
|
+
last_err = f"ArcGIS error payload: {data['error']}"
|
|
50
|
+
elif "features" not in data:
|
|
51
|
+
last_err = f"response missing 'features' (keys={list(data)[:6]})"
|
|
52
|
+
else:
|
|
53
|
+
return data
|
|
54
|
+
except (httpx.HTTPError, ValueError) as e: # ValueError also covers JSON decode errors
|
|
55
|
+
last_err = repr(e)
|
|
56
|
+
time.sleep(1.0 * (attempt + 1)) # linear backoff between retries
|
|
57
|
+
raise RuntimeError(
|
|
58
|
+
f"ArcGIS page failed after {max_retries} attempts "
|
|
59
|
+
f"(resultOffset={params.get('resultOffset')}): {last_err}")
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def fetch_feature_service(
|
|
63
|
+
service_url: str,
|
|
64
|
+
where: str = "1=1",
|
|
65
|
+
out_fields: str = "*",
|
|
66
|
+
page_size: int = 2000,
|
|
67
|
+
max_records: int | None = None,
|
|
68
|
+
delay: float = 0.5,
|
|
69
|
+
) -> list[dict[str, Any]]:
|
|
70
|
+
"""Fetch all records from an ArcGIS Feature Service.
|
|
71
|
+
|
|
72
|
+
Args:
|
|
73
|
+
service_url: The Feature Service layer URL
|
|
74
|
+
(e.g., https://services1.arcgis.com/.../FeatureServer/0)
|
|
75
|
+
where: SQL WHERE clause for filtering
|
|
76
|
+
out_fields: Comma-separated field names or "*" for all
|
|
77
|
+
page_size: Records per request (max varies by service, 2000 is safe)
|
|
78
|
+
max_records: Optional cap on total records fetched
|
|
79
|
+
delay: Seconds between paginated requests (polite scraping)
|
|
80
|
+
|
|
81
|
+
Returns:
|
|
82
|
+
List of feature attribute dicts (geometry stripped to lat/lng if present)
|
|
83
|
+
|
|
84
|
+
Pagination terminates only on a *valid* empty page or exceededTransferLimit=False;
|
|
85
|
+
transient per-page failures are retried (see _query_page), never treated as done.
|
|
86
|
+
"""
|
|
87
|
+
all_records = []
|
|
88
|
+
offset = 0
|
|
89
|
+
url = f"{service_url}/query"
|
|
90
|
+
|
|
91
|
+
with httpx.Client(timeout=60.0) as client:
|
|
92
|
+
while True:
|
|
93
|
+
params = {
|
|
94
|
+
"where": where,
|
|
95
|
+
"outFields": out_fields,
|
|
96
|
+
"f": "json",
|
|
97
|
+
"resultOffset": offset,
|
|
98
|
+
"resultRecordCount": page_size,
|
|
99
|
+
"outSR": 4326, # WGS84
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
data = _query_page(client, url, params)
|
|
103
|
+
features = data.get("features", [])
|
|
104
|
+
if not features:
|
|
105
|
+
break
|
|
106
|
+
|
|
107
|
+
for f in features:
|
|
108
|
+
record = f.get("attributes", {})
|
|
109
|
+
# Extract geometry as flat lat/lng if present
|
|
110
|
+
geom = f.get("geometry")
|
|
111
|
+
if geom:
|
|
112
|
+
record["_longitude"] = geom.get("x")
|
|
113
|
+
record["_latitude"] = geom.get("y")
|
|
114
|
+
all_records.append(record)
|
|
115
|
+
|
|
116
|
+
if max_records and len(all_records) >= max_records:
|
|
117
|
+
all_records = all_records[:max_records]
|
|
118
|
+
break
|
|
119
|
+
|
|
120
|
+
if not data.get("exceededTransferLimit", False):
|
|
121
|
+
break
|
|
122
|
+
|
|
123
|
+
# Increment by actual records returned, not page_size.
|
|
124
|
+
# Some services cap below our page_size (e.g., 1000 max).
|
|
125
|
+
offset += len(features)
|
|
126
|
+
time.sleep(delay)
|
|
127
|
+
|
|
128
|
+
return all_records
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
"""Base adapter interface for civic data sources."""
|
|
2
|
+
|
|
3
|
+
from abc import ABC, abstractmethod
|
|
4
|
+
from dataclasses import dataclass
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
@dataclass
|
|
9
|
+
class IngestResult:
|
|
10
|
+
"""Result of a data ingestion run."""
|
|
11
|
+
adapter: str
|
|
12
|
+
source_dataset: str
|
|
13
|
+
record_count: int
|
|
14
|
+
status: str # "ok" or "error"
|
|
15
|
+
error_message: str | None = None
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class CivicDataAdapter(ABC):
|
|
19
|
+
"""Abstract base class for all civic data adapters.
|
|
20
|
+
|
|
21
|
+
Each adapter handles one data domain (crime, 311, property, etc.)
|
|
22
|
+
and knows how to:
|
|
23
|
+
1. Fetch raw data from the source API
|
|
24
|
+
2. Store raw records in bronze layer (JSONB, append-only)
|
|
25
|
+
3. Transform raw records into silver layer (typed, geocoded)
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
@property
|
|
29
|
+
@abstractmethod
|
|
30
|
+
def adapter_name(self) -> str:
|
|
31
|
+
"""Unique identifier for this adapter (e.g., 'crime', 'service_requests')."""
|
|
32
|
+
...
|
|
33
|
+
|
|
34
|
+
@abstractmethod
|
|
35
|
+
def fetch_raw(self, dataset_id: str, **kwargs) -> list[dict[str, Any]]:
|
|
36
|
+
"""Fetch raw records from the source API.
|
|
37
|
+
|
|
38
|
+
Returns a list of raw JSON-serializable dicts.
|
|
39
|
+
"""
|
|
40
|
+
...
|
|
41
|
+
|
|
42
|
+
@abstractmethod
|
|
43
|
+
def load_bronze(self, records: list[dict[str, Any]], dataset_id: str) -> int:
|
|
44
|
+
"""Store raw records in the bronze layer. Returns count of rows inserted."""
|
|
45
|
+
...
|
|
46
|
+
|
|
47
|
+
@abstractmethod
|
|
48
|
+
def promote_to_silver(self, dataset_id: str) -> int:
|
|
49
|
+
"""Transform bronze records into silver layer. Returns count of rows promoted."""
|
|
50
|
+
...
|
|
51
|
+
|
|
52
|
+
def ingest(self, dataset_id: str, **kwargs) -> IngestResult:
|
|
53
|
+
"""Full ingestion pipeline: fetch → bronze → silver."""
|
|
54
|
+
try:
|
|
55
|
+
raw = self.fetch_raw(dataset_id, **kwargs)
|
|
56
|
+
bronze_count = self.load_bronze(raw, dataset_id)
|
|
57
|
+
silver_count = self.promote_to_silver(dataset_id)
|
|
58
|
+
return IngestResult(
|
|
59
|
+
adapter=self.adapter_name,
|
|
60
|
+
source_dataset=dataset_id,
|
|
61
|
+
record_count=silver_count,
|
|
62
|
+
status="ok",
|
|
63
|
+
)
|
|
64
|
+
except Exception as e:
|
|
65
|
+
return IngestResult(
|
|
66
|
+
adapter=self.adapter_name,
|
|
67
|
+
source_dataset=dataset_id,
|
|
68
|
+
record_count=0,
|
|
69
|
+
status="error",
|
|
70
|
+
error_message=str(e),
|
|
71
|
+
)
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
"""Socrata Open Data adapter — DORMANT (activates at city #2).
|
|
2
|
+
|
|
3
|
+
STATUS: placeholder, not implemented. v1 is Louisville-only (crime + 311) on
|
|
4
|
+
ArcGIS Hub; see `adapters/arcgis.py` for the working pattern. Socrata is the
|
|
5
|
+
multi-city unlock — Cincinnati, Chicago, and ~100 other US cities publish on the
|
|
6
|
+
Socrata platform (SODA API). When city #2 lands, implement `fetch_dataset`
|
|
7
|
+
against the SODA endpoint, mirroring the shape `arcgis.fetch_feature_service`
|
|
8
|
+
returns (a list of flat attribute dicts) so the downstream ingest/validation
|
|
9
|
+
spine is unchanged.
|
|
10
|
+
|
|
11
|
+
Design note (from the kickoff): this file was listed in the seed manifest as
|
|
12
|
+
inherited from civic-graph, but civic-graph never actually authored a Socrata
|
|
13
|
+
adapter — only the ArcGIS one exists there. So this is a fresh, honest stub, not
|
|
14
|
+
a lift. Multi-city is the thesis, not the v1 scope; do not build it out until a
|
|
15
|
+
real second city is in scope (guardrail: no future-proofing).
|
|
16
|
+
|
|
17
|
+
SODA query pattern (for reference when implemented):
|
|
18
|
+
GET https://{domain}/resource/{dataset_id}.json?$limit=50000&$offset=0
|
|
19
|
+
Pagination via $limit/$offset; optional $$app_token header to raise rate limits.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def fetch_dataset(*args, **kwargs):
|
|
24
|
+
raise NotImplementedError(
|
|
25
|
+
"The Socrata adapter is dormant. v1 is Louisville-only (ArcGIS). "
|
|
26
|
+
"Implement this against the SODA API when city #2 (Cincinnati/Chicago) is in scope, "
|
|
27
|
+
"returning the same flat-dict shape as adapters.arcgis.fetch_feature_service."
|
|
28
|
+
)
|