spacescans 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- spacescans-0.3.0/LICENSE +21 -0
- spacescans-0.3.0/PKG-INFO +439 -0
- spacescans-0.3.0/README.md +379 -0
- spacescans-0.3.0/pyproject.toml +113 -0
- spacescans-0.3.0/setup.cfg +4 -0
- spacescans-0.3.0/src/spacescans/__init__.py +42 -0
- spacescans-0.3.0/src/spacescans/__main__.py +7 -0
- spacescans-0.3.0/src/spacescans/_capabilities.py +233 -0
- spacescans-0.3.0/src/spacescans/_extras.py +65 -0
- spacescans-0.3.0/src/spacescans/bridge/__init__.py +19 -0
- spacescans-0.3.0/src/spacescans/bridge/__main__.py +7 -0
- spacescans-0.3.0/src/spacescans/bridge/args.py +135 -0
- spacescans-0.3.0/src/spacescans/bridge/cmd_catalog.py +274 -0
- spacescans-0.3.0/src/spacescans/bridge/cmd_data.py +159 -0
- spacescans-0.3.0/src/spacescans/bridge/cmd_datasets.py +149 -0
- spacescans-0.3.0/src/spacescans/bridge/cmd_link.py +503 -0
- spacescans-0.3.0/src/spacescans/bridge/cohorts.py +58 -0
- spacescans-0.3.0/src/spacescans/bridge/commands.py +375 -0
- spacescans-0.3.0/src/spacescans/bridge/configs.py +180 -0
- spacescans-0.3.0/src/spacescans/bridge/errors.py +258 -0
- spacescans-0.3.0/src/spacescans/bridge/events.py +129 -0
- spacescans-0.3.0/src/spacescans/bridge/main.py +251 -0
- spacescans-0.3.0/src/spacescans/bridge/protocol.py +153 -0
- spacescans-0.3.0/src/spacescans/bridge/worker.py +186 -0
- spacescans-0.3.0/src/spacescans/catalog/__init__.py +87 -0
- spacescans-0.3.0/src/spacescans/catalog/_coverage.py +36 -0
- spacescans-0.3.0/src/spacescans/catalog/_features.py +79 -0
- spacescans-0.3.0/src/spacescans/catalog/_registry.py +313 -0
- spacescans-0.3.0/src/spacescans/catalog/_templates.py +84 -0
- spacescans-0.3.0/src/spacescans/catalog/features.json +237 -0
- spacescans-0.3.0/src/spacescans/catalog/recipes.py +675 -0
- spacescans-0.3.0/src/spacescans/catalog/templates/c3/acag_grid_demo.yaml +19 -0
- spacescans-0.3.0/src/spacescans/catalog/templates/c3/bg_us_2020_demo.yaml +75 -0
- spacescans-0.3.0/src/spacescans/catalog/templates/c3/bg_us_demo.yaml +68 -0
- spacescans-0.3.0/src/spacescans/catalog/templates/c3/county_us_demo.yaml +21 -0
- spacescans-0.3.0/src/spacescans/catalog/templates/c3/custom_grid.yaml +31 -0
- spacescans-0.3.0/src/spacescans/catalog/templates/c3/nhd_demo.yaml +18 -0
- spacescans-0.3.0/src/spacescans/catalog/templates/c3/noise_grid_demo.yaml +18 -0
- spacescans-0.3.0/src/spacescans/catalog/templates/c3/temis_grid_demo.yaml +30 -0
- spacescans-0.3.0/src/spacescans/catalog/templates/c3/tiger_roads_demo.yaml +24 -0
- spacescans-0.3.0/src/spacescans/catalog/templates/c3/tract_us_demo.yaml +68 -0
- spacescans-0.3.0/src/spacescans/catalog/templates/c3/vnl_grid_demo.yaml +20 -0
- spacescans-0.3.0/src/spacescans/catalog/templates/c3/zcta5_us_demo.yaml +21 -0
- spacescans-0.3.0/src/spacescans/catalog/templates/c4/acag_demo.yaml +32 -0
- spacescans-0.3.0/src/spacescans/catalog/templates/c4/bg_ndi_demo.yaml +44 -0
- spacescans-0.3.0/src/spacescans/catalog/templates/c4/bg_wi_demo.yaml +39 -0
- spacescans-0.3.0/src/spacescans/catalog/templates/c4/county_cbp_demo.yaml +51 -0
- spacescans-0.3.0/src/spacescans/catalog/templates/c4/custom_areal.yaml +41 -0
- spacescans-0.3.0/src/spacescans/catalog/templates/c4/nhd_bluespace_demo.yaml +38 -0
- spacescans-0.3.0/src/spacescans/catalog/templates/c4/noise_demo.yaml +34 -0
- spacescans-0.3.0/src/spacescans/catalog/templates/c4/temis_demo.yaml +37 -0
- spacescans-0.3.0/src/spacescans/catalog/templates/c4/tiger_roads_demo.yaml +34 -0
- spacescans-0.3.0/src/spacescans/catalog/templates/c4/tract_faqsd_demo.yaml +33 -0
- spacescans-0.3.0/src/spacescans/catalog/templates/c4/tract_fara_demo.yaml +34 -0
- spacescans-0.3.0/src/spacescans/catalog/templates/c4/vnl_demo.yaml +33 -0
- spacescans-0.3.0/src/spacescans/catalog/templates/c4/zcta5_cbp_demo.yaml +50 -0
- spacescans-0.3.0/src/spacescans/catalog/variables.json +264 -0
- spacescans-0.3.0/src/spacescans/catalog/variables.schema.json +73 -0
- spacescans-0.3.0/src/spacescans/cli/__init__.py +51 -0
- spacescans-0.3.0/src/spacescans/cli/__main__.py +3 -0
- spacescans-0.3.0/src/spacescans/cli/_init_config.py +34 -0
- spacescans-0.3.0/src/spacescans/cli/_quickstart.py +88 -0
- spacescans-0.3.0/src/spacescans/cli/_run.py +25 -0
- spacescans-0.3.0/src/spacescans/cohort.py +683 -0
- spacescans-0.3.0/src/spacescans/config_resolution.py +132 -0
- spacescans-0.3.0/src/spacescans/custom/__init__.py +182 -0
- spacescans-0.3.0/src/spacescans/custom/boundaries.py +60 -0
- spacescans-0.3.0/src/spacescans/custom/dated.py +172 -0
- spacescans-0.3.0/src/spacescans/custom/definition.py +701 -0
- spacescans-0.3.0/src/spacescans/custom/diagnose.py +187 -0
- spacescans-0.3.0/src/spacescans/custom/errors.py +29 -0
- spacescans-0.3.0/src/spacescans/custom/features.py +480 -0
- spacescans-0.3.0/src/spacescans/custom/keys.py +56 -0
- spacescans-0.3.0/src/spacescans/custom/periods.py +326 -0
- spacescans-0.3.0/src/spacescans/custom/prepare.py +239 -0
- spacescans-0.3.0/src/spacescans/custom/prepare_raster.py +221 -0
- spacescans-0.3.0/src/spacescans/custom/prepare_table.py +436 -0
- spacescans-0.3.0/src/spacescans/custom/raster.py +252 -0
- spacescans-0.3.0/src/spacescans/custom/registry.py +867 -0
- spacescans-0.3.0/src/spacescans/custom/safety.py +71 -0
- spacescans-0.3.0/src/spacescans/custom/steps.py +396 -0
- spacescans-0.3.0/src/spacescans/custom/table.py +264 -0
- spacescans-0.3.0/src/spacescans/custom/units.py +81 -0
- spacescans-0.3.0/src/spacescans/custom/values.py +82 -0
- spacescans-0.3.0/src/spacescans/data/__init__.py +160 -0
- spacescans-0.3.0/src/spacescans/data/_adopt.py +158 -0
- spacescans-0.3.0/src/spacescans/data/_fetch.py +1033 -0
- spacescans-0.3.0/src/spacescans/data/_import.py +220 -0
- spacescans-0.3.0/src/spacescans/data/_mirror.py +291 -0
- spacescans-0.3.0/src/spacescans/data/_prepare.py +196 -0
- spacescans-0.3.0/src/spacescans/data/_providers.py +288 -0
- spacescans-0.3.0/src/spacescans/data/_requirements.py +130 -0
- spacescans-0.3.0/src/spacescans/data/_status.py +507 -0
- spacescans-0.3.0/src/spacescans/data/_verify.py +90 -0
- spacescans-0.3.0/src/spacescans/data/cli.py +455 -0
- spacescans-0.3.0/src/spacescans/data/errors.py +91 -0
- spacescans-0.3.0/src/spacescans/data/extract.py +264 -0
- spacescans-0.3.0/src/spacescans/data/layout.py +265 -0
- spacescans-0.3.0/src/spacescans/data/lock.py +99 -0
- spacescans-0.3.0/src/spacescans/data/manifest.json +546 -0
- spacescans-0.3.0/src/spacescans/data/manifest.py +579 -0
- spacescans-0.3.0/src/spacescans/data/manifest.schema.json +328 -0
- spacescans-0.3.0/src/spacescans/data/paths.py +76 -0
- spacescans-0.3.0/src/spacescans/data/receipts.py +107 -0
- spacescans-0.3.0/src/spacescans/data/sources.py +162 -0
- spacescans-0.3.0/src/spacescans/data/transport.py +202 -0
- spacescans-0.3.0/src/spacescans/engine/__init__.py +0 -0
- spacescans-0.3.0/src/spacescans/engine/duckdb_engine.py +228 -0
- spacescans-0.3.0/src/spacescans/engine/utils.py +18 -0
- spacescans-0.3.0/src/spacescans/geometry/__init__.py +0 -0
- spacescans-0.3.0/src/spacescans/geometry/buffers.py +45 -0
- spacescans-0.3.0/src/spacescans/geometry/grid_weights.py +186 -0
- spacescans-0.3.0/src/spacescans/geometry/overlap.py +547 -0
- spacescans-0.3.0/src/spacescans/geometry/proximity.py +73 -0
- spacescans-0.3.0/src/spacescans/io/__init__.py +0 -0
- spacescans-0.3.0/src/spacescans/io/readers.py +51 -0
- spacescans-0.3.0/src/spacescans/io/spatial_readers.py +84 -0
- spacescans-0.3.0/src/spacescans/io/writers.py +44 -0
- spacescans-0.3.0/src/spacescans/linkage/__init__.py +0 -0
- spacescans-0.3.0/src/spacescans/linkage/acag_linkage.py +127 -0
- spacescans-0.3.0/src/spacescans/linkage/boundary_overlap_fast_linkage.py +59 -0
- spacescans-0.3.0/src/spacescans/linkage/boundary_overlap_linkage.py +56 -0
- spacescans-0.3.0/src/spacescans/linkage/cbp_fallback_linkage.py +161 -0
- spacescans-0.3.0/src/spacescans/linkage/faqsd_linkage.py +39 -0
- spacescans-0.3.0/src/spacescans/linkage/fara_linkage.py +345 -0
- spacescans-0.3.0/src/spacescans/linkage/grid_weights_linkage.py +57 -0
- spacescans-0.3.0/src/spacescans/linkage/gridded_linkage.py +102 -0
- spacescans-0.3.0/src/spacescans/linkage/helpers.py +203 -0
- spacescans-0.3.0/src/spacescans/linkage/nhd_proximity_linkage.py +458 -0
- spacescans-0.3.0/src/spacescans/linkage/precomputed_areal_linkage.py +137 -0
- spacescans-0.3.0/src/spacescans/linkage/precomputed_static_linkage.py +102 -0
- spacescans-0.3.0/src/spacescans/linkage/proximity_linkage.py +172 -0
- spacescans-0.3.0/src/spacescans/linkage/static_areal_linkage.py +56 -0
- spacescans-0.3.0/src/spacescans/linkage/tiger_proximity_linkage.py +286 -0
- spacescans-0.3.0/src/spacescans/linkage/yearly_areal_bg_vintage_linkage.py +158 -0
- spacescans-0.3.0/src/spacescans/linkage/yearly_areal_linkage.py +74 -0
- spacescans-0.3.0/src/spacescans/models/__init__.py +1 -0
- spacescans-0.3.0/src/spacescans/models/config.py +226 -0
- spacescans-0.3.0/src/spacescans/models/protocols.py +49 -0
- spacescans-0.3.0/src/spacescans/models/raster_meta.py +17 -0
- spacescans-0.3.0/src/spacescans/models/specs.py +71 -0
- spacescans-0.3.0/src/spacescans/orchestrate/__init__.py +158 -0
- spacescans-0.3.0/src/spacescans/orchestrate/cache.py +862 -0
- spacescans-0.3.0/src/spacescans/orchestrate/context.py +96 -0
- spacescans-0.3.0/src/spacescans/orchestrate/coverage.py +196 -0
- spacescans-0.3.0/src/spacescans/orchestrate/events.py +291 -0
- spacescans-0.3.0/src/spacescans/orchestrate/execute.py +621 -0
- spacescans-0.3.0/src/spacescans/orchestrate/link.py +840 -0
- spacescans-0.3.0/src/spacescans/orchestrate/locks.py +193 -0
- spacescans-0.3.0/src/spacescans/orchestrate/memory.py +126 -0
- spacescans-0.3.0/src/spacescans/orchestrate/memory_table.csv +33 -0
- spacescans-0.3.0/src/spacescans/orchestrate/merge.py +435 -0
- spacescans-0.3.0/src/spacescans/orchestrate/provenance.py +153 -0
- spacescans-0.3.0/src/spacescans/orchestrate/render.py +106 -0
- spacescans-0.3.0/src/spacescans/orchestrate/run.py +468 -0
- spacescans-0.3.0/src/spacescans/orchestrate/runs.py +446 -0
- spacescans-0.3.0/src/spacescans/pipeline/__init__.py +0 -0
- spacescans-0.3.0/src/spacescans/pipeline/loader.py +55 -0
- spacescans-0.3.0/src/spacescans/pipeline/registry.py +199 -0
- spacescans-0.3.0/src/spacescans/pipeline/runner.py +58 -0
- spacescans-0.3.0/src/spacescans/plugins/__init__.py +0 -0
- spacescans-0.3.0/src/spacescans/plugins/prep/__init__.py +0 -0
- spacescans-0.3.0/src/spacescans/plugins/prep/modis.py +6 -0
- spacescans-0.3.0/src/spacescans/plugins/prep/prism.py +6 -0
- spacescans-0.3.0/src/spacescans/plugins/prep/tiger.py +6 -0
- spacescans-0.3.0/src/spacescans/plugins/readers/__init__.py +0 -0
- spacescans-0.3.0/src/spacescans/plugins/readers/acag.py +137 -0
- spacescans-0.3.0/src/spacescans/plugins/readers/faqsd.py +352 -0
- spacescans-0.3.0/src/spacescans/plugins/readers/modis_ndvi.py +109 -0
- spacescans-0.3.0/src/spacescans/plugins/readers/modis_part1.py +94 -0
- spacescans-0.3.0/src/spacescans/plugins/readers/nhd.py +84 -0
- spacescans-0.3.0/src/spacescans/plugins/readers/noise.py +123 -0
- spacescans-0.3.0/src/spacescans/plugins/readers/prism.py +130 -0
- spacescans-0.3.0/src/spacescans/plugins/readers/prism_part1.py +214 -0
- spacescans-0.3.0/src/spacescans/plugins/readers/temis.py +277 -0
- spacescans-0.3.0/src/spacescans/plugins/readers/tiger_roads.py +87 -0
- spacescans-0.3.0/src/spacescans/plugins/readers/vnl.py +152 -0
- spacescans-0.3.0/src/spacescans/py.typed +0 -0
- spacescans-0.3.0/src/spacescans/resources/__init__.py +0 -0
- spacescans-0.3.0/src/spacescans/resources/configs/__init__.py +0 -0
- spacescans-0.3.0/src/spacescans/resources/configs/quickstart/__init__.py +0 -0
- spacescans-0.3.0/src/spacescans/resources/configs/quickstart/end_to_end.yaml +23 -0
- spacescans-0.3.0/src/spacescans/resources/configs/templates/__init__.py +0 -0
- spacescans-0.3.0/src/spacescans/resources/configs/templates/c3/__init__.py +0 -0
- spacescans-0.3.0/src/spacescans/resources/configs/templates/c3/county.yaml +20 -0
- spacescans-0.3.0/src/spacescans/resources/configs/templates/c3/zcta5.yaml +20 -0
- spacescans-0.3.0/src/spacescans/resources/configs/templates/c4/__init__.py +0 -0
- spacescans-0.3.0/src/spacescans/resources/configs/templates/c4/cbp_fallback.yaml +45 -0
- spacescans-0.3.0/src/spacescans/resources/configs/templates/c4/zbp.yaml +44 -0
- spacescans-0.3.0/src/spacescans/resources/data/__init__.py +0 -0
- spacescans-0.3.0/src/spacescans/resources/data/sample_counties.cpg +1 -0
- spacescans-0.3.0/src/spacescans/resources/data/sample_counties.dbf +0 -0
- spacescans-0.3.0/src/spacescans/resources/data/sample_counties.prj +1 -0
- spacescans-0.3.0/src/spacescans/resources/data/sample_counties.shp +0 -0
- spacescans-0.3.0/src/spacescans/resources/data/sample_counties.shx +0 -0
- spacescans-0.3.0/src/spacescans/resources/data/sample_patients.parquet +0 -0
- spacescans-0.3.0/src/spacescans/resources/data/sample_zcta5.cpg +1 -0
- spacescans-0.3.0/src/spacescans/resources/data/sample_zcta5.dbf +0 -0
- spacescans-0.3.0/src/spacescans/resources/data/sample_zcta5.prj +1 -0
- spacescans-0.3.0/src/spacescans/resources/data/sample_zcta5.shp +0 -0
- spacescans-0.3.0/src/spacescans/resources/data/sample_zcta5.shx +0 -0
- spacescans-0.3.0/src/spacescans/tools/__init__.py +0 -0
- spacescans-0.3.0/src/spacescans/tools/temis_convert.py +136 -0
- spacescans-0.3.0/src/spacescans/transforms/__init__.py +0 -0
- spacescans-0.3.0/src/spacescans/transforms/date_parse.py +39 -0
- spacescans-0.3.0/src/spacescans/transforms/derive.py +63 -0
- spacescans-0.3.0/src/spacescans/transforms/filter.py +17 -0
- spacescans-0.3.0/src/spacescans/transforms/recode.py +45 -0
- spacescans-0.3.0/src/spacescans/transforms/spatial_fix.py +75 -0
- spacescans-0.3.0/src/spacescans.egg-info/PKG-INFO +439 -0
- spacescans-0.3.0/src/spacescans.egg-info/SOURCES.txt +261 -0
- spacescans-0.3.0/src/spacescans.egg-info/dependency_links.txt +1 -0
- spacescans-0.3.0/src/spacescans.egg-info/entry_points.txt +2 -0
- spacescans-0.3.0/src/spacescans.egg-info/requires.txt +40 -0
- spacescans-0.3.0/src/spacescans.egg-info/top_level.txt +1 -0
- spacescans-0.3.0/tests/test_acag_linkage_grouping.py +103 -0
- spacescans-0.3.0/tests/test_acag_orientation.py +70 -0
- spacescans-0.3.0/tests/test_base_import_light.py +184 -0
- spacescans-0.3.0/tests/test_bridge.py +472 -0
- spacescans-0.3.0/tests/test_bridge_phase_b.py +1093 -0
- spacescans-0.3.0/tests/test_bridge_real_link.py +103 -0
- spacescans-0.3.0/tests/test_build_county_buildings_smoke.py +159 -0
- spacescans-0.3.0/tests/test_building_area_filter.py +64 -0
- spacescans-0.3.0/tests/test_building_quadkey.py +33 -0
- spacescans-0.3.0/tests/test_cbp_fallback_linkage.py +250 -0
- spacescans-0.3.0/tests/test_cli_smoke.py +31 -0
- spacescans-0.3.0/tests/test_cohort.py +251 -0
- spacescans-0.3.0/tests/test_config_resolution.py +124 -0
- spacescans-0.3.0/tests/test_demo_conus_adapter.py +50 -0
- spacescans-0.3.0/tests/test_extras_gating.py +28 -0
- spacescans-0.3.0/tests/test_faqsd_episode.py +111 -0
- spacescans-0.3.0/tests/test_fara_linkage.py +327 -0
- spacescans-0.3.0/tests/test_gridded_linkage.py +151 -0
- spacescans-0.3.0/tests/test_hdf4_gating.py +9 -0
- spacescans-0.3.0/tests/test_imports.py +41 -0
- spacescans-0.3.0/tests/test_init_config.py +47 -0
- spacescans-0.3.0/tests/test_io_base.py +20 -0
- spacescans-0.3.0/tests/test_linkage_helpers.py +37 -0
- spacescans-0.3.0/tests/test_models.py +32 -0
- spacescans-0.3.0/tests/test_nc_gating.py +12 -0
- spacescans-0.3.0/tests/test_nhd_proximity_linkage.py +133 -0
- spacescans-0.3.0/tests/test_no_v1_imports.py +20 -0
- spacescans-0.3.0/tests/test_orchestrate_core.py +832 -0
- spacescans-0.3.0/tests/test_overlap_fast_anchor.py +57 -0
- spacescans-0.3.0/tests/test_participant_level.py +976 -0
- spacescans-0.3.0/tests/test_pipeline_smoke.py +312 -0
- spacescans-0.3.0/tests/test_precomputed_areal_linkage.py +102 -0
- spacescans-0.3.0/tests/test_precomputed_static_linkage.py +90 -0
- spacescans-0.3.0/tests/test_quickstart.py +21 -0
- spacescans-0.3.0/tests/test_r_package_version.py +30 -0
- spacescans-0.3.0/tests/test_rda_gating.py +16 -0
- spacescans-0.3.0/tests/test_reader_fastpaths.py +207 -0
- spacescans-0.3.0/tests/test_sample_patients_from_buildings.py +117 -0
- spacescans-0.3.0/tests/test_temporal_episode_grouping.py +286 -0
- spacescans-0.3.0/tests/test_tier2_cross_track.py +182 -0
- spacescans-0.3.0/tests/test_tier2_golden_parity.py +406 -0
- spacescans-0.3.0/tests/test_tiger_proximity_search.py +98 -0
- spacescans-0.3.0/tests/test_transforms_base.py +11 -0
- spacescans-0.3.0/tests/test_upstream_bridge_support.py +327 -0
- spacescans-0.3.0/tests/test_web_parity_cohort.py +262 -0
- spacescans-0.3.0/tests/test_web_parity_coverage.py +117 -0
- spacescans-0.3.0/tests/test_web_parity_merge.py +240 -0
- spacescans-0.3.0/tests/test_web_parity_r1.py +73 -0
spacescans-0.3.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 spacescans authors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,439 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: spacescans
|
|
3
|
+
Version: 0.3.0
|
|
4
|
+
Summary: Link time-referenced geolocation data (location episodes) to the spatial and contextual exposome: spatial weights (C3) and day-weighted linkage (C4), per participant or per episode
|
|
5
|
+
Author: Regenstrief Institute
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://www.spacescans.com
|
|
8
|
+
Project-URL: Repository, https://github.com/IU-Ultraman/spacescans
|
|
9
|
+
Project-URL: Issues, https://github.com/IU-Ultraman/spacescans/issues
|
|
10
|
+
Project-URL: Changelog, https://github.com/IU-Ultraman/spacescans/blob/main/CHANGELOG.md
|
|
11
|
+
Keywords: exposome,geospatial,ehr,patient-exposure-linkage
|
|
12
|
+
Classifier: Development Status :: 3 - Alpha
|
|
13
|
+
Classifier: Intended Audience :: Science/Research
|
|
14
|
+
Classifier: Operating System :: MacOS
|
|
15
|
+
Classifier: Operating System :: POSIX :: Linux
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
21
|
+
Classifier: Topic :: Scientific/Engineering
|
|
22
|
+
Requires-Python: >=3.10
|
|
23
|
+
Description-Content-Type: text/markdown
|
|
24
|
+
License-File: LICENSE
|
|
25
|
+
Requires-Dist: pydantic>=2.0
|
|
26
|
+
Requires-Dist: pyyaml
|
|
27
|
+
Requires-Dist: pandas>=2.0
|
|
28
|
+
Requires-Dist: numpy
|
|
29
|
+
Requires-Dist: duckdb<2,>=0.9
|
|
30
|
+
Requires-Dist: pyarrow
|
|
31
|
+
Requires-Dist: mercantile>=1.2
|
|
32
|
+
Provides-Extra: geo
|
|
33
|
+
Requires-Dist: geopandas>=0.14; extra == "geo"
|
|
34
|
+
Requires-Dist: rasterio>=1.3; extra == "geo"
|
|
35
|
+
Requires-Dist: shapely>=2.0; extra == "geo"
|
|
36
|
+
Requires-Dist: exactextract>=0.2; extra == "geo"
|
|
37
|
+
Provides-Extra: rda
|
|
38
|
+
Requires-Dist: pyreadr; extra == "rda"
|
|
39
|
+
Provides-Extra: hdf4
|
|
40
|
+
Requires-Dist: pyhdf; extra == "hdf4"
|
|
41
|
+
Provides-Extra: nc
|
|
42
|
+
Requires-Dist: xarray; extra == "nc"
|
|
43
|
+
Requires-Dist: netCDF4; extra == "nc"
|
|
44
|
+
Provides-Extra: all
|
|
45
|
+
Requires-Dist: geopandas>=0.14; extra == "all"
|
|
46
|
+
Requires-Dist: rasterio>=1.3; extra == "all"
|
|
47
|
+
Requires-Dist: shapely>=2.0; extra == "all"
|
|
48
|
+
Requires-Dist: exactextract>=0.2; extra == "all"
|
|
49
|
+
Requires-Dist: mercantile>=1.2; extra == "all"
|
|
50
|
+
Requires-Dist: pyreadr; extra == "all"
|
|
51
|
+
Requires-Dist: pyhdf; extra == "all"
|
|
52
|
+
Requires-Dist: xarray; extra == "all"
|
|
53
|
+
Requires-Dist: netCDF4; extra == "all"
|
|
54
|
+
Provides-Extra: dev
|
|
55
|
+
Requires-Dist: pytest>=7.0; extra == "dev"
|
|
56
|
+
Requires-Dist: pytest-xdist; extra == "dev"
|
|
57
|
+
Requires-Dist: ruff; extra == "dev"
|
|
58
|
+
Requires-Dist: jsonschema>=4.21; extra == "dev"
|
|
59
|
+
Dynamic: license-file
|
|
60
|
+
|
|
61
|
+
# spacescans
|
|
62
|
+
|
|
63
|
+
**Link time-referenced geolocation data to the spatial and contextual exposome.**
|
|
64
|
+
|
|
65
|
+
spacescans links *location episodes* to environmental and contextual exposures. A location
|
|
66
|
+
episode is a participant identifier, a location (longitude and latitude) and the first and last
|
|
67
|
+
day spent there; a residential history is a series of them. For every episode spacescans
|
|
68
|
+
computes a buffer-weighted value of each exposure source (or the distance to the nearest
|
|
69
|
+
feature) and weights it by the days the episode overlaps each source period. By default the
|
|
70
|
+
result has **one row per participant**: each exposure averaged over all of the participant's
|
|
71
|
+
episodes, weighted by days. One row per location episode is available with
|
|
72
|
+
`output_level="episode"`.
|
|
73
|
+
|
|
74
|
+
Exposures come from a catalog of 11 public data products (each site downloads them itself; see
|
|
75
|
+
[Getting the exposure data](#getting-the-exposure-data)) or from datasets you register: tables
|
|
76
|
+
keyed by census geography, rasters, and point, line or polygon features. The same engine runs
|
|
77
|
+
behind the Python API, the command line, the R package in [`r/`](https://github.com/IU-Ultraman/spacescans/tree/main/r)
|
|
78
|
+
and the SPACESCANS web application.
|
|
79
|
+
|
|
80
|
+
[](https://www.python.org/)
|
|
81
|
+
[](https://www.spacescans.com)
|
|
82
|
+
|
|
83
|
+
## Install
|
|
84
|
+
|
|
85
|
+
```bash
|
|
86
|
+
pip install "spacescans[geo]"
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
The development version installs from GitHub:
|
|
90
|
+
|
|
91
|
+
```bash
|
|
92
|
+
pip install "spacescans[geo] @ git+https://github.com/IU-Ultraman/spacescans.git"
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
You install and import `spacescans`. (Before 0.3.0 the distribution was called
|
|
96
|
+
`spacescans-pipeline`; it was never on PyPI.) It needs Python 3.10 to 3.13 on Linux or macOS. The base install is light; the extras add what each kind of
|
|
97
|
+
source needs:
|
|
98
|
+
|
|
99
|
+
| Extra | Needed for | Adds |
|
|
100
|
+
| --- | --- | --- |
|
|
101
|
+
| (none) | the catalog, `spacescans data`, cohort checks, exposure-value aggregation from existing weights | pandas, DuckDB, pyarrow, pydantic |
|
|
102
|
+
| `geo` | spatial weighting, which every catalog source needs; registered rasters and features; the quickstart | geopandas, rasterio, shapely, exactextract |
|
|
103
|
+
| `rda` | sources distributed as R data files: EPA Walkability Index, Community Organization Density, FARA Food Access, Neighborhood Deprivation Index | pyreadr |
|
|
104
|
+
| `nc` | ACAG PM2.5 Composition | xarray, netCDF4 |
|
|
105
|
+
| `hdf4` | TEMIS UV Exposure | pyhdf (wheels for common platforms; otherwise it builds against the system HDF4 library) |
|
|
106
|
+
| `all` | everything above | |
|
|
107
|
+
|
|
108
|
+
`pip install "spacescans[geo,rda,nc]"` covers every catalog source except TEMIS UV
|
|
109
|
+
Exposure. An optional reader or pattern whose extra is missing raises `MissingExtraError`, which
|
|
110
|
+
names the `pip install` to run.
|
|
111
|
+
|
|
112
|
+
## Quickstart (no data needed)
|
|
113
|
+
|
|
114
|
+
```bash
|
|
115
|
+
spacescans quickstart --output-dir ./demo-out
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
The quickstart needs the `geo` extra. It runs one spatial-weighting step (C3) on synthetic
|
|
119
|
+
sample data that comes with the package: 270 m buffers around 10 synthetic locations,
|
|
120
|
+
intersected with three sample county polygons. It writes the spatial weights to
|
|
121
|
+
`./demo-out/quickstart_county_weights.parquet`. It does not link exposure values; the next
|
|
122
|
+
section does.
|
|
123
|
+
|
|
124
|
+
## Link location episodes (Python)
|
|
125
|
+
|
|
126
|
+
### Your cohort
|
|
127
|
+
|
|
128
|
+
One row per location episode; a participant who moved has one row per location.
|
|
129
|
+
|
|
130
|
+
| Column | Content |
|
|
131
|
+
| --- | --- |
|
|
132
|
+
| `pid` | participant identifier (a `PATID` column is accepted too) |
|
|
133
|
+
| `startDate`, `endDate` | first and last day at the location: `2017-08-19` (ISO, preferred) or `8/19/2017` |
|
|
134
|
+
| `longitude`, `latitude` | the location, in decimal degrees |
|
|
135
|
+
|
|
136
|
+
Every row needs a `pid` and a location: the cohort check refuses a file with a blank `pid` or a
|
|
137
|
+
missing, non-numeric or out-of-range coordinate, or with an `endDate` before its `startDate`,
|
|
138
|
+
before anything runs. A blank date is allowed, with a warning: that episode contributes no days.
|
|
139
|
+
Only these five columns are written to disk, into a private (0700) run directory; other columns
|
|
140
|
+
stay in your session. Cohort values (identifiers, dates, coordinates) never appear in messages,
|
|
141
|
+
events, warnings or provenance.
|
|
142
|
+
|
|
143
|
+
### Link
|
|
144
|
+
|
|
145
|
+
```python
|
|
146
|
+
import pandas as pd
|
|
147
|
+
from spacescans import data
|
|
148
|
+
from spacescans.orchestrate import link
|
|
149
|
+
|
|
150
|
+
# Once: download NPS Soundscape Noise (1.2 GB) from the project mirror.
|
|
151
|
+
data.fetch(["noise"], data_dir="~/ss-data")
|
|
152
|
+
|
|
153
|
+
cohort = pd.DataFrame({
|
|
154
|
+
"pid": ["P1", "P1", "P2", "P3"],
|
|
155
|
+
"startDate": ["2015-01-01", "2017-06-01", "2016-03-15", "2014-07-01"],
|
|
156
|
+
"endDate": ["2017-05-31", "2019-12-31", "2019-12-31", "2018-06-30"],
|
|
157
|
+
"longitude": [-86.158, -86.131, -85.680, -87.630],
|
|
158
|
+
"latitude": [39.768, 39.790, 40.193, 41.880],
|
|
159
|
+
})
|
|
160
|
+
|
|
161
|
+
res = link(cohort, ["noise"], data_dir="~/ss-data")
|
|
162
|
+
res.exposures # one row per participant: pid, l50dba_exi, l50dba_imp, l50dba_nat
|
|
163
|
+
res.coverage # linkage rate of each variable: the share of participants with a value
|
|
164
|
+
res.features # feature dictionary: what each output column is, with its unit
|
|
165
|
+
res.provenance # versions, data receipts, cache keys and settings; no cohort values
|
|
166
|
+
|
|
167
|
+
by_episode = link(cohort, ["noise"], data_dir="~/ss-data", output_level="episode")
|
|
168
|
+
```
|
|
169
|
+
|
|
170
|
+
`cohort` can also be the path of a CSV file. Other arguments: `buffer_m` (buffer radius in
|
|
171
|
+
meters, default 270), `raster_res_m` (default 25), `cache` (a directory that keeps spatial
|
|
172
|
+
weights between calls, so further sources on the same geography reuse them), `custom`
|
|
173
|
+
(registered datasets, below), `on_error` (`"stop"` or `"continue"` when a source fails),
|
|
174
|
+
`threads` and `memory_limit` (DuckDB limits per step). `spacescans.catalog.variables()` lists
|
|
175
|
+
the catalog keys; `spacescans.catalog.coverage(cohort, keys)` checks before a run which episodes
|
|
176
|
+
fall inside each source's spatial and temporal coverage.
|
|
177
|
+
|
|
178
|
+
### Participant level and episode level
|
|
179
|
+
|
|
180
|
+
With `output_level="participant"` (the default) the result has one row per participant id in
|
|
181
|
+
the cohort, in order of first appearance: `pid`, then the requested variables, with no dates,
|
|
182
|
+
coordinates or episode ids. For participant *i* and variable *v*:
|
|
183
|
+
|
|
184
|
+
```text
|
|
185
|
+
x̄(i,v) = Σ_e Σ_t d(e,t) · z(e,t,v) / Σ_e Σ_t d(e,t)
|
|
186
|
+
```
|
|
187
|
+
|
|
188
|
+
where *e* runs over the participant's episodes, *t* over the source periods (year, two-week
|
|
189
|
+
period, day; a single period for time-invariant sources) that overlap the episode,
|
|
190
|
+
z(e,t,v) is the spatial estimate for the episode's location in that period (the
|
|
191
|
+
buffer-weighted mean, or the distance to the nearest feature), and d(e,t) is the number of
|
|
192
|
+
days of the episode in that period, first and last day included. Only periods with a defined
|
|
193
|
+
estimate count, separately for each variable; a participant with no contributing day keeps
|
|
194
|
+
their row, with missing values. This is the same as averaging the episode values, each
|
|
195
|
+
weighted by its contributing days.
|
|
196
|
+
|
|
197
|
+
- **Overlapping location episodes.** When two episodes of one participant overlap in time, each
|
|
198
|
+
contributes its own days: a calendar day that both cover counts once for each of the two
|
|
199
|
+
locations, so the participant value is a mean over episode-days, not over calendar days. Trim
|
|
200
|
+
the overlaps in your cohort if a day should count only once.
|
|
201
|
+
- **Derived variables** are derived from the participant's means: FARA Food Access's flags (such
|
|
202
|
+
as `LILATracts_1And10`) are recoded from the participant's mean rates and urban share, so they
|
|
203
|
+
stay 0 or 1, and each ACAG `_nbm` (non-biomass) column is the participant's mean of the species
|
|
204
|
+
minus the participant's mean of its biomass part (`_bm`). At the episode level the same rules
|
|
205
|
+
apply to each episode's means.
|
|
206
|
+
|
|
207
|
+
With `output_level="episode"` the result has one row per location episode, with `episode_id`
|
|
208
|
+
(the 0-based row number of the episode in your input) and the variables.
|
|
209
|
+
|
|
210
|
+
Linkage rates are computed at the level of the output: the share of participants with a value,
|
|
211
|
+
or the share of episodes in episode mode.
|
|
212
|
+
|
|
213
|
+
## Register your own datasets
|
|
214
|
+
|
|
215
|
+
`spacescans.custom` keeps a library of datasets you add. Registration validates the files and
|
|
216
|
+
prepares them once (normalized Parquet for tables, a checked grid and period index for rasters,
|
|
217
|
+
GeoParquet in EPSG:5070 for features), stores them with their metadata (`dataset.json`), and
|
|
218
|
+
every later linkage reuses the prepared files.
|
|
219
|
+
|
|
220
|
+
```python
|
|
221
|
+
from pathlib import Path
|
|
222
|
+
from spacescans import custom
|
|
223
|
+
|
|
224
|
+
custom.register(["clinics.gpkg"], name="clinics", kind="features", temporal="static")
|
|
225
|
+
custom.register(sorted(Path("pm_biweekly").glob("*.tif")), name="pm_biweekly",
|
|
226
|
+
kind="raster", temporal="biweekly")
|
|
227
|
+
|
|
228
|
+
custom.list_datasets() # DatasetRecord: id, name, kind, temporal, variables, metadata
|
|
229
|
+
res = link(cohort, ["noise"], data_dir="~/ss-data", custom=["clinics", "pm_biweekly"])
|
|
230
|
+
custom.remove_dataset("pm_biweekly")
|
|
231
|
+
```
|
|
232
|
+
|
|
233
|
+
| `kind` | Files | `temporal` | Linked as |
|
|
234
|
+
| --- | --- | --- | --- |
|
|
235
|
+
| `polygon_table` | CSV keyed by 2010 census tract, block group, ZIP Code Tabulation Area or county | `static`, `yearly` (a year column), `daily` (a date column) | area-weighted mean over the buffer |
|
|
236
|
+
| `raster` | GeoTIFF: one, or one per period on a common grid | `static`, `yearly`, `biweekly`, `daily` | buffer-weighted mean of the cells |
|
|
237
|
+
| `features` | points, lines or polygons (one geometry type per dataset) with a coordinate reference system, as GeoPackage, GeoJSON, zipped Shapefile or GeoParquet | `static`, `yearly` (one file per year, or a year column) | distance in meters from the location to the nearest feature (`<name>_dist_m` unless you name the variable) |
|
|
238
|
+
|
|
239
|
+
`register()` also takes `key` (a table's geography and key column, such as
|
|
240
|
+
`{"boundary": "tract", "column": "GEOID"}`; for features optionally a year column or a layer),
|
|
241
|
+
`variables` (name, unit and description of each value), `metadata` (source, version,
|
|
242
|
+
description, license note, visibility, owner) and `overwrite`. A record has
|
|
243
|
+
an `id`, a `name` unique in its library, the checksum of the original files and the spacescans
|
|
244
|
+
version that prepared it. The library lives in `root=`, else
|
|
245
|
+
`$SPACESCANS_DATASETS_DIR`, else the user data directory (`platformdirs.user_data_dir("spacescans")`
|
|
246
|
+
followed by `datasets`). Each record's `visibility` (`"private"` or `"deployment"`) is
|
|
247
|
+
informational in the package; the web application enforces it, sharing a `"deployment"` dataset
|
|
248
|
+
with every user of that deployment. `get_dataset(name_or_id)` returns one record. The reference
|
|
249
|
+
is [docs/custom-datasets.md](https://github.com/IU-Ultraman/spacescans/blob/main/docs/custom-datasets.md).
|
|
250
|
+
|
|
251
|
+
`custom` also accepts in-memory definitions made with `custom.define_polygon()` and
|
|
252
|
+
`custom.define_raster()`, as a mapping of variable key to definition.
|
|
253
|
+
|
|
254
|
+
## Getting the exposure data
|
|
255
|
+
|
|
256
|
+
No exposure data comes with spacescans. `spacescans data` knows where each catalog variable's
|
|
257
|
+
data comes from and what a complete copy looks like under the data directory
|
|
258
|
+
(`spacescans/data/manifest.json`); it needs no extras.
|
|
259
|
+
|
|
260
|
+
```bash
|
|
261
|
+
spacescans data status ndi walkability --data-dir ~/ss-data # present, missing, sizes, next step
|
|
262
|
+
spacescans data fetch noise tiger_proximity --data-dir ~/ss-data --dry-run
|
|
263
|
+
spacescans data fetch noise tiger_proximity --data-dir ~/ss-data # resumable; re-run after a drop
|
|
264
|
+
spacescans data verify --data-dir ~/ss-data # sha256 of everything placed
|
|
265
|
+
spacescans data adopt --data-dir /existing/web/pipeline-data # receipts for data placed earlier
|
|
266
|
+
```
|
|
267
|
+
|
|
268
|
+
- Files the project mirrors download from the project's OneDrive mirror (or `--mirror URL|DIR`,
|
|
269
|
+
`$SPACESCANS_DATA_MIRROR`), resume after interruptions, are checked against their published
|
|
270
|
+
sha256, and are recorded in `<data-dir>/.spacescans/receipts/`.
|
|
271
|
+
- EPA FAQSD Air Quality, TEMIS UV Exposure and ACAG PM2.5 Composition download from their
|
|
272
|
+
providers once you accept the provider's terms in the call (`--accept-terms ID`; `status` and
|
|
273
|
+
`fetch --dry-run` print each ID with its terms). VIIRS Nighttime Lights needs a manual
|
|
274
|
+
download; `status` prints the instructions, license and attribution. Provider terms are never
|
|
275
|
+
accepted on your behalf.
|
|
276
|
+
- The same functions are available from Python: `spacescans.data.status()`, `fetch()`,
|
|
277
|
+
`verify()`, `adopt()`, `terms()`, `import_files()`.
|
|
278
|
+
|
|
279
|
+
The catalog (every source covers the conterminous United States; the time-varying sources cover
|
|
280
|
+
2013–2019, the Neighborhood Deprivation Index 2012–2022):
|
|
281
|
+
|
|
282
|
+
| Key | Source | Spatial data | Time step | Extras |
|
|
283
|
+
| --- | --- | --- | --- | --- |
|
|
284
|
+
| `walkability` | EPA Walkability Index | polygon (block group) | time-invariant | `geo`, `rda` |
|
|
285
|
+
| `cbp_zcta5` | Community Organization Density | polygon (ZCTA, county fallback) | yearly | `geo`, `rda` |
|
|
286
|
+
| `fara_tract` | FARA Food Access | polygon (census tract) | yearly | `geo`, `rda` |
|
|
287
|
+
| `ndi` | Neighborhood Deprivation Index | polygon (block group, 2010 and 2020) | yearly | `geo`, `rda` |
|
|
288
|
+
| `faqsd` | EPA FAQSD Air Quality | polygon (census tract) | daily | `geo` |
|
|
289
|
+
| `noise` | NPS Soundscape Noise | raster (270 m) | time-invariant | `geo` |
|
|
290
|
+
| `vnl` | VIIRS Nighttime Lights | raster (15″) | yearly | `geo` |
|
|
291
|
+
| `acag` | ACAG PM2.5 Composition | raster (0.01°) | biweekly | `geo`, `nc` |
|
|
292
|
+
| `temis` | TEMIS UV Exposure | raster (0.25°) | daily | `geo`, `hdf4` |
|
|
293
|
+
| `nhd_bluespace` | NHD Bluespace | line (distance to the nearest water feature) | time-invariant | `geo` |
|
|
294
|
+
| `tiger_proximity` | TIGER Road Distance | line (distance to the nearest road) | yearly | `geo` |
|
|
295
|
+
|
|
296
|
+
## Command line
|
|
297
|
+
|
|
298
|
+
```bash
|
|
299
|
+
export SPACESCANS_DATA_DIR=/path/to/exposome-data
|
|
300
|
+
export SPACESCANS_OUTPUT_DIR=/path/to/results
|
|
301
|
+
|
|
302
|
+
# Copy the editable template configs, then point their paths at your data
|
|
303
|
+
spacescans init-config --out ./configs
|
|
304
|
+
|
|
305
|
+
# Run the pipelines in dependency order
|
|
306
|
+
spacescans run ./configs/c3/county.yaml
|
|
307
|
+
spacescans run ./configs/c3/zcta5.yaml
|
|
308
|
+
spacescans run ./configs/c4/zbp.yaml # depends on C3 zcta5
|
|
309
|
+
spacescans run ./configs/c4/cbp_fallback.yaml # depends on C3 county + C4 zbp
|
|
310
|
+
```
|
|
311
|
+
|
|
312
|
+
A C4 config's `time.output_grouping` is `patient` (one row per participant, the default) or
|
|
313
|
+
`episode` (one row per location episode); the templates state it. Paths resolve in three tiers,
|
|
314
|
+
**CLI flag > `$SPACESCANS_DATA_DIR` > YAML `base_dir:`**, with `${SPACESCANS_DATA_DIR}`
|
|
315
|
+
expansion inside YAML paths.
|
|
316
|
+
|
|
317
|
+
## How it works
|
|
318
|
+
|
|
319
|
+
Every linkage runs in two steps, both driven by YAML configs and the same DuckDB engine:
|
|
320
|
+
|
|
321
|
+
1. **Spatial weighting (C3, needs `geo`)**: the share of each location's buffer that falls in
|
|
322
|
+
each polygon or raster cell, or the distance from the location to the nearest line or
|
|
323
|
+
feature. The weights depend only on the cohort's locations, the source geography, the
|
|
324
|
+
buffer radius and the rasterization resolution, so they are computed once per `link()` call
|
|
325
|
+
(and across calls with a `cache` directory) and reused by every further source on the same
|
|
326
|
+
geography.
|
|
327
|
+
2. **Exposure-value aggregation (C4, base install)**: the weights applied to each source
|
|
328
|
+
period's values, weighted in turn by the days each episode overlaps the period, per episode
|
|
329
|
+
or per participant.
|
|
330
|
+
|
|
331
|
+
### C3 patterns
|
|
332
|
+
|
|
333
|
+
| Pattern | What it does | Typical data |
|
|
334
|
+
| --- | --- | --- |
|
|
335
|
+
| `boundary_overlap_fast` | buffer × polygon area-overlap weights (per-tile bulk rasterization) | block group, tract, ZCTA, county |
|
|
336
|
+
| `boundary_overlap` | the same without per-tile rasterization (slower) | |
|
|
337
|
+
| `grid_weights` | buffer × raster-cell coverage weights | ACAG, noise, TEMIS, VIIRS |
|
|
338
|
+
| `tiger_proximity` | distance from each location to the nearest primary and secondary road, per year | TIGER roads |
|
|
339
|
+
| `nhd_proximity` | distance from each location to the nearest flowline, waterbody, area and coastline | NHD |
|
|
340
|
+
|
|
341
|
+
Distances to your own point, line or polygon features take one of two routes, which give the
|
|
342
|
+
same exact nearest-feature distance:
|
|
343
|
+
|
|
344
|
+
- **Registered features** (`custom.register(..., kind="features")`, then `link(custom=[...])`;
|
|
345
|
+
also R's `ss_register_dataset()` and the web application's feature uploads): the distances
|
|
346
|
+
from each location to the nearest feature are computed at the start of each link
|
|
347
|
+
(`spacescans.custom.features`, in EPSG:5070 meters) and aggregated by the `static_areal` or
|
|
348
|
+
`yearly_areal` C4 pattern. No C3 step.
|
|
349
|
+
- **Hand-written YAML** run with `spacescans run`: the `proximity` pattern does both steps in
|
|
350
|
+
one. Its config names the features (`source.file`: one vector or GeoParquet file, or one per
|
|
351
|
+
year of `time.years`, or one file with `source.year_col`), the output column
|
|
352
|
+
(`exposure.value_cols`, default `distance_m`) and optionally `source.category_col` (one
|
|
353
|
+
column per category); distances are in meters in `buffer.aea_crs` (an Albers equal-area
|
|
354
|
+
projection of the conterminous US) and `time.output_grouping` applies as in every C4 config.
|
|
355
|
+
|
|
356
|
+
### C4 patterns
|
|
357
|
+
|
|
358
|
+
| Pattern | Time | What it does | Typical data |
|
|
359
|
+
| --- | --- | --- | --- |
|
|
360
|
+
| `static_areal` | time-invariant | area- or cell-weighted value, weighted by episode days | walkability, noise |
|
|
361
|
+
| `yearly_areal` | yearly | area-weighted yearly values, weighted by the days in each year | ZCTA business patterns |
|
|
362
|
+
| `yearly_areal_bg_vintage` | yearly | `yearly_areal` over 2010 and 2020 block groups, by year | NDI |
|
|
363
|
+
| `cbp_fallback` | yearly | ZCTA values, falling back to county values where there is no ZCTA value | Community Organization Density |
|
|
364
|
+
| `fara_tract` | yearly | `yearly_areal` with the FARA flag recodes (needs `rda`) | FARA |
|
|
365
|
+
| `faqsd_daily_areal` | daily | daily tract values with day-level overlap | EPA FAQSD |
|
|
366
|
+
| `gridded` | yearly, biweekly, daily | per-period values over the C3 weights of raster cells or polygons (needs `geo`) | VIIRS, TEMIS; registered daily tables and daily or biweekly rasters |
|
|
367
|
+
| `acag_multi` | biweekly | the ACAG species over the C3 cell weights (needs `geo`, `nc`) | ACAG |
|
|
368
|
+
| `precomputed_areal` | yearly | per-(location, year) values such as road distances, weighted by days | TIGER distances |
|
|
369
|
+
| `precomputed_static` | time-invariant | per-location values such as water distances, weighted by days | NHD distances |
|
|
370
|
+
|
|
371
|
+
## Limits
|
|
372
|
+
|
|
373
|
+
- Catalog sources cover the conterminous United States and, when they vary over time, at least
|
|
374
|
+
2013–2019.
|
|
375
|
+
- Each pipeline step runs as a single process, using about one processor core. Support for
|
|
376
|
+
several cores is planned.
|
|
377
|
+
- The daily sources need memory roughly in proportion to the number of episodes: EPA FAQSD Air
|
|
378
|
+
Quality and TEMIS UV Exposure ran out of memory in a 24 GiB container at 100,000 episodes.
|
|
379
|
+
Measured peaks per source are in
|
|
380
|
+
[docs/memory.md](https://github.com/IU-Ultraman/spacescans/blob/main/docs/memory.md).
|
|
381
|
+
|
|
382
|
+
## R package
|
|
383
|
+
|
|
384
|
+
The R package in [`r/`](https://github.com/IU-Ultraman/spacescans/tree/main/r) (`ss_link()`,
|
|
385
|
+
`ss_fetch()`, `ss_register_dataset()` and friends) runs this package in a separate Python
|
|
386
|
+
process through a JSON-file bridge ([docs/bridge-protocol.md](https://github.com/IU-Ultraman/spacescans/blob/main/docs/bridge-protocol.md)).
|
|
387
|
+
It is released with the same version as the Python package. Install it with
|
|
388
|
+
`remotes::install_github("IU-Ultraman/spacescans", subdir = "r")`, then run `ss_install()`
|
|
389
|
+
once; see its [README](https://github.com/IU-Ultraman/spacescans/blob/main/r/README.md).
|
|
390
|
+
|
|
391
|
+
## Project layout
|
|
392
|
+
|
|
393
|
+
```text
|
|
394
|
+
src/spacescans/
|
|
395
|
+
├── orchestrate/ link(): a cohort and variables in, exposures out; run dirs, cache, merge
|
|
396
|
+
├── catalog/ the catalog variables, feature dictionary, recipes and templates
|
|
397
|
+
├── custom/ registered datasets and uploaded values (tables, rasters, features)
|
|
398
|
+
├── data/ spacescans data: manifest, status, fetch, verify, adopt
|
|
399
|
+
├── cohort.py cohort checks and date parsing
|
|
400
|
+
├── bridge/ python -m spacescans.bridge, the JSON-file protocol the R package uses
|
|
401
|
+
├── cli/ run / quickstart / init-config / data
|
|
402
|
+
├── config_resolution.py 3-tier path resolution (CLI > env > YAML) + ${VAR} expansion
|
|
403
|
+
├── pipeline/ config loader, pattern registry and runner
|
|
404
|
+
├── models/ pydantic config schema
|
|
405
|
+
├── engine/ DuckDB aggregation engine
|
|
406
|
+
├── io/ readers (Parquet, CSV, .Rda) and writers
|
|
407
|
+
├── geometry/ buffers, overlap, grid weights, proximity (geo)
|
|
408
|
+
├── transforms/ filter / recode / derive / date_parse / spatial_fix
|
|
409
|
+
├── linkage/ linkage-pattern implementations
|
|
410
|
+
├── plugins/ source-specific readers (extras-gated)
|
|
411
|
+
└── resources/ synthetic sample data and template configs
|
|
412
|
+
```
|
|
413
|
+
|
|
414
|
+
`import spacescans` loads only the base dependencies.
|
|
415
|
+
|
|
416
|
+
## Development
|
|
417
|
+
|
|
418
|
+
```bash
|
|
419
|
+
pip install -e '.[all,dev]'
|
|
420
|
+
pytest # full suite (base + any extras present in the environment)
|
|
421
|
+
pytest -m "not extras" # base-only tests
|
|
422
|
+
pytest -m geo # geo-extra tests
|
|
423
|
+
```
|
|
424
|
+
|
|
425
|
+
Opt-in suites (real data, golden references, web parity) are described in
|
|
426
|
+
[docs/tier2-notes.md](https://github.com/IU-Ultraman/spacescans/blob/main/docs/tier2-notes.md).
|
|
427
|
+
Changes are listed in [CHANGELOG.md](https://github.com/IU-Ultraman/spacescans/blob/main/CHANGELOG.md);
|
|
428
|
+
the release procedure is [docs/release.md](https://github.com/IU-Ultraman/spacescans/blob/main/docs/release.md).
|
|
429
|
+
|
|
430
|
+
## License
|
|
431
|
+
|
|
432
|
+
spacescans is released under the MIT License
|
|
433
|
+
([LICENSE](https://github.com/IU-Ultraman/spacescans/blob/main/LICENSE)); the R package in
|
|
434
|
+
`r/` is released under the same license. The license covers the software only: each catalog
|
|
435
|
+
source keeps its provider's terms, which `spacescans data status` prints.
|
|
436
|
+
|
|
437
|
+
---
|
|
438
|
+
|
|
439
|
+
Learn more at [www.spacescans.com](https://www.spacescans.com).
|