usdata 0.33.0__tar.gz → 0.34.0__tar.gz

{usdata-0.33.0 → usdata-0.34.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: usdata
-Version: 0.33.0
+Version: 0.34.0
 Summary: Unified Python SDK and CLI for discovering, fetching, and tracking provenance of U.S. public scientific data
 Keywords: noaa,usgs,open-data,scientific-data,provenance,reproducible-research,weather,climate,meteorology
 Author: Jake Van Slyke
@@ -113,7 +113,7 @@ breaking changes.
 | [docs.usdata.dev](https://docs.usdata.dev/) | How to use it: [install](https://docs.usdata.dev/install/), [getting started](https://docs.usdata.dev/getting-started/), guides, dataset notes, and reference |
 | [Severe-weather case study](https://usdata.dev/studies/severe-weather-case-study/) | One tornado, six sources, one manifest and lockfile, ending in a citation |
 
-Thirty-one datasets are available today and seventeen more are planned, grouped
+Thirty-four datasets are available today and seventeen more are planned, grouped
 by agency and product family in the [catalog](docs/providers/README.md).
 
 ## How this compares

{usdata-0.33.0 → usdata-0.34.0}/README.md

@@ -68,7 +68,7 @@ breaking changes.
 | [docs.usdata.dev](https://docs.usdata.dev/) | How to use it: [install](https://docs.usdata.dev/install/), [getting started](https://docs.usdata.dev/getting-started/), guides, dataset notes, and reference |
 | [Severe-weather case study](https://usdata.dev/studies/severe-weather-case-study/) | One tornado, six sources, one manifest and lockfile, ending in a citation |
 
-Thirty-one datasets are available today and seventeen more are planned, grouped
+Thirty-four datasets are available today and seventeen more are planned, grouped
 by agency and product family in the [catalog](docs/providers/README.md).
 
 ## How this compares

{usdata-0.33.0 → usdata-0.34.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "usdata"
-version = "0.33.0"
+version = "0.34.0"
 description = "Unified Python SDK and CLI for discovering, fetching, and tracking provenance of U.S. public scientific data"
 readme = "README.md"
 license = "Apache-2.0"

{usdata-0.33.0 → usdata-0.34.0}/pyproject.toml.orig

@@ -1,6 +1,6 @@
 [project]
 name = "usdata"
-version = "0.33.0"
+version = "0.34.0"
 description = "Unified Python SDK and CLI for discovering, fetching, and tracking provenance of U.S. public scientific data"
 readme = "README.md"
 license = "Apache-2.0"

{usdata-0.33.0 → usdata-0.34.0}/src/usdata/data/registry.yaml

@@ -55,7 +55,7 @@ systems:
   noaa:goes-r:
     name: "GOES-R Series"
     provider: noaa
-    homepage: https://www.goes-r.gov/
+    homepage: https://www.nesdis.noaa.gov/our-satellites/currently-flying/geostationary-satellites
   noaa:ncep-models:
     name: "NCEP model output"
     provider: noaa
@@ -64,6 +64,10 @@ systems:
     name: "CO-OPS Data API"
     provider: noaa
     homepage: https://tidesandcurrents.noaa.gov/
+  noaa:dat:
+    name: "NWS Damage Assessment Toolkit (DAT)"
+    provider: noaa
+    homepage: https://apps.dat.noaa.gov/StormDamage/DamageViewer/
 
 datasets:
   # ---------------------------------------------------------------- NOAA
@@ -212,22 +216,25 @@ datasets:
     system: noaa:goes-r
     domain: weather-satellites
     since: "0.8"
-    title: GOES-R ABI Cloud and Moisture Imagery
+    title: GOES-R ABI Imagery and Cloud-Top Products
     description: >-
-      Single-channel CONUS (ABI-L2-CMIPC) and mesoscale (ABI-L2-CMIPM) imagery from
-      GOES-16, 17, 18, and 19 in anonymous NOAA S3 buckets. Select an explicit
-      satellite, channel, scan-start interval, and M1 or M2 for mesoscale; each asset is a complete
-      NetCDF scene with no geographic or variable subsetting.
-    keywords: [satellite, imagery, goes, abi, clouds, infrared, reflectance, netcdf, conus, mesoscale]
+      ABI Level 2 products from GOES-16, 17, 18, and 19 in anonymous NOAA S3 buckets:
+      single-channel (ABI-L2-CMIP) and 16-band (ABI-L2-MCMIP) Cloud and Moisture Imagery,
+      and cloud-top height (ACHA), temperature (ACHT), pressure (CTP), and phase (ACTP),
+      each in the CONUS, full-disk, and M1/M2 mesoscale sectors NOAA produces it for. Select
+      an explicit satellite, one product directory, a scan-start interval, a channel for
+      single-channel imagery, and M1 or M2 for mesoscale; each asset is a complete NetCDF
+      scene with no geographic or variable subsetting.
+    keywords: [satellite, imagery, goes, abi, clouds, cloud, infrared, reflectance, netcdf, conus, mesoscale, disk, multiband, height, pressure, phase]
     protocol: s3
     homepage: https://registry.opendata.aws/noaa-goes/
     license: US Government Work (public domain)
     temporal_extent: { start: "2017-02-28T00:00:00Z" }
     capabilities: { spatial_subset: false, temporal_subset: true, variable_subset: false }
-    summary: GOES CONUS and mesoscale imagery
+    summary: GOES ABI imagery, multiband scenes, and cloud-top products
     formats: [NetCDF4]
-    selection: Whole single-channel scenes by inclusive UTC scan-start time and explicit mesoscale sector
-    inputs: Satellite, channel, both timestamps; product and sector for mesoscale
+    selection: Whole scenes of one ABI product by inclusive UTC scan-start time, with one channel or mesoscale sector where the product has them
+    inputs: Satellite and both timestamps; product (default ABI-L2-CMIPC); channel for CMIP products; sector for mesoscale products
     reader: netcdf
     guide: docs/providers/noaa-goes.md
     examples:
@@ -235,16 +242,52 @@ datasets:
       - examples/studies/goes-mesoscale/goes-mesoscale.ipynb
       - examples/studies/event-context/event-context.ipynb
     resolution:
-      spatial: "0.5 km to 2 km at nadir, by ABI band"
-      temporal: "CONUS every 5 minutes; two mesoscale sectors every 60 seconds or one every 30 seconds"
+      spatial: "Imagery 0.5 km to 2 km at nadir by band, MCMIP bands on the 2 km grid; cloud-top products 2 km to 10 km at nadir"
+      temporal: "CONUS every 5 minutes; full disk every 10 minutes in Mode 6 (15 in Mode 3, 5 in Mode 4); two mesoscale sectors every 60 seconds or one every 30 seconds"
     update_frequency: "New data is added as soon as it's available"
     citation: >-
       NOAA Geostationary Operational Environmental Satellites (GOES) 16, 17, 18 & 19 was
       accessed on [date] from https://registry.opendata.aws/noaa-goes
     terms: https://www.noaa.gov/information-technology/open-data-dissemination
     variables:
-      - { name: "CMI", description: "Cloud and moisture imagery; reflectance factor or brightness temperature by band" }
-      - { name: "DQF", description: "Per-pixel data quality flags for CMI" }
+      - { name: "CMI", description: "CMIP cloud and moisture imagery; reflectance factor or brightness temperature by band" }
+      - { name: "DQF", units: "1", description: "Per-pixel data quality flags in CMIP and every cloud-top product" }
+      - { name: "CMI_C01", units: "1", description: "MCMIP band 1 (0.47 µm, blue visible) reflectance factor on the 2 km grid" }
+      - { name: "CMI_C02", units: "1", description: "MCMIP band 2 (0.64 µm, red visible) reflectance factor on the 2 km grid" }
+      - { name: "CMI_C03", units: "1", description: "MCMIP band 3 (0.86 µm, veggie near-infrared) reflectance factor on the 2 km grid" }
+      - { name: "CMI_C04", units: "1", description: "MCMIP band 4 (1.37 µm, cirrus near-infrared) reflectance factor on the 2 km grid" }
+      - { name: "CMI_C05", units: "1", description: "MCMIP band 5 (1.6 µm, snow/ice near-infrared) reflectance factor on the 2 km grid" }
+      - { name: "CMI_C06", units: "1", description: "MCMIP band 6 (2.2 µm, cloud particle size near-infrared) reflectance factor on the 2 km grid" }
+      - { name: "CMI_C07", units: "K", description: "MCMIP band 7 (3.9 µm, shortwave window infrared) brightness temperature on the 2 km grid" }
+      - { name: "CMI_C08", units: "K", description: "MCMIP band 8 (6.2 µm, upper-level water vapor) brightness temperature on the 2 km grid" }
+      - { name: "CMI_C09", units: "K", description: "MCMIP band 9 (6.9 µm, mid-level water vapor) brightness temperature on the 2 km grid" }
+      - { name: "CMI_C10", units: "K", description: "MCMIP band 10 (7.3 µm, lower-level water vapor) brightness temperature on the 2 km grid" }
+      - { name: "CMI_C11", units: "K", description: "MCMIP band 11 (8.4 µm, cloud-top phase infrared) brightness temperature on the 2 km grid" }
+      - { name: "CMI_C12", units: "K", description: "MCMIP band 12 (9.6 µm, ozone infrared) brightness temperature on the 2 km grid" }
+      - { name: "CMI_C13", units: "K", description: "MCMIP band 13 (10.3 µm, clean longwave window) brightness temperature on the 2 km grid" }
+      - { name: "CMI_C14", units: "K", description: "MCMIP band 14 (11.2 µm, longwave window) brightness temperature on the 2 km grid" }
+      - { name: "CMI_C15", units: "K", description: "MCMIP band 15 (12.3 µm, dirty longwave window) brightness temperature on the 2 km grid" }
+      - { name: "CMI_C16", units: "K", description: "MCMIP band 16 (13.3 µm, CO2 longwave infrared) brightness temperature on the 2 km grid" }
+      - { name: "DQF_C01", units: "1", description: "MCMIP per-pixel data quality flags for CMI_C01" }
+      - { name: "DQF_C02", units: "1", description: "MCMIP per-pixel data quality flags for CMI_C02" }
+      - { name: "DQF_C03", units: "1", description: "MCMIP per-pixel data quality flags for CMI_C03" }
+      - { name: "DQF_C04", units: "1", description: "MCMIP per-pixel data quality flags for CMI_C04" }
+      - { name: "DQF_C05", units: "1", description: "MCMIP per-pixel data quality flags for CMI_C05" }
+      - { name: "DQF_C06", units: "1", description: "MCMIP per-pixel data quality flags for CMI_C06" }
+      - { name: "DQF_C07", units: "1", description: "MCMIP per-pixel data quality flags for CMI_C07" }
+      - { name: "DQF_C08", units: "1", description: "MCMIP per-pixel data quality flags for CMI_C08" }
+      - { name: "DQF_C09", units: "1", description: "MCMIP per-pixel data quality flags for CMI_C09" }
+      - { name: "DQF_C10", units: "1", description: "MCMIP per-pixel data quality flags for CMI_C10" }
+      - { name: "DQF_C11", units: "1", description: "MCMIP per-pixel data quality flags for CMI_C11" }
+      - { name: "DQF_C12", units: "1", description: "MCMIP per-pixel data quality flags for CMI_C12" }
+      - { name: "DQF_C13", units: "1", description: "MCMIP per-pixel data quality flags for CMI_C13" }
+      - { name: "DQF_C14", units: "1", description: "MCMIP per-pixel data quality flags for CMI_C14" }
+      - { name: "DQF_C15", units: "1", description: "MCMIP per-pixel data quality flags for CMI_C15" }
+      - { name: "DQF_C16", units: "1", description: "MCMIP per-pixel data quality flags for CMI_C16" }
+      - { name: "HT", units: "m", description: "ACHA cloud-top height" }
+      - { name: "TEMP", units: "K", description: "ACHT cloud-top temperature" }
+      - { name: "PRES", units: "hPa", description: "CTP cloud-top pressure" }
+      - { name: "Phase", units: "1", description: "ACTP cloud-top phase: clear sky, liquid water, supercooled liquid water, mixed phase, ice, or unknown" }
     limits: { max_window: P7D }
     adapter: usdata.providers.noaa.goes:GoesAbi
 
@@ -418,6 +461,118 @@ datasets:
       - { name: "mt", description: "Wind only, from 2006: EG estimated gust, MG measured gust, MS measured sustained, ES estimated sustained" }
     adapter: usdata.providers.noaa.spc:SpcTornadoReports
 
+  - id: noaa:nws-damage-surveys
+    provider: noaa
+    status: available
+    system: noaa:dat
+    domain: severe-weather
+    since: "0.34"
+    title: NWS Damage Survey Points, Tornado Tracks, and Damage Areas
+    description: >-
+      Post-event damage surveys by National Weather Service offices, from the Damage
+      Assessment Toolkit's public ArcGIS feature service: rated damage points (layer 0),
+      tornado tracks as lines (layer 1), and damage areas as polygons (layer 2). One layer
+      per query, selected by storm time in UTC, with server-side envelope, office, and EF
+      rating filters, delivered as GeoJSON pages of at most the layer's record limit, cut
+      by object-id range. The service is edited live and calls its data preliminary; an
+      as_of instant reads its archive, kept since November 2020, so a page cannot drift.
+      Most surveys date from 2010 on; offices have also entered historical reconstructions
+      back to 1855. Attributes differ by layer; the catalog lists the common ones.
+    keywords: [tornado, damage survey, damage assessment toolkit, dat, ef scale, tracks, paths, nws, geojson, arcgis]
+    protocol: http
+    homepage: https://apps.dat.noaa.gov/StormDamage/DamageViewer/
+    license: US Government Work (public domain)
+    temporal_extent: { start: "1855-05-22T00:00:00Z" }
+    capabilities: { spatial_subset: true, temporal_subset: true, variable_subset: false }
+    summary: NWS damage survey points, tornado tracks, and damage areas
+    formats: [GeoJSON]
+    selection: One layer's features with storm time in an inclusive UTC window, optionally in a box; one page per object-id run
+    inputs: Both timestamps and a layer; optionally a bbox, offices, EF ratings, undated features, or as_of
+    reader: null
+    guide: docs/providers/noaa-nws-damage-surveys.md
+    examples:
+      - examples/datasets/noaa-nws-damage-surveys/noaa-nws-damage-surveys.ipynb
+    resolution:
+      spatial: "One feature per surveyed damage point, tornado track, or damage area"
+      temporal: "Storm time to the minute, in UTC"
+    update_frequency: >-
+      As offices finish surveys, with no schedule: "Once quality-controlled, the data
+      becomes available externally via a
+      public web portal, as well as public Geographic Information System (GIS) services"
+    citation: >-
+      NOAA National Weather Service, Damage Assessment Toolkit (DamageViewer feature
+      service), accessed via usdata
+    terms: https://www.weather.gov/disclaimer
+    variables:
+      - { name: objectid, description: "Feature id, unique within its layer; pages are cut by it" }
+      - { name: globalid, description: "Feature GUID, unique across the service" }
+      - { name: stormdate, units: "milliseconds since 1970-01-01 UTC", description: "Storm time; null on a few features, and the 1970-01-01T00:00Z placeholder on eleven points" }
+      - { name: starttime, units: "milliseconds since 1970-01-01 UTC", description: "Lines: when the tornado began; the window's field on lines" }
+      - { name: endtime, units: "milliseconds since 1970-01-01 UTC", description: "Lines: when the tornado ended" }
+      - { name: efscale, description: "Rating as text: EF0 to EF5, EF3+, EFU, UNKNOWN, N/A, TSTM/Wind, or Tropical" }
+      - { name: efnum, description: "Lines: EF number, -99 when unknown" }
+      - { name: office, description: "Points: surveying NWS office code" }
+      - { name: wfo, description: "Lines: surveying NWS office code, or several joined by commas" }
+      - { name: event_id, description: "An office's label for the event; often empty" }
+      - { name: damage_txt, description: "Points: EF-scale damage indicator, such as One- or Two-Family Residences (FR12)" }
+      - { name: dod_txt, description: "Points: degree of damage to that indicator, in words" }
+      - { name: windspeed, description: "Points: estimated wind speed as text; the service states no unit" }
+      - { name: length, description: "Lines and polygons: path length; the service states no unit" }
+      - { name: width, description: "Lines and polygons: path width; the service states no unit" }
+      - { name: injuries, units: "count", description: "Injuries" }
+      - { name: fatalities, units: "count", description: "Lines and polygons: fatalities; points call it deaths" }
+      - { name: image, description: "Points: file name of a photo taken there, often empty; the photos are noaa:nws-damage-photos" }
+      - { name: comments, description: "Surveyor's notes" }
+    adapter: usdata.providers.noaa.nws_damage:NwsDamageSurveys
+
+  - id: noaa:nws-damage-photos
+    provider: noaa
+    status: available
+    system: noaa:dat
+    domain: severe-weather
+    since: "0.34"
+    title: NWS Damage Survey Photos
+    description: >-
+      Photographs attached to the damage points of National Weather Service damage
+      surveys, from the Damage Assessment Toolkit's public ArcGIS feature service. Listing
+      finds the points whose storm time falls in a UTC window of at most seven days,
+      optionally in a box or by office and EF rating, then lists their attachments; each
+      JPEG or PNG is one asset with its listed size, its point's location and storm time,
+      and the point's id for joining to noaa:nws-damage-surveys. Older points often carry a
+      small thumbnail beside each photo, listed as served. The few files attached to
+      tracks and areas, mostly PDFs and other images, are not served.
+    keywords: [tornado, damage survey, photos, images, damage assessment toolkit, dat, ef scale, nws, arcgis]
+    protocol: http
+    homepage: https://apps.dat.noaa.gov/StormDamage/DamageViewer/
+    license: >-
+      Public domain as NWS information; third-party images are used under their providers'
+      licenses
+    temporal_extent: { start: "1974-04-03T00:00:00Z" }
+    capabilities: { spatial_subset: true, temporal_subset: true, variable_subset: false }
+    summary: Photos from NWS damage surveys, one file per damage point attachment
+    formats: [JPEG, PNG]
+    selection: Each JPEG or PNG attached to a damage point with storm time in an inclusive UTC window of at most 7 days
+    inputs: Both timestamps; optionally a bbox, offices, EF ratings, undated points, or as_of
+    reader: null
+    guide: docs/providers/noaa-nws-damage-photos.md
+    examples:
+      - examples/datasets/noaa-nws-damage-photos/noaa-nws-damage-photos.ipynb
+    resolution:
+      spatial: "One photo or more per surveyed damage point"
+      temporal: "The storm time of the point the photo is attached to, in UTC"
+    update_frequency: >-
+      As offices finish surveys, with no schedule: "Once quality-controlled, the data
+      becomes available externally via a
+      public web portal, as well as public Geographic Information System (GIS) services"
+    citation: >-
+      NOAA National Weather Service, Damage Assessment Toolkit (DamageViewer feature
+      service) damage point photographs, accessed via usdata
+    terms: https://www.weather.gov/disclaimer
+    variables:
+      - { name: image, description: "The attached photograph, JPEG or PNG, as served" }
+    limits: { max_window: P7D }
+    adapter: usdata.providers.noaa.nws_damage:NwsDamagePhotos
+
   - id: noaa:hurdat2
     provider: noaa
     status: available
@@ -1939,6 +2094,67 @@ datasets:
       temporal_subset: false
       variable_subset: true
 
+  - id: noaa:emergency-response-imagery
+    provider: noaa
+    status: available
+    domain: natural-hazards
+    since: "0.34"
+    title: NGS Emergency Response Imagery
+    description: >-
+      Aerial imagery the National Geodetic Survey flew after hurricanes, floods, fires, and
+      tornadoes, and in yearly pre-event coastal surveys, in the public noaa-eri-pds S3 bucket
+      (NOAA Open Data Dissemination). The bucket holds 52 event folders from Hurricane Katrina
+      (2005) onward; only three are tornadoes (Joplin 2011, Illinois 2015, Nashville 2020). Each
+      event holds folders of GeoTIFFs, usually one per flight: mosaic tiles named by their
+      north-west corner, 90 or 45 arc-seconds on a side (2.5 km UTM tiles in 2009 and 2011),
+      from 0.15 MB to 1.7 GB each, plus individually georeferenced frames in several events
+      and one flight mosaic per folder in some recent ones. Select an event and optionally its
+      folders; a bbox keeps files by the footprint their names give, and folders whose file names
+      give none refuse it. Files are fetched whole and no reader is bundled.
+    keywords:
+      - aerial imagery
+      - disaster response
+      - damage assessment
+      - hurricanes
+      - tornado
+      - floods
+      - orthomosaic
+      - geotiff
+      - ngs
+      - pre-event
+    protocol: s3
+    homepage: https://storms.ngs.noaa.gov/
+    license: US Government Work (public domain)
+    spatial_extent: { west: -124.8, south: 17.6, east: -64.5, north: 48.5 }
+    temporal_extent: { start: "2005-08-30T00:00:00Z" }
+    capabilities: { spatial_subset: true, temporal_subset: false, variable_subset: false }
+    summary: Post-event aerial imagery tiles
+    formats: [GeoTIFF]
+    selection: >-
+      Whole GeoTIFFs of one event folder, optionally by subfolder and by the footprint that tile
+      names give
+    inputs: An event folder; optionally collection folders and a bbox
+    reader: null
+    guide: docs/providers/noaa-emergency-response-imagery.md
+    examples:
+      - examples/datasets/noaa-emergency-response-imagery/noaa-emergency-response-imagery.ipynb
+    resolution:
+      spatial: >-
+        Approximate ground sample distance stated per event, such as 35 cm (1.14 feet) for
+        Katrina and Joplin, ~15 cm for Nashville, and 15 - 30 cm for Helene
+      temporal: One or more flights per event, one folder per flight
+    update_frequency: Manually when needed
+    citation: >-
+      National Geodetic Survey, NOAA Emergency Response Imagery, accessed via usdata from
+      https://registry.opendata.aws/noaa-eri
+    terms: https://www.noaa.gov/information-technology/open-data-dissemination
+    variables:
+      - { name: Red, description: "Red band, 8-bit" }
+      - { name: Green, description: "Green band, 8-bit" }
+      - { name: Blue, description: "Blue band, 8-bit" }
+      - { name: Alpha, description: "Transparency band marking imaged pixels, in the collections that carry one" }
+    adapter: usdata.providers.noaa.eri:EmergencyResponseImagery
+
   - id: fema:disaster-declarations
     provider: fema
     status: available