dpolcat 0.2.0.dev1__py3-none-any.whl

dpolcat/utils.py

@@ -0,0 +1,412 @@
+import marimo
+
+__generated_with = "0.25.1"
+app = marimo.App(width="medium")
+
+with app.setup:
+    from collections import defaultdict
+    from collections.abc import Iterable
+    from functools import partial
+    from pathlib import Path
+    import tomllib
+    from typing import Literal
+    import xml.etree.ElementTree as ET
+
+    import bokeh.models
+    import dask
+    # from dask.diagnostics import ProgressBar
+    import geopandas as gpd
+    import numpy as np
+    import odc.stac
+    # from osgeo import gdal
+    import pandas as pd
+    import pystac
+    import pystac_client
+    # import planetary_computer
+    import shapely.geometry as sg
+    import xarray
+    import holoviews as hv
+    import hvplot.pandas
+    import hvplot.xarray
+    import rioxarray
+    import zarr
+
+    import dpolcat.core as dpolcat
+    import dpolcat.fast_frost as ff
+    from dpolcat.pc_rio_driver import PCRioDriver
+
+    # gdal.UseExceptions()
+
+    # hv.extension("bokeh")
+
+    # Want total coverage, with some wiggle room due to reprojection
+    MIN_COVERAGE_FRACTION = 0.99
+
+
+@app.cell
+def _():
+    import marimo as mo
+
+    return (mo,)
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(r"""
+    Luke McQuade, October 2026
+
+    # `dpolcat utils`
+
+    Utility/helper functions etc. for working with *dpolcat* in various settings.
+
+    /// admonition | Info
+
+    This notebook is designed around the concept of [marimo reusable functions](https://docs.marimo.io/guides/reusing_functions/). Be careful changing things!
+
+    ///
+    """)
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(r"""
+    ## Search
+
+    Finding Sentinel-1 STAC items that work well together for a *dpolcat* time series is a little more involved than for optical imagery. For consistent comparisons over time, especially in hilly/mountainous areas of interest (AoIs), the sensing geometry should be as similar as possible. I.e., items should should have the same orbit direction and relative orbit number. However, items covering the AoI with the same `datatake_id` (continuous take over a day's orbit), can be safely combined (which are then referred to as 'scenes'). These are the rules governing the search functions here.
+    """)
+    return
+
+
+@app.function
+def find_stac_items(
+    aoi: gpd.GeoDataFrame, date_range: str
+) -> gpd.GeoDataFrame:
+    """
+    Find Sentinel-1 RTC scenes from the Microsoft Planetary Computer catalog intersecting the `aoi`.
+    - `aoi` is a single-row GeoDataFrame.
+    - `date_range` is a string with format `{from_date}/{to_date}`, e.g., "2020-01-01/2020-03-31".
+
+    Returns a GeoDataFrame representing the STAC items, ordered by datetime. The original pystac.Item is stashed in the _item field.
+    """
+    STAC_COLLECTION = "sentinel-1-rtc"
+
+    catalog = pystac_client.Client.open(
+        "https://planetarycomputer.microsoft.com/api/stac/v1",
+    )
+
+    if len(aoi.geometry) != 1:
+        raise ValueError(
+            "Expected only a single geometry for the given AoI."
+        )
+
+    search = catalog.search(
+        collections=[STAC_COLLECTION],
+        datetime=date_range,
+        intersects=aoi.to_crs("EPSG:4326").geometry[0],
+    )
+
+    items = search.item_collection()
+
+    items_gdf = gpd.GeoDataFrame.from_features(items, crs="EPSG:4326")
+
+    # Add the item ID, which isn't included in the above.
+    items_gdf["id"] = [item.id for item in items]
+
+    # datetime is originally a string, so convert it.
+    items_gdf["datetime"] = pd.to_datetime(items_gdf["datetime"])
+
+    # Stash the original item (a shallow copy) for later.
+    items_gdf["_item"] = items
+
+    return items_gdf.sort_values("datetime")
+
+
+@app.function
+def calc_coverages(
+    items_gdf: gpd.GeoDataFrame, aoi: gpd.GeoDataFrame
+) -> pd.DataFrame:
+    """
+    Calculate full vs partial scene coverages by orbit for the given AoI, grouped by `datatake_id`, i.e.,
+    approx. same time.
+    - `items_gdf` is a GeoDataFrame representing the STAC items.
+    - `aoi` is a single-row GeoDataFrame.
+
+    Returns a DataFrame containing the orbit, and number of `full_scenes` and `partial_scenes` covered,
+    ordered by number of `full_scenes` (highest first).
+    """
+
+    # Reproject to UTM for area calculation, which is planar.
+    utm_crs = items_gdf.estimate_utm_crs()
+    items_gdf = items_gdf.to_crs(utm_crs)
+    aoi = aoi.to_crs(utm_crs)
+
+    intersection = items_gdf.dissolve(
+        by=["sat:orbit_state", "sat:relative_orbit", "s1:datatake_id"]
+    ).intersection(aoi.geometry[0])
+    overlap_fraction = (intersection.area / aoi.area[0]).rename(
+        "overlap_fraction"
+    )
+
+    full_scenes_count = (
+        overlap_fraction[overlap_fraction >= MIN_COVERAGE_FRACTION]
+        .groupby(["sat:orbit_state", "sat:relative_orbit"])
+        .count()
+    )
+    partial_scenes_count = (
+        overlap_fraction[overlap_fraction < MIN_COVERAGE_FRACTION]
+        .groupby(["sat:orbit_state", "sat:relative_orbit"])
+        .count()
+    )
+
+    df_coverages = pd.DataFrame(
+        {
+            "full_scenes": full_scenes_count,
+            "partial_scenes": partial_scenes_count,
+        }
+    ).fillna(0)
+
+    return df_coverages.sort_values("full_scenes", ascending=False)
+
+
+@app.function
+def select_by_orbit(
+    items_gdf,
+    orbit_direction: Literal["ascending", "descending"],
+    relative_orbit: int,
+) -> Iterable[pystac.Item]:
+    """
+    Selects the STAC items for the given orbit direction and relative orbit.
+    - `items_gdf` is a GeoDataFrame representing the source STAC items.
+    - `orbit_direction` is either "ascending" or "descending".
+    - `relative_orbit` is the relative orbit number.
+    Returns a filtered collection of `pystac.Item`s.
+    """
+    return items_gdf.loc[
+        (items_gdf["sat:orbit_state"] == orbit_direction)
+        & (items_gdf["sat:relative_orbit"] == relative_orbit)
+    ]
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(r"""
+    ### Test/demo
+    """)
+    return
+
+
+@app.cell
+def _():
+    test_aoi = gpd.read_file("gis/aois/salzburg_south_4326.gpkg")
+    test_date_range = "2025-03-01/2025-03-31"
+    return test_aoi, test_date_range
+
+
+@app.cell
+def _(test_aoi, test_date_range):
+    test_items = find_stac_items(aoi=test_aoi, date_range=test_date_range)
+    return (test_items,)
+
+
+@app.cell
+def _(test_items):
+    test_items
+    return
+
+
+@app.cell
+def _(test_aoi, test_items):
+    test_scene_coverages = calc_coverages(test_items, aoi=test_aoi)
+    test_scene_coverages
+    return (test_scene_coverages,)
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(r"""
+    Find the orbit with the highest number of fully-covering scenes.
+    """)
+    return
+
+
+@app.cell
+def _(test_scene_coverages):
+    _best_match = test_scene_coverages.reset_index().iloc[0]
+    test_best_match = _best_match["sat:orbit_state"], _best_match["sat:relative_orbit"]
+    test_best_match
+    return (test_best_match,)
+
+
+@app.cell
+def _(test_best_match, test_items):
+    test_sel_items = select_by_orbit(test_items,*test_best_match)
+    test_sel_items["_item"].values
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(r"""
+    ## Analysis
+    """)
+    return
+
+
+@app.function
+def frost_xr(
+    da: xarray.DataArray, nanny=True, damping_factor=2.0, win_size=5
+) -> xarray.DataArray:
+    """Apply 2D (y, x) Frost filter to 3D (y, x, time) xarray DataArray; See underlying filter function for damping_factor and win_size."""
+    # Mostly Claude (Opus 5.5)
+    f = ff.frost_filter_nanny if nanny else ff.frost_filter_fast
+    g = np.vectorize(
+        partial(f, damping_factor=damping_factor, win_size=win_size),
+        signature="(m,n)->(m,n)",
+    )
+
+    # In case da holds a materialized np array, operate directly...
+    if da.chunks is None:
+        return da.copy(data=g(da.data))
+
+
+    # ...otherwise, set up parallel calc with overlapping windows
+    # (because it's a neighbourhood operation).
+    h = win_size // 2
+    data = da.data.map_overlap(
+        g,
+        depth=(0, h, h),
+        boundary="none",
+        meta=np.array((), dtype=da.dtype),
+    )
+    return da.copy(data=data)
+
+
+@app.function
+def normalize_db(arr, min=-30, max=10):
+    """Shift, scale and clip the input array in dB to the range 0 to 1"""
+    # E.g. range -30 to +10 -> 0.0 to 1.0
+
+    return ((arr - min) / (max - min)).clip(0.0, 1.0)
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(r"""
+    ## Visualization
+    """)
+    return
+
+
+@app.function
+def make_cmap():
+    """Make a HoloViews-compatible categorical colormap (string key is needed)."""
+    return {
+        str(i): list(dpolcat.color_list[i])
+        for i in range(len(dpolcat.color_list))
+    }
+
+
+@app.function
+def get_hex_colors(cat_indexes = range(dpolcat.NUM_CATEGORIES)):
+    """Make a hex-string-format colour list for the given categories."""
+    return [
+        f"#{r:02x}{g:02x}{b:02x}"
+        for r, g, b in dpolcat.color_list[cat_indexes]
+    ]
+
+
+@app.function
+def make_legend(size=350):
+    """Create a legend (in the form of a Holoviews plot)."""
+    n_steps = 100
+    steps = np.linspace(-30.5, 10.5, n_steps)
+    vv_ramp = np.vstack([steps] * n_steps)
+    vh_ramp = vv_ramp.T
+
+    cat_values = xarray.DataArray(
+        dpolcat.categorize_np(vv_ramp, vh_ramp),
+        coords=[steps, steps],
+        dims=["VH", "VV"],
+    )
+
+    label_data = []
+    for i in range(1, dpolcat.NUM_CATEGORIES):
+        box = cat_values.where(cat_values == i, drop=True)
+        center_x = float(box["VV"].mean())
+        center_y = float(box["VH"].mean())
+        label_data.append((center_x, center_y, str(i)))
+
+    labels = hv.Labels(
+        label_data, kdims=["VV", "VH"], vdims=["text"]
+    ).opts(text_color="black")
+
+    return (
+        cat_values.astype("str")
+        .hvplot(cmap=make_cmap())
+        .opts(
+            width=size,
+            height=size,
+            title="NoData (0) is black",
+            xlabel="VV (dB)",
+            ylabel="VH (dB)",
+        )
+        * labels
+    )
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(r"""
+    ### Test/demo
+    """)
+    return
+
+
+@app.cell
+def _():
+    make_legend()
+    return
+
+
+@app.cell
+def _():
+    _n = dpolcat.NUM_CATEGORIES
+    _color = get_hex_colors(range(_n))
+    pd.DataFrame(
+        {
+            "category": range(_n),
+            "count": [10] * _n,
+            "color": _color,
+        }
+    ).hvplot.bar(x="category", y="count", color="color")
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(r"""
+    ### QGIS Colour Map
+
+    Create a QGIS Color Map file that can be used for dpolcat layer symbology.
+    """)
+    return
+
+
+@app.function
+def write_qgis_colormap():
+    lines = []
+    
+    for i, c in enumerate(dpolcat.color_list):
+        label = i
+        a = 255
+        r, g, b = c
+        lines.append(f"{i} {r} {g} {b} {a} {label}\n")
+    
+    f = open("dpolcat-qgis-cmap.clr", "w")
+    f.writelines(lines)
+    f.close()
+
+
+if __name__ == "__main__":
+    app.run()

dpolcat-0.2.0.dev1.dist-info/METADATA

@@ -0,0 +1,202 @@
+Metadata-Version: 2.5
+Name: dpolcat
+Version: 0.2.0.dev1
+Summary: Semantic categorization of dual-polarimetric synthetic aperture radar (SAR) imagery.
+Project-URL: Homepage, https://github.com/lukesdm/dpolcat
+Project-URL: Issues, https://github.com/lukesdm/dpolcat/issues
+Author-email: Luke McQuade <2824165+lukesdm@users.noreply.github.com>
+License-Expression: Apache-2.0
+License-File: LICENSE
+Requires-Python: >=3.12
+Requires-Dist: geopandas<2,>=1.2.0
+Requires-Dist: marimo<0.26,>=0.25.1
+Requires-Dist: numba<0.69,>=0.68.0
+Requires-Dist: odc-stac<0.6,>=0.5.3
+Requires-Dist: pandas<4,>=3.0.6
+Requires-Dist: planetary-computer<2,>=1.0.0
+Requires-Dist: pystac-client<0.10,>=0.9.0
+Requires-Dist: rioxarray<0.24,>=0.23.0
+Requires-Dist: xarray<2027,>=2026.9.0
+Provides-Extra: gdal
+Requires-Dist: gdal<4,>=3.13.3; extra == 'gdal'
+Description-Content-Type: text/markdown
+
+# dpolcat
+
+*dpolcat* is a tool for the semantic categorization of dual-polarimetric synthetic aperture radar (SAR) imagery. (Specifically, Sentinel-1 with VV and VH polarizations at present.)
+
+It is inspired by Dr. Andrea Baraldi's [Satellite Image Automatic Mapper™](http://siam.andreabaraldi.com/) (SIAM™) concept of *spectral categories* for optical data, the [Sen2Cube](https://www.sen2cube.at/) project, and Luuk van Der Meer's [Semantique](https://github.com/zgis/semantique).
+
+Status: This is a research project with occasional updates.
+
+## Overview
+
+Different objects on the earth's surface scatter the microwave signals that radar satellites emit and receive, in different ways.
+
+![Scatterer types](media/scatterer-types.png)
+
+We bin the backscatter received in the same- (VV) and cross-polarized (VH) signals into distinct *polarimetric categories* that give a hint at what's on the ground.
+
+![Our polarimetric categories](media/categories.png)
+
+TODO: 👆 update this (new categories).
+
+We can then make inferences based on how these categories are distributed in an area, or change over time.
+
+
+![Demo panels, flood mapping](media/demo-panels.png)
+
+TODO: 👆 update this (colour scheme, result output).
+
+[*]: [Copernicus EMSR517](https://emergency.copernicus.eu/mapping/ems-product-component/EMSR517_AOI06_DEL_MONIT01_r1_RTP03/1)
+
+
+This is implemented in a Python library that performs scaling, speckle-filtering, and then categorization, using a decision-tree algorithm.
+
+![Block diagram](media/block-diagram.png).
+
+TODO: 👆 scaler should be linear to dB; add speckle filter.
+
+It uses the [Numba](https://numba.pydata.org/) just-in-time compiler to increase performance, and [Dask](https://www.dask.org) for parallelization.
+
+### Colour palette
+
+A note on the colour palette. The choices **loosely** follow these rules:
+- Natural phenomena map to pale colours.
+- Man-made phenonema map to bright colours.  
+
+Sometimes the colours resemble certain land cover types. This may not be coincidence. BUT, there's no one-to-one mapping of land cover type to category/colour.
+
+
+### Category numbering
+
+The numbering of the categories, i.e., their indices, has a property with an interesting side effect: they loosely increase in order of  magnitude of the VV+VH vector they represent. This means modelling and visualization can sometimes use the category index alone as an input. E.g.,
+
+TODO: B/W & viridis map of Graz + caption.
+
+## Environment
+
+Launch marimo notebooks via either:
+- Pixi - run `pixi run marimo edit` (a browser opens automatically).
+- Docker - use locally with, e.g., `docker compose up` and browse to localhost:8080. ⚠️ This opens a notebook server without authentication, make sure to be behind a properly configured firewall.
+
+
+## Contents
+
+    '🟢' indicates the file can be run as a marimo notebook.
+
+#### 📄 `dpolcat.py`
+
+The *dpolcat* module, containing the main algorithms/functions for performing polarimetric categorization.
+
+
+#### 📄🟢 `dpolcat.processor.py`
+
+A parameterized notebook containing a batch processing workflow.
+
+
+#### 📄 `dpolcat_modelling.py`
+
+Helpers for creating semantic models with categories.
+
+
+#### 📄🟢 `dpolcat.utils.py`
+
+Contains reusable utility/helper functions etc. for working with *dpolcat* in various settings.
+
+
+#### 📄🟢 `demo_01_intro.py`
+
+An introduction to *dpolcat*.
+
+
+#### 📄🟢 `demo_02_flood.py`
+
+Demonstration of a simple flood modelling and mapping workflow using *dpolcat*.
+
+
+#### 📄🟢 `demo_03_batch_processing.py`
+
+A batch processing/datacube-writing demonstration.
+
+
+#### 📄🟢 `demo_04_urban.py`
+
+Demonstration of some simple urban change modelling and mapping.  
+
+
+#### 📄🟢 `gis/inverness_*.py`
+
+Exploratory analysis of SAR signals and dpolcat processing in a study area of Inverness, Scotland.
+
+
+#### 📄 `pc_rio_driver.py`
+
+A utility class for [odc-stac](https://odc-stac.readthedocs.io/en/latest/intro.html) which takes care of Planetary Computer asset signing.
+
+
+#### 📄 `fast_frost.py`
+
+A numba-based implementation of a Frost speckle filter.
+
+
+#### 📁 `gis/duisburg`
+
+Supplementary folder for the demo notebook's flood mapping example, containing a QGIS project and associated data for accuracy assessment.
+
+
+#### 📁 `gis/graz`
+
+Supplementary folder inc. QGIS project for the Graz-related demos. 
+
+
+#### 📁 `archive`
+
+Some things based on earlier versions, which may be useful for additional context.
+
+
+
+
+
+
+
+#### 📄 `media/EARSeL2024-Slides.pdf`
+
+Slides presented at the 43rd EARSeL Symposium in Manchester, UK, 17-June-2024. (Based on an older version.)
+
+
+#### 📄 `media/poster.pdf`
+
+A simple poster about the project presented internally (based on an even older version).
+
+
+## Credits
+
+Imagery: Contains modified Copernicus Sentinel data, processed by ESA and others.
+
+Flood reference: [Copernicus Emergency Mapping EMSR517](https://emergency.copernicus.eu/mapping/ems-product-component/EMSR517_AOI06_DEL_MONIT01_r1_RTP03/1)
+
+Graz aerial imagery: TODO: add source.
+
+## Authors
+
+Created by Luke McQuade at [Z_GIS](https://www.plus.ac.at/geoinformatik/?lang=en), originally as part of the the Applied Geoinformatics MSc programme.
+
+## Acknowledgements
+
+We would like to greatly thank the members of the [EO Analytics Group](https://www.plus.ac.at/geoinformatik/research/research-areas/eo-analytics/?lang=en) and Dr. Zhara Dabiri of the [Risk, Hazard and Climate Lab](https://www.plus.ac.at/geoinformatik/research/research-areas/risk-hazard-climate/?lang=en) for their ideas and support, and Assoc. Prof. Hermann Klug and Dr. Bernhard Zagel for their instruction and continuous feedback.
+
+Further developments have been part of my masters thesis, supervised by Prof. Dr. Dirk Tiede, Dr. Martin Sudmanns, and Dr. Zhara Dabiri.  
+
+
+## References
+
+Augustin, H., Sudmanns, M., Tiede, D., Lang, S., & Baraldi, A. (2019). Semantic Earth observation data cubes. Data, 4(3), 102.
+DOI: 10.3390/data4030102
+
+Baraldi, A., Humber, M.L., Tiede, D., Lang, S. (2018). GEO-CEOS stage 4 validation of the Satellite Image Automatic Mapper lightweight computer program for ESA Earth observation level 2 product generation – Part 2: Validation. Cogent Geosci. 4, 1–52.
+DOI: 10.1080/23312041.2018.1467254
+
+
+Meyer, F. (2019), Ch. 2, The SAR Handbook: Comprehensive Methodologies for Forest Monitoring and Biomass Estimation.
+DOI: 10.25966/nr2c-s697

dpolcat-0.2.0.dev1.dist-info/RECORD

@@ -0,0 +1,11 @@
+dpolcat/__init__.py,sha256=3JcOfymPgsGgWUAvwMAroqk1JdKGC99cnE77qqYQCfQ,92
+dpolcat/core.py,sha256=NzNtjK28KYITnIyFEFRpt37lRrkfsYD1ZDFlLtt6Gm4,4059
+dpolcat/fast_frost.py,sha256=tIzWu7w-Fv95DV-nFQU8oQoorepk44HKNyYTyte3bVw,6441
+dpolcat/modelling.py,sha256=A-yhtbhIL8sRx-nBRvbTU6xW52J8FCmhTEUMIlz29GM,3853
+dpolcat/pc_rio_driver.py,sha256=o8XdBI-q4HU8OX6fFpe90B45NgH8rKhlgkKmyA_a-EM,1021
+dpolcat/processor.py,sha256=RIQ-beRJQcfdZZJcbgPTRjLYuVY-m4zWpJL9GxV3Kww,22119
+dpolcat/utils.py,sha256=U6JwCAjHqtZUI4Qw9ijFBlwMxyceRXBwcAK2XMroDMQ,11244
+dpolcat-0.2.0.dev1.dist-info/METADATA,sha256=6yu5q1fnND-1ZdUjC2yLgi-5Lr4AejZjqeNvUK6xiwE,7239
+dpolcat-0.2.0.dev1.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
+dpolcat-0.2.0.dev1.dist-info/licenses/LICENSE,sha256=fkPhb5zI5mg5fHiiRNNb-HVxPHsFF0tsk9a_eFtMm2o,11541
+dpolcat-0.2.0.dev1.dist-info/RECORD,,

dpolcat-0.2.0.dev1.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.33.0
+Root-Is-Purelib: true
+Tag: py3-none-any