paddocktimeseries 0.8.0__py3-none-any.whl

PaddockTS/FractionalCover/__init__.py

@@ -0,0 +1 @@
+from .compute_fractional_cover import compute_fractional_cover

PaddockTS/FractionalCover/_unmix.py

@@ -0,0 +1,45 @@
+"""Fractional cover unmixing.
+
+Inference logic and the bundled tflite model files in `_models/` are adapted
+from the fractionalcover3 package by Robert Denham, distributed under the
+MIT License. See PaddockTS/LICENSES/fractionalcover3.LICENSE.
+"""
+from importlib import resources
+
+import numpy as np
+from tensorflow import lite as tflite
+
+
+_MODELS = (
+    "fcModel_32x32x32.tflite",
+    "fcModel_64x64x64.tflite",
+    "fcModel_256x64x256.tflite",
+    "fcModel_256x128x256.tflite",
+)
+
+
+def get_model(n: int = 2):
+    """Load the n-th bundled fractional cover tflite model (1-indexed, 1-4)."""
+    path = resources.files(__package__) / "_models" / _MODELS[n - 1]
+    return tflite.Interpreter(model_path=str(path))
+
+
+def unmix_fractional_cover(surface_reflectance, fc_model, in_null=0, out_null=0):
+    """Unmix a 6-band surface reflectance array into bare/green/non-green fractions.
+
+    The blue band is dropped before inference (the model was trained on the
+    remaining 5 bands). Returns a (3, nrows, ncols) float32 array.
+    """
+    inshape = surface_reflectance[1:].shape
+    ref_data = np.reshape(surface_reflectance[1:], (inshape[0], -1)).T
+
+    input_details = fc_model.get_input_details()
+    output_details = fc_model.get_output_details()
+    fc_model.resize_tensor_input(input_details[0]["index"], ref_data.shape)
+    fc_model.allocate_tensors()
+    fc_model.set_tensor(input_details[0]["index"], ref_data.astype(np.float32))
+    fc_model.invoke()
+    fc_layers = fc_model.get_tensor(output_details[0]["index"]).T
+    output_fc = np.reshape(fc_layers, (3, inshape[1], inshape[2]))
+    output_fc[output_fc == in_null] = out_null
+    return output_fc

PaddockTS/FractionalCover/check_if_valid_fractional_cover_exists.py

@@ -0,0 +1,49 @@
+"""Cache-validity check for the computed fractional-cover zarr.
+
+A valid cache requires both the zarr directory and a ``_SUCCESS`` marker
+file *inside* it (written by :func:`compute_fractional_cover` only after
+the zarr write completes). Without the marker the zarr is treated as a
+partial / interrupted write and recomputed.
+"""
+
+from os.path import exists
+
+
+def check_if_valid_fractional_cover_exists(fractional_cover_path: str) -> bool:
+    """Return True iff a successfully-written fractional-cover zarr is on disk."""
+    return exists(fractional_cover_path) and exists(f'{fractional_cover_path}/_SUCCESS')
+
+
+def test_no_zarr_no_marker():
+    import tempfile
+    with tempfile.TemporaryDirectory() as tmpdir:
+        return check_if_valid_fractional_cover_exists(f'{tmpdir}/missing.zarr') is False
+
+
+def test_zarr_without_marker():
+    import tempfile, os
+    with tempfile.TemporaryDirectory() as tmpdir:
+        zarr_path = f'{tmpdir}/data.zarr'
+        os.makedirs(zarr_path)
+        return check_if_valid_fractional_cover_exists(zarr_path) is False
+
+
+def test_zarr_with_marker():
+    import tempfile, os
+    with tempfile.TemporaryDirectory() as tmpdir:
+        zarr_path = f'{tmpdir}/data.zarr'
+        os.makedirs(zarr_path)
+        open(f'{zarr_path}/_SUCCESS', 'w').close()
+        return check_if_valid_fractional_cover_exists(zarr_path) is True
+
+
+def test():
+    return all([
+        test_no_zarr_no_marker(),
+        test_zarr_without_marker(),
+        test_zarr_with_marker(),
+    ])
+
+
+if __name__ == '__main__':
+    print(test())

PaddockTS/FractionalCover/compute_fractional_cover.py

@@ -0,0 +1,208 @@
+"""Spectral unmixing of Sentinel-2 reflectance into bg / pv / npv fractions.
+
+The model is a small TFLite MLP adapted from
+`fractionalcover3 <https://github.com/jrsrp/fractionalcover3>`_ by
+Robert Denham (MIT-licensed; see
+``PaddockTS/LICENSES/fractionalcover3.LICENSE``). Four model variants
+ship with the package, indexed ``n=1..4`` from least to most complex;
+``n=4`` is the default and most accurate.
+
+Output bands:
+
+- ``bg`` — bare ground fraction
+- ``pv`` — green (photosynthetic) vegetation fraction
+- ``npv`` — non-green (non-photosynthetic) vegetation fraction
+
+Fractions are produced per-pixel per-timestep and persisted to
+``Paths(troi).fractional_cover`` as Zarr v2. Input reflectance comes
+from the machine-wide pysentinel2 cube (cloud-masked, on read).
+"""
+
+import os
+import warnings
+
+os.environ.setdefault('TF_CPP_MIN_LOG_LEVEL', '3')
+warnings.filterwarnings('ignore', module='tensorflow')
+warnings.filterwarnings('ignore', module='keras')
+
+import numpy as np
+import xarray as xr
+from os import makedirs
+from os.path import exists
+from datetime import datetime
+from xarray import Dataset
+from troi import Troi
+from PaddockTS.paths import Paths
+from PaddockTS.FractionalCover.check_if_valid_fractional_cover_exists import check_if_valid_fractional_cover_exists
+
+
+BANDS = ['nbart_blue', 'nbart_green', 'nbart_red', 'nbart_nir_1', 'nbart_swir_2', 'nbart_swir_3']
+
+
+def compute_fractional_cover(troi: Troi, ds_sentinel2=None, model_n: int = 4, correction: bool = False):
+    """Run the TFLite unmixing model over every Sentinel-2 timestep.
+
+    Stacks the six Sentinel-2 SR bands into a ``(time, band, y, x)``
+    tensor, scales reflectance, and invokes the chosen model variant
+    once per timestep. The result is written to
+    ``Paths(troi).fractional_cover`` and returned as an xarray Dataset
+    with ``bg``, ``pv``, ``npv`` data variables on dims
+    ``(time, y, x)``.
+
+    Args:
+        troi: The :class:`troi.Troi`.
+        ds_sentinel2: Optional in-memory Sentinel-2 dataset. If ``None``,
+            the cloud-masked window is read from the machine-wide
+            pysentinel2 cube (downloading only what's missing). Must
+            contain the six bands in :data:`BANDS`.
+        model_n: Which bundled model variant to use (``1..4``). Higher
+            ``n`` is more accurate but slower; ``n=4`` is the default.
+        correction: If ``True``, apply per-band sensor calibration
+            factors (gains and offsets fitted in the upstream
+            fractionalcover3 work) instead of the simple ``* 0.0001``
+            DN-to-reflectance scaling. Use only when your inputs match
+            the calibration assumptions of the original model.
+
+    Returns:
+        xarray.Dataset: Dataset with variables ``bg``, ``pv``, ``npv``
+        on dims ``(time, y, x)``. Also persisted to
+        ``Paths(troi).fractional_cover``.
+    """
+    from PaddockTS.FractionalCover._unmix import unmix_fractional_cover, get_model
+
+    fractional_cover_path = Paths(troi).fractional_cover
+    if check_if_valid_fractional_cover_exists(fractional_cover_path):
+        try:
+            return xr.open_zarr(fractional_cover_path, chunks=None, decode_coords='all')
+        except Exception as e:
+            print(f'Fractional-cover cache at {fractional_cover_path} unreadable ({e}); recomputing')
+
+    if ds_sentinel2 is None:
+        from pysentinel2.cube import Cube
+        ds = Cube(config=troi.config).get_ds_troi(troi, clean=True)
+    else:
+        ds = ds_sentinel2
+
+    if correction:
+        factors = (np.array([0.9551, 1.0582, 0.9871, 1.0187, 0.9528, 0.9688]) +
+                   np.array([-0.0022, 0.0031, 0.0064, 0.012, 0.0079, -0.0042])
+                   ).astype(np.float32)[:, np.newaxis, np.newaxis]
+    else:
+        factors = np.float32(0.0001)
+
+    # Stream in small time batches, appending each to the zarr as it's
+    # unmixed. Materialising the whole window at once — six bands x every
+    # timestep as float — needs >10 GB for a multi-year troi and OOMs
+    # 8 GB machines; a 32-frame batch stays around 200 MB regardless of
+    # the time range. The model is loaded once, not per timestep.
+    #
+    # Unmixing runs serially on purpose: 4 threads with per-thread
+    # interpreters measured 615.2 s vs 618.3 s serial on an 8-year window
+    # (340% CPU for zero wall gain — the matmul is memory-bandwidth
+    # bound). If this step ever needs to be faster, batch several frames
+    # into one Invoke rather than adding threads.
+    model = get_model(n=model_n)
+    makedirs(os.path.dirname(fractional_cover_path), exist_ok=True)
+    timestamp = datetime.utcnow().isoformat() + 'Z'
+
+    n_time = ds.sizes['time']
+    batch = 32
+    for t0 in range(0, n_time, batch):
+        sub = ds.isel(time=slice(t0, t0 + batch))
+        inref = np.stack([sub[b].values for b in BANDS], axis=1).astype(np.float32)
+        # Cleaned cube bands are int16 with masked pixels at the nodata
+        # sentinel; convert to NaN so the unmixing propagates missingness
+        # instead of ingesting -999 as reflectance.
+        for bi, b in enumerate(BANDS):
+            nodata = ds[b].attrs.get('nodata')
+            if nodata is not None:
+                inref[:, bi][inref[:, bi] == float(nodata)] = np.nan
+        inref[inref == 0] = np.nan
+        inref *= factors
+        fractions = np.empty((inref.shape[0], 3, inref.shape[2], inref.shape[3]),
+                             dtype=np.float32)
+        for i in range(inref.shape[0]):
+            fractions[i] = unmix_fractional_cover(inref[i], fc_model=model)
+
+        coords = {'time': sub.time, 'y': ds.y, 'x': ds.x}
+        batch_ds = xr.Dataset({
+            'bg': xr.DataArray(fractions[:, 0], dims=['time', 'y', 'x'], coords=coords),
+            'pv': xr.DataArray(fractions[:, 1], dims=['time', 'y', 'x'], coords=coords),
+            'npv': xr.DataArray(fractions[:, 2], dims=['time', 'y', 'x'], coords=coords),
+        })
+        if t0 == 0:
+            batch_ds = batch_ds.assign_attrs(fractional_cover_computed_at=timestamp)
+            batch_ds.to_zarr(fractional_cover_path, mode='w', zarr_format=2,
+                             encoding={v: {'chunks': (1, ds.sizes['y'], ds.sizes['x'])}
+                                       for v in ('bg', 'pv', 'npv')})
+        else:
+            batch_ds.to_zarr(fractional_cover_path, append_dim='time', zarr_format=2)
+
+    # Touch _SUCCESS *after* the zarr write completes; its presence is what
+    # the next call uses as the cache-validity check.
+    with open(f'{fractional_cover_path}/_SUCCESS', 'w') as f:
+        f.write(timestamp)
+    # Re-open from disk: a lazy, zarr-backed view rather than the in-RAM
+    # batches — downstream consumers slice what they need.
+    return xr.open_zarr(fractional_cover_path, chunks=None, decode_coords='all')
+
+
+def _temp_troi():
+    import tempfile
+    from datetime import date
+    from troi import Config
+    tmpdir = tempfile.mkdtemp(prefix='paddockts_fc_test_')
+    cfg = Config(out_dir=tmpdir, tmp_dir=tmpdir)
+    return Troi(
+        bbox=[148.36265, -33.52606, 148.38265, -33.50606],
+        start=date(2024, 1, 1), end=date(2024, 1, 21),
+        stub=f'fc_{os.path.basename(tmpdir)}', config=cfg,
+    )
+
+
+def test_compute_writes_zarr_and_marker():
+    """First call computes, writes fractional_cover.zarr, and touches _SUCCESS."""
+    q = _temp_troi()
+    ds = compute_fractional_cover(q)
+    fc = Paths(q).fractional_cover
+    if not exists(fc):
+        return False
+    if not exists(f'{fc}/_SUCCESS'):
+        return False
+    return all(name in ds.data_vars for name in ('bg', 'pv', 'npv'))
+
+
+def test_repeated_call_uses_cache():
+    """Second call with same troi reuses the zarr (no rewrite)."""
+    q = _temp_troi()
+    compute_fractional_cover(q)
+    fc = Paths(q).fractional_cover
+    mtime_before = os.path.getmtime(fc)
+    compute_fractional_cover(q)
+    mtime_after = os.path.getmtime(fc)
+    return mtime_before == mtime_after
+
+
+def test_missing_marker_triggers_recompute():
+    """A cache with the zarr present but no _SUCCESS file is recomputed."""
+    q = _temp_troi()
+    compute_fractional_cover(q)
+    fc = Paths(q).fractional_cover
+    marker = f'{fc}/_SUCCESS'
+    os.remove(marker)
+    mtime_before = os.path.getmtime(fc)
+    compute_fractional_cover(q)
+    mtime_after = os.path.getmtime(fc)
+    return exists(marker) and mtime_after > mtime_before
+
+
+def test():
+    return all([
+        test_compute_writes_zarr_and_marker(),
+        test_repeated_call_uses_cache(),
+        test_missing_marker_triggers_recompute(),
+    ])
+
+
+if __name__ == '__main__':
+    print(test())

PaddockTS/PaddockSegmentation/_presegment.py

@@ -0,0 +1,130 @@
+import numpy as np
+import xarray as xr
+from numpy.typing import NDArray
+from datetime import datetime
+from os import makedirs
+from os.path import exists, dirname
+from troi import Troi
+from PaddockTS.paths import Paths
+from PaddockTS.PaddockSegmentation.check_if_valid_preseg_exists import check_if_valid_preseg_exists
+
+
+def completion(arr):
+    """Gap-fill NaN values in a (H, W, T) time series by forward-filling then backfilling."""
+    mask = np.isnan(arr)
+    idx = np.where(~mask, np.arange(mask.shape[-1]), 0)
+    np.maximum.accumulate(idx, axis=-1, out=idx)
+    i, j = np.meshgrid(
+        np.arange(idx.shape[0]), np.arange(idx.shape[1]), indexing="ij"
+    )
+    dat = arr[i[:, :, np.newaxis], j[:, :, np.newaxis], idx]
+    if np.isnan(np.sum(dat[:, :, 0])):
+        fill = np.nanmean(dat, axis=-1)
+        for t in range(dat.shape[-1]):
+            m = np.isnan(dat[:, :, t])
+            if m.any():
+                dat[m, t] = fill[m]
+            else:
+                break
+    return dat
+
+
+def fourier_mean(x, n=3, step=5):
+    """Extract n Fourier band means from a (H, W, T) time series -> (H, W, n)."""
+    spectra = np.abs(np.fft.fft(x, axis=2))
+    result = np.empty((x.shape[0], x.shape[1], n), dtype=np.float32)
+    for k in range(n):
+        result[:, :, k] = np.mean(spectra[:, :, 1 + k * step:(k + 1) * step + 1], axis=2)
+    return result
+
+
+def compute_ndwi_fourier(ds: xr.Dataset) -> NDArray[np.float32]:
+    """Compute NDWI from sentinel-2 bands, gap-fill, then Fourier summary -> (H, W, 3)."""
+    green = ds["nbart_green"].transpose("y", "x", "time").values.astype(np.float32)
+    nir = ds["nbart_nir_1"].transpose("y", "x", "time").values.astype(np.float32)
+
+    # 0 = unobserved; negative = the cleaned cube's int16 nodata sentinel
+    # (-999). Both are missing.
+    green[green <= 0] = np.nan
+    nir[nir <= 0] = np.nan
+    green /= 10000.0
+    nir /= 10000.0
+
+    ndwi_obs = (green - nir) / (green + nir)
+    ndwi_obs[~np.isfinite(ndwi_obs)] = np.nan
+
+    ndwi = completion(ndwi_obs)
+    return fourier_mean(ndwi).astype(np.float32)
+
+
+def rescale_uint8(im: NDArray[np.float32]) -> NDArray[np.uint8]:
+    """Per-band min-max normalization to [0, 255] uint8."""
+    if im.ndim == 2:
+        im = im[:, :, None]
+    out = np.empty_like(im, dtype=np.uint8)
+    for i in range(im.shape[2]):
+        band = im[:, :, i]
+        finite = np.isfinite(band)
+        vmin, vmax = np.nanmin(band), np.nanmax(band)
+        if not np.any(finite) or vmax <= vmin:
+            out[:, :, i] = 0
+            continue
+        scaled = np.clip((band - vmin) / (vmax - vmin), 0, 1)
+        scaled[~finite] = 0.0
+        out[:, :, i] = (scaled * 255).astype(np.uint8)
+    return out
+
+
+def save_geotiff(ds: xr.Dataset, uint8_image: NDArray[np.uint8], path: str) -> None:
+    """Save a 3-band uint8 image as a georeferenced GeoTIFF using CRS/transform from ds."""
+    import rioxarray  # noqa: F401
+
+    if uint8_image.ndim == 2:
+        uint8_image = np.repeat(uint8_image[:, :, None], 3, axis=2)
+    elif uint8_image.shape[2] == 1:
+        uint8_image = np.repeat(uint8_image, 3, axis=2)
+    elif uint8_image.shape[2] == 2:
+        uint8_image = np.concatenate([uint8_image, uint8_image[:, :, :1]], axis=2)
+    elif uint8_image.shape[2] > 3:
+        uint8_image = uint8_image[:, :, :3]
+
+    da = xr.DataArray(
+        uint8_image,
+        coords={"y": ds.y.values, "x": ds.x.values, "band": np.arange(1, 4)},
+        dims=("y", "x", "band"),
+    )
+    da.rio.set_spatial_dims(x_dim="x", y_dim="y", inplace=True)
+    if ds.rio.crs is not None:
+        da.rio.write_crs(ds.rio.crs, inplace=True)
+    try:
+        da.rio.write_transform(ds.rio.transform(), inplace=True)
+    except Exception:
+        pass
+    da.transpose("band", "y", "x").rio.to_raster(path)
+
+
+def presegment(troi: Troi, ds_sentinel2=None) -> str:
+    """
+    Create a 3-band uint8 NDWI Fourier GeoTIFF from Sentinel-2 data.
+    Returns the path to the saved preseg GeoTIFF.
+    """
+    preseg_path = Paths(troi).preseg
+    if check_if_valid_preseg_exists(preseg_path):
+        return preseg_path
+
+    if ds_sentinel2 is None:
+        from pysentinel2.cube import Cube
+        ds = Cube(config=troi.config).get_ds_troi(troi, clean=True)
+    else:
+        ds = ds_sentinel2
+    features = compute_ndwi_fourier(ds)
+    uint8_image = rescale_uint8(features)
+
+    makedirs(dirname(preseg_path), exist_ok=True)
+    save_geotiff(ds, uint8_image, preseg_path)
+    # Touch ``<tif>._SUCCESS`` *after* the TIFF write completes; its presence
+    # is what the next call uses as the cache-validity check.
+    with open(f'{preseg_path}._SUCCESS', 'w') as f:
+        f.write(datetime.utcnow().isoformat() + 'Z')
+    print(f"Saved preseg to {preseg_path}")
+    return preseg_path

PaddockTS/PaddockSegmentation/check_if_valid_paddocks_exists.py

@@ -0,0 +1,60 @@
+"""Cache-validity check for the SAM-segmented paddocks GeoPackage.
+
+A valid cache requires both the ``.gpkg`` file and a ``.gpkg._SUCCESS``
+sentinel sitting beside it (written by :func:`get_paddocks` only after
+the GeoPackage write completes). The marker is a *sibling file* rather
+than an inside-the-store file because GeoPackages are single-file
+SQLite databases, not directories — so the zarr-style "marker inside
+the dir" convention doesn't apply.
+
+"""
+
+from os.path import exists
+
+
+def check_if_valid_paddocks_exists(paddocks_path: str) -> bool:
+    """Return True iff a successfully-written paddocks GeoPackage is on disk."""
+    return exists(paddocks_path) and exists(f'{paddocks_path}._SUCCESS')
+
+
+def test_no_file_no_marker():
+    import tempfile
+    with tempfile.TemporaryDirectory() as tmpdir:
+        return check_if_valid_paddocks_exists(f'{tmpdir}/missing.gpkg') is False
+
+
+def test_file_without_marker():
+    import tempfile
+    with tempfile.TemporaryDirectory() as tmpdir:
+        gpkg = f'{tmpdir}/p.gpkg'
+        open(gpkg, 'w').close()
+        return check_if_valid_paddocks_exists(gpkg) is False
+
+
+def test_file_with_marker():
+    import tempfile
+    with tempfile.TemporaryDirectory() as tmpdir:
+        gpkg = f'{tmpdir}/p.gpkg'
+        open(gpkg, 'w').close()
+        open(f'{gpkg}._SUCCESS', 'w').close()
+        return check_if_valid_paddocks_exists(gpkg) is True
+
+
+def test_marker_without_file():
+    import tempfile
+    with tempfile.TemporaryDirectory() as tmpdir:
+        open(f'{tmpdir}/p.gpkg._SUCCESS', 'w').close()
+        return check_if_valid_paddocks_exists(f'{tmpdir}/p.gpkg') is False
+
+
+def test():
+    return all([
+        test_no_file_no_marker(),
+        test_file_without_marker(),
+        test_file_with_marker(),
+        test_marker_without_file(),
+    ])
+
+
+if __name__ == '__main__':
+    print(test())

PaddockTS/PaddockSegmentation/check_if_valid_preseg_exists.py

@@ -0,0 +1,51 @@
+"""Cache-validity check for the preseg GeoTIFF.
+
+A valid cache requires both the ``.tif`` file and a ``.tif._SUCCESS``
+sentinel sitting beside it (written by :func:`presegment` only after the
+TIFF write completes). The marker is a *sibling file* because GeoTIFFs
+are single files, not directory stores.
+
+Mirrors :func:`PaddockTS.PaddockSegmentation.check_if_valid_paddocks_exists`.
+"""
+
+from os.path import exists
+
+
+def check_if_valid_preseg_exists(preseg_path: str) -> bool:
+    """Return True iff a successfully-written preseg TIFF is on disk."""
+    return exists(preseg_path) and exists(f'{preseg_path}._SUCCESS')
+
+
+def test_no_file_no_marker():
+    import tempfile
+    with tempfile.TemporaryDirectory() as tmpdir:
+        return check_if_valid_preseg_exists(f'{tmpdir}/missing.tif') is False
+
+
+def test_file_without_marker():
+    import tempfile
+    with tempfile.TemporaryDirectory() as tmpdir:
+        tif = f'{tmpdir}/p.tif'
+        open(tif, 'w').close()
+        return check_if_valid_preseg_exists(tif) is False
+
+
+def test_file_with_marker():
+    import tempfile
+    with tempfile.TemporaryDirectory() as tmpdir:
+        tif = f'{tmpdir}/p.tif'
+        open(tif, 'w').close()
+        open(f'{tif}._SUCCESS', 'w').close()
+        return check_if_valid_preseg_exists(tif) is True
+
+
+def test():
+    return all([
+        test_no_file_no_marker(),
+        test_file_without_marker(),
+        test_file_with_marker(),
+    ])
+
+
+if __name__ == '__main__':
+    print(test())