satcube 0.1.12__py3-none-any.whl → 0.1.14__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of satcube might be problematic. Click here for more details.
- satcube/__init__.py +3 -3
- satcube/align.py +149 -0
- satcube/cloud_detection.py +176 -104
- satcube/download.py +52 -44
- satcube/utils.py +61 -2
- {satcube-0.1.12.dist-info → satcube-0.1.14.dist-info}/METADATA +5 -10
- satcube-0.1.14.dist-info/RECORD +14 -0
- satcube-0.1.12.dist-info/RECORD +0 -13
- {satcube-0.1.12.dist-info → satcube-0.1.14.dist-info}/LICENSE +0 -0
- {satcube-0.1.12.dist-info → satcube-0.1.14.dist-info}/WHEEL +0 -0
satcube/__init__.py
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
from satcube.cloud_detection import cloud_masking
|
|
2
|
-
from satcube.download import
|
|
2
|
+
from satcube.download import download
|
|
3
|
+
from satcube.align import align
|
|
3
4
|
|
|
4
5
|
|
|
5
|
-
|
|
6
|
-
__all__ = ["cloud_masking", "download_data"]
|
|
6
|
+
__all__ = ["cloud_masking", "download", "align"]
|
|
7
7
|
|
|
8
8
|
import importlib.metadata
|
|
9
9
|
__version__ = importlib.metadata.version("satcube")
|
satcube/align.py
ADDED
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import pathlib
|
|
4
|
+
from typing import List, Tuple
|
|
5
|
+
import pickle
|
|
6
|
+
|
|
7
|
+
import numpy as np
|
|
8
|
+
import rasterio
|
|
9
|
+
import xarray as xr
|
|
10
|
+
from affine import Affine
|
|
11
|
+
|
|
12
|
+
import satalign as sat
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def align(
|
|
16
|
+
input_dir: str | pathlib.Path,
|
|
17
|
+
output_dir: str | pathlib.Path,
|
|
18
|
+
*,
|
|
19
|
+
channel: str = "mean",
|
|
20
|
+
crop_center: int = 128,
|
|
21
|
+
num_threads: int = 2,
|
|
22
|
+
save_tiffs: bool = True,
|
|
23
|
+
) -> Tuple[xr.DataArray, List[np.ndarray]]:
|
|
24
|
+
"""Align all masked Sentinel‑2 tiles found in *input_dir*.
|
|
25
|
+
|
|
26
|
+
Parameters
|
|
27
|
+
----------
|
|
28
|
+
input_dir
|
|
29
|
+
Directory containing masked Sentinel‑2 *TIFF* tiles produced by the
|
|
30
|
+
previous preprocessing stage.
|
|
31
|
+
output_dir
|
|
32
|
+
Directory where the alignment artefacts (``datacube.pickle``) and, if
|
|
33
|
+
requested, one aligned *GeoTIFF* per date will be written.
|
|
34
|
+
channel
|
|
35
|
+
Datacube band used by the *PCC* model for correlation. ``"mean"`` is
|
|
36
|
+
recommended because it carries fewer noise artefacts.
|
|
37
|
+
crop_center
|
|
38
|
+
Half‑size (in pixels) of the square window extracted around the scene
|
|
39
|
+
centre that is fed to the correlation engine.
|
|
40
|
+
num_threads
|
|
41
|
+
Number of CPU threads for the multi‑core phase‑correlation run.
|
|
42
|
+
save_tiffs
|
|
43
|
+
If *True* (default) the aligned datacube is exported to tiled *COGs* via
|
|
44
|
+
:func:`save_aligned_cube_to_tiffs`.
|
|
45
|
+
pickle_datacube
|
|
46
|
+
If *True* (default) the raw (unaligned) datacube is pickled to
|
|
47
|
+
``datacube.pickle`` inside *output_dir* for reproducibility/debugging.
|
|
48
|
+
|
|
49
|
+
Returns
|
|
50
|
+
-------
|
|
51
|
+
aligned_cube, warp_matrices
|
|
52
|
+
*aligned_cube* is the spatially aligned datacube as an
|
|
53
|
+
:class:`xarray.DataArray`; *warp_matrices* is the list of 3 × 3 affine
|
|
54
|
+
homography matrices (one per time step) returned by the
|
|
55
|
+
:pyclass:`~satalign.PCC` engine.
|
|
56
|
+
"""
|
|
57
|
+
input_path = pathlib.Path(input_dir).expanduser().resolve()
|
|
58
|
+
output_path = pathlib.Path(output_dir).expanduser().resolve()
|
|
59
|
+
output_path.mkdir(parents=True, exist_ok=True)
|
|
60
|
+
|
|
61
|
+
# ── 1. Build datacube ────────────────────────────────────────────
|
|
62
|
+
da = sat.utils.create_array(input_path)
|
|
63
|
+
|
|
64
|
+
# ── 2. Select reference slice (highest cloud‑score CDF) ───────────
|
|
65
|
+
da_sorted = da.sortby("cs_cdf", ascending=False)
|
|
66
|
+
ref_slice = da_sorted.isel(time=0)
|
|
67
|
+
reference = ref_slice.where((ref_slice != 0) & (ref_slice != 65535))
|
|
68
|
+
|
|
69
|
+
# ── 3. Instantiate and run PCC model ─────────────────────────────
|
|
70
|
+
pcc_model = sat.PCC(
|
|
71
|
+
datacube=da,
|
|
72
|
+
reference=reference,
|
|
73
|
+
channel=channel,
|
|
74
|
+
crop_center=crop_center,
|
|
75
|
+
num_threads=num_threads,
|
|
76
|
+
)
|
|
77
|
+
aligned_cube, warp_matrices = pcc_model.run_multicore()
|
|
78
|
+
|
|
79
|
+
# ── 4. Optionally export as Cloud‑Optimised GeoTIFFs ────────────
|
|
80
|
+
if save_tiffs:
|
|
81
|
+
save_aligned_cube_to_tiffs(aligned_cube, output_path)
|
|
82
|
+
|
|
83
|
+
return aligned_cube, warp_matrices
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def save_aligned_cube_to_tiffs(
|
|
87
|
+
aligned_cube: xr.DataArray,
|
|
88
|
+
out_dir: str | pathlib.Path,
|
|
89
|
+
*,
|
|
90
|
+
block_size: int = 128,
|
|
91
|
+
) -> None:
|
|
92
|
+
"""Write each time slice of *aligned_cube* to an individual tiled COG.
|
|
93
|
+
|
|
94
|
+
The filenames follow the pattern ``YYYY‑MM‑DD.tif``.
|
|
95
|
+
|
|
96
|
+
Parameters
|
|
97
|
+
----------
|
|
98
|
+
aligned_cube
|
|
99
|
+
Datacube returned by :func:`align_datacube`.
|
|
100
|
+
out_dir
|
|
101
|
+
Target directory; it will be created if it does not exist.
|
|
102
|
+
block_size
|
|
103
|
+
Internal tile size (*rasterio* ``blockxsize`` and ``blockysize``).
|
|
104
|
+
"""
|
|
105
|
+
out_dir_path = pathlib.Path(out_dir).expanduser().resolve()
|
|
106
|
+
out_dir_path.mkdir(parents=True, exist_ok=True)
|
|
107
|
+
|
|
108
|
+
# ── 1. Build affine transform from x/y coordinate vectors ────────
|
|
109
|
+
x_vals = aligned_cube.x.values
|
|
110
|
+
y_vals = aligned_cube.y.values
|
|
111
|
+
x_res = float(x_vals[1] - x_vals[0]) # positive (east)
|
|
112
|
+
y_res = float(y_vals[1] - y_vals[0]) # negative (north‑up)
|
|
113
|
+
transform = (
|
|
114
|
+
Affine.translation(x_vals[0] - x_res / 2.0, y_vals[0] - y_res / 2.0)
|
|
115
|
+
* Affine.scale(x_res, y_res)
|
|
116
|
+
)
|
|
117
|
+
|
|
118
|
+
# ── 2. Retrieve CRS from datacube attributes ────────────────────
|
|
119
|
+
attrs = aligned_cube.attrs
|
|
120
|
+
crs: str | None = attrs.get("crs_wkt")
|
|
121
|
+
if crs is None and "crs_epsg" in attrs:
|
|
122
|
+
crs = f"EPSG:{int(attrs['crs_epsg'])}"
|
|
123
|
+
|
|
124
|
+
# ── 3. Loop over acquisition dates and write GeoTIFFs ────────────
|
|
125
|
+
for t in aligned_cube.time.values:
|
|
126
|
+
date_str = str(t)[:10] # YYYY‑MM‑DD
|
|
127
|
+
da_t = aligned_cube.sel(time=t)
|
|
128
|
+
|
|
129
|
+
# Ensure (band, y, x) memory layout for rasterio
|
|
130
|
+
data = da_t.transpose("band", "y", "x").values
|
|
131
|
+
|
|
132
|
+
profile = {
|
|
133
|
+
"driver": da_t.attrs.get("driver", "GTiff"),
|
|
134
|
+
"height": da_t.sizes["y"],
|
|
135
|
+
"width": da_t.sizes["x"],
|
|
136
|
+
"count": da_t.sizes["band"],
|
|
137
|
+
"dtype": str(da_t.dtype),
|
|
138
|
+
"transform": transform,
|
|
139
|
+
"crs": crs,
|
|
140
|
+
"nodata": int(getattr(da_t, "nodata", 0)),
|
|
141
|
+
"tiled": True,
|
|
142
|
+
"blockxsize": block_size,
|
|
143
|
+
"blockysize": block_size,
|
|
144
|
+
"interleave": "band",
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
outfile = out_dir_path / f"{date_str}.tif"
|
|
148
|
+
with rasterio.open(outfile, "w", **profile) as dst:
|
|
149
|
+
dst.write(data)
|
satcube/cloud_detection.py
CHANGED
|
@@ -12,28 +12,158 @@ Example
|
|
|
12
12
|
|
|
13
13
|
from __future__ import annotations
|
|
14
14
|
|
|
15
|
-
import
|
|
16
|
-
from pathlib import Path
|
|
17
|
-
from typing import List
|
|
15
|
+
import pathlib
|
|
18
16
|
|
|
19
17
|
import mlstac
|
|
20
18
|
import numpy as np
|
|
21
19
|
import rasterio as rio
|
|
20
|
+
from rasterio.windows import Window
|
|
22
21
|
import torch
|
|
22
|
+
import pandas as pd
|
|
23
|
+
from tqdm import tqdm
|
|
24
|
+
import rasterio as rio
|
|
25
|
+
from rasterio.merge import merge
|
|
26
|
+
|
|
27
|
+
from satcube.utils import define_iteration, DeviceManager
|
|
28
|
+
import warnings
|
|
29
|
+
warnings.filterwarnings(
|
|
30
|
+
"ignore",
|
|
31
|
+
message="The secret HF_TOKEN does not exist in your Colab secrets.",
|
|
32
|
+
category=UserWarning,
|
|
33
|
+
module=r"huggingface_hub\.utils\._.*",
|
|
34
|
+
)
|
|
35
|
+
|
|
23
36
|
|
|
24
|
-
from satcube.utils import DeviceManager, _reset_gpu
|
|
25
37
|
|
|
26
38
|
|
|
27
|
-
def
|
|
28
|
-
|
|
29
|
-
|
|
39
|
+
def infer_cloudmask(
|
|
40
|
+
input_path: str | pathlib.Path,
|
|
41
|
+
output_path: str | pathlib.Path,
|
|
42
|
+
cloud_model: torch.nn.Module,
|
|
30
43
|
*,
|
|
31
|
-
|
|
32
|
-
|
|
44
|
+
chunk_size: int = 512,
|
|
45
|
+
overlap: int = 32,
|
|
46
|
+
device: str = "cpu",
|
|
47
|
+
save_mask: bool = False,
|
|
48
|
+
prefix: str = ""
|
|
49
|
+
) -> pathlib.Path:
|
|
50
|
+
"""
|
|
51
|
+
Predict 'image_path' in overlapping patches of 'chunk_size' x 'chunk_size',
|
|
52
|
+
but only write the valid (inner) region to avoid seam artifacts.
|
|
53
|
+
|
|
54
|
+
This uses partial overlap logic:
|
|
55
|
+
- For interior tiles, skip overlap//2 on each side.
|
|
56
|
+
- For boundary tiles, we skip only the interior side to avoid losing data at the edges.
|
|
57
|
+
|
|
58
|
+
Parameters
|
|
59
|
+
----------
|
|
60
|
+
image_path : Path to input image.
|
|
61
|
+
output_path : Path to output single-band mask.
|
|
62
|
+
cloud_model : PyTorch model (already loaded with weights).
|
|
63
|
+
chunk_size : Size of each tile to read from the source image (default 512).
|
|
64
|
+
overlap : Overlap in pixels between adjacent tiles (default 32).
|
|
65
|
+
device : "cpu" or "cuda:0".
|
|
66
|
+
|
|
67
|
+
Returns
|
|
68
|
+
-------
|
|
69
|
+
pathlib.Path : The path to the created output image.
|
|
70
|
+
"""
|
|
71
|
+
# image_path = "/home/contreras/Documents/GitHub/satcube2/raw/2018-07-15_6q49m.tif"
|
|
72
|
+
# output_path = "/home/contreras/Documents/GitHub/satcube2/masked/2018-07-15_6q49m.tif"
|
|
73
|
+
|
|
74
|
+
input_path = pathlib.Path(input_path)
|
|
75
|
+
output_path = pathlib.Path(output_path)
|
|
76
|
+
|
|
77
|
+
# 1) Validate metadata
|
|
78
|
+
with rio.open(input_path) as src:
|
|
79
|
+
meta = src.profile
|
|
80
|
+
if not meta.get("tiled", False):
|
|
81
|
+
raise ValueError("The input image is not marked as tiled in its metadata.")
|
|
82
|
+
# Ensure the internal blocksize matches chunk_size
|
|
83
|
+
if chunk_size % meta["blockxsize"] != 0 and meta["blockxsize"] <= chunk_size:
|
|
84
|
+
raise ValueError(f"Image blocks must be {chunk_size}x{chunk_size}, "
|
|
85
|
+
f"got {meta['blockxsize']}x{meta['blockysize']}")
|
|
86
|
+
height, width = meta["height"], meta["width"]
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
# 2) Crear un buffer en RAM (float32)
|
|
90
|
+
full_mask = np.zeros((height, width), dtype=np.float32)
|
|
91
|
+
|
|
92
|
+
# 3) Iterar por ventanas
|
|
93
|
+
|
|
94
|
+
coords = define_iteration((height, width), chunk_size, overlap)
|
|
95
|
+
|
|
96
|
+
with rio.open(input_path) as src:
|
|
97
|
+
|
|
98
|
+
for (row_off, col_off) in tqdm(
|
|
99
|
+
coords,
|
|
100
|
+
desc=f"{prefix} Inference on {input_path.name}",
|
|
101
|
+
position=1,
|
|
102
|
+
leave=False):
|
|
103
|
+
|
|
104
|
+
window = Window(col_off, row_off, chunk_size, chunk_size)
|
|
105
|
+
patch = src.read(window=window) / 1e4
|
|
106
|
+
patch_tensor = torch.from_numpy(patch).float().unsqueeze(0).to(device)
|
|
107
|
+
result = cloud_model(patch_tensor).cpu().numpy().astype(np.uint8)
|
|
108
|
+
|
|
109
|
+
if col_off == 0:
|
|
110
|
+
offset_x = 0
|
|
111
|
+
else:
|
|
112
|
+
offset_x = col_off + overlap // 2
|
|
113
|
+
if row_off == 0:
|
|
114
|
+
offset_y = 0
|
|
115
|
+
else:
|
|
116
|
+
offset_y = row_off + overlap // 2
|
|
117
|
+
if (offset_x + chunk_size) == width:
|
|
118
|
+
length_x = chunk_size
|
|
119
|
+
sub_x_start = 0
|
|
120
|
+
else:
|
|
121
|
+
length_x = chunk_size - (overlap // 2)
|
|
122
|
+
sub_x_start = overlap // 2 if col_off != 0 else 0
|
|
123
|
+
|
|
124
|
+
if (offset_y + chunk_size) == height:
|
|
125
|
+
length_y = chunk_size
|
|
126
|
+
sub_y_start = 0
|
|
127
|
+
else:
|
|
128
|
+
length_y = chunk_size - (overlap // 2)
|
|
129
|
+
sub_y_start = overlap // 2 if row_off != 0 else 0
|
|
130
|
+
|
|
131
|
+
full_mask[
|
|
132
|
+
offset_y : offset_y + length_y,
|
|
133
|
+
offset_x : offset_x + length_x
|
|
134
|
+
] = result[
|
|
135
|
+
sub_y_start : sub_y_start + length_y,
|
|
136
|
+
sub_x_start : sub_x_start + length_x
|
|
137
|
+
]
|
|
138
|
+
|
|
139
|
+
if save_mask:
|
|
140
|
+
out_meta = meta.copy()
|
|
141
|
+
out_meta.update(count=1, dtype="uint8", nodata=255)
|
|
142
|
+
output_mask = output_path.parent / (output_path.stem + "_mask.tif")
|
|
143
|
+
with rio.open(output_mask, "w", **out_meta) as dst:
|
|
144
|
+
dst.write(full_mask, 1)
|
|
145
|
+
|
|
146
|
+
with rio.open(input_path) as src_img:
|
|
147
|
+
data = src_img.read()
|
|
148
|
+
img_prof = src_img.profile.copy()
|
|
149
|
+
|
|
150
|
+
masked = data.copy()
|
|
151
|
+
masked[:, full_mask != 0] = 65535
|
|
152
|
+
img_prof.update(dtype="uint16", nodata=65535)
|
|
153
|
+
|
|
154
|
+
with rio.open(output_path, "w", **img_prof) as dst:
|
|
155
|
+
dst.write(masked)
|
|
156
|
+
|
|
157
|
+
return output_path
|
|
158
|
+
|
|
159
|
+
def cloud_masking(
|
|
160
|
+
input: str | pathlib.Path = "raw",
|
|
161
|
+
output: str | pathlib.Path = "masked",
|
|
162
|
+
model_path: str | pathlib.Path = "SEN2CloudEnsemble",
|
|
33
163
|
save_mask: bool = False,
|
|
34
164
|
device: str = "cpu",
|
|
35
|
-
|
|
36
|
-
) ->
|
|
165
|
+
cache: bool = True
|
|
166
|
+
) -> list[pathlib.Path]:
|
|
37
167
|
"""Write cloud-masked Sentinel-2 images.
|
|
38
168
|
|
|
39
169
|
Parameters
|
|
@@ -41,7 +171,8 @@ def cloud_masking(
|
|
|
41
171
|
input
|
|
42
172
|
Path to a single ``.tif`` file **or** a directory containing them.
|
|
43
173
|
output
|
|
44
|
-
Destination directory (created
|
|
174
|
+
Destination directory (created i
|
|
175
|
+
f missing).
|
|
45
176
|
tile, pad
|
|
46
177
|
Tile size and padding (pixels) when tiling is required.
|
|
47
178
|
save_mask
|
|
@@ -56,14 +187,12 @@ def cloud_masking(
|
|
|
56
187
|
list[pathlib.Path]
|
|
57
188
|
Paths to the generated masked images.
|
|
58
189
|
"""
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
src = Path(input).expanduser().resolve()
|
|
62
|
-
dst_dir = Path(output).expanduser().resolve()
|
|
190
|
+
src = pathlib.Path(input).expanduser().resolve()
|
|
191
|
+
dst_dir = pathlib.Path(output).expanduser().resolve()
|
|
63
192
|
dst_dir.mkdir(parents=True, exist_ok=True)
|
|
64
193
|
|
|
65
194
|
# Collect files to process -------------------------------------------------
|
|
66
|
-
tif_paths
|
|
195
|
+
tif_paths = []
|
|
67
196
|
if src.is_dir():
|
|
68
197
|
tif_paths = [p for p in src.rglob("*.tif")]
|
|
69
198
|
elif src.is_file() and src.suffix.lower() == ".tif":
|
|
@@ -76,95 +205,38 @@ def cloud_masking(
|
|
|
76
205
|
print(f"[cloud_masking] No .tif files found in {src}")
|
|
77
206
|
return []
|
|
78
207
|
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
if not dir.exists():
|
|
82
|
-
|
|
208
|
+
if not pathlib.Path(model_path).exists():
|
|
83
209
|
mlstac.download(
|
|
84
210
|
file = "https://huggingface.co/tacofoundation/CloudSEN12-models/resolve/main/SEN2CloudEnsemble/mlm.json",
|
|
85
|
-
output_dir =
|
|
211
|
+
output_dir = model_path
|
|
86
212
|
)
|
|
87
213
|
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
# ----------------------- inference -----------------------------------
|
|
116
|
-
if not do_tiling: # full frame
|
|
117
|
-
with rio.open(tif_path) as src_img, torch.inference_mode():
|
|
118
|
-
img = src_img.read().astype(np.float32) / 1e4
|
|
119
|
-
h32, w32 = (h + 31) // 32 * 32, (w + 31) // 32 * 32
|
|
120
|
-
pad_b, pad_r = h32 - h, w32 - w
|
|
121
|
-
tensor = torch.from_numpy(img).unsqueeze(0)
|
|
122
|
-
if pad_b or pad_r:
|
|
123
|
-
tensor = torch.nn.functional.pad(tensor, (0, pad_r, 0, pad_b))
|
|
124
|
-
mask = dm.model(tensor.to(dm.device)).squeeze(0)
|
|
125
|
-
full_mask[:] = mask[..., :h, :w].cpu().numpy().astype(np.uint8)
|
|
126
|
-
else: # tiled
|
|
127
|
-
with rio.open(tif_path) as src_img, torch.inference_mode():
|
|
128
|
-
for y0 in range(0, h, tile):
|
|
129
|
-
for x0 in range(0, w, tile):
|
|
130
|
-
y0r, x0r = max(0, y0 - pad), max(0, x0 - pad)
|
|
131
|
-
y1r, x1r = min(h, y0 + tile + pad), min(w, x0 + tile + pad)
|
|
132
|
-
win = rio.windows.Window(x0r, y0r, x1r - x0r, y1r - y0r)
|
|
133
|
-
|
|
134
|
-
patch = src_img.read(window=win).astype(np.float32) / 1e4
|
|
135
|
-
tensor = torch.from_numpy(patch).unsqueeze(0).to(dm.device)
|
|
136
|
-
mask = dm.model(tensor).squeeze(0).cpu().numpy().astype(np.uint8)
|
|
137
|
-
|
|
138
|
-
y_in0 = pad if y0r else 0
|
|
139
|
-
x_in0 = pad if x0r else 0
|
|
140
|
-
y_in1 = mask.shape[0] - (pad if y1r < h else 0)
|
|
141
|
-
x_in1 = mask.shape[1] - (pad if x1r < w else 0)
|
|
142
|
-
core = mask[y_in0:y_in1, x_in0:x_in1]
|
|
143
|
-
full_mask[y0 : y0 + core.shape[0], x0 : x0 + core.shape[1]] = core
|
|
144
|
-
|
|
145
|
-
# ----------------------- output --------------------------------------
|
|
146
|
-
if save_mask:
|
|
147
|
-
with rio.open(mask_path, "w", **mask_prof) as dst:
|
|
148
|
-
dst.write(full_mask, 1)
|
|
149
|
-
|
|
150
|
-
with rio.open(tif_path) as src_img:
|
|
151
|
-
data = src_img.read()
|
|
152
|
-
img_prof = src_img.profile.copy()
|
|
153
|
-
|
|
154
|
-
masked = data.copy()
|
|
155
|
-
masked[:, full_mask != 0] = 65535
|
|
156
|
-
img_prof.update(dtype="uint16", nodata=65535)
|
|
157
|
-
|
|
158
|
-
with rio.open(masked_path, "w", **img_prof) as dst:
|
|
159
|
-
dst.write(masked)
|
|
160
|
-
|
|
161
|
-
masked_paths.append(masked_path)
|
|
162
|
-
dt = time.perf_counter() - t0
|
|
163
|
-
print(f"[{idx}/{len(tif_paths)}] {rel} → done in {dt:.1f}s")
|
|
164
|
-
|
|
165
|
-
if dm.device == "cuda":
|
|
166
|
-
_reset_gpu()
|
|
214
|
+
model = mlstac.load(model_path)
|
|
215
|
+
cloud_model = DeviceManager(model, init_device=device).model
|
|
216
|
+
cloud_model.eval()
|
|
217
|
+
|
|
218
|
+
outs = [
|
|
219
|
+
infer_cloudmask(
|
|
220
|
+
input_path=p,
|
|
221
|
+
output_path=dst_dir / p.name,
|
|
222
|
+
cloud_model=cloud_model,
|
|
223
|
+
device=device,
|
|
224
|
+
save_mask=save_mask,
|
|
225
|
+
prefix=f"[{i+1}/{len(tif_paths)}] "
|
|
226
|
+
)
|
|
227
|
+
for i, p in enumerate(tif_paths)
|
|
228
|
+
]
|
|
229
|
+
|
|
230
|
+
df_out = (
|
|
231
|
+
pd.Series(outs, name="path")
|
|
232
|
+
.to_frame()
|
|
233
|
+
.assign(path_str=lambda df: df["path"].astype(str))
|
|
234
|
+
.assign(
|
|
235
|
+
date = lambda df: df["path_str"].str.extract(r"(\d{4}-\d{2}-\d{2})", expand=False),
|
|
236
|
+
score = lambda df: df["path_str"].str.extract(r"_(\d+)\.tif$", expand=False).astype(int) / 100
|
|
237
|
+
)
|
|
238
|
+
.drop(columns="path_str") # ya no la necesitamos
|
|
239
|
+
.sort_values("score", ascending=False, ignore_index=True)
|
|
240
|
+
)
|
|
167
241
|
|
|
168
|
-
|
|
169
|
-
print(f"Processed {len(masked_paths)} image(s) in {total_time:.1f}s.")
|
|
170
|
-
return masked_paths
|
|
242
|
+
return df_out
|
satcube/download.py
CHANGED
|
@@ -1,57 +1,65 @@
|
|
|
1
|
-
import
|
|
2
|
-
import
|
|
3
|
-
import cubexpress
|
|
1
|
+
import sys, time, threading, itertools
|
|
2
|
+
import cubexpress as ce
|
|
4
3
|
import pandas as pd
|
|
5
4
|
|
|
6
|
-
|
|
7
|
-
def download_data(
|
|
8
|
-
*, # keyword-only
|
|
5
|
+
def download(
|
|
9
6
|
lon: float,
|
|
10
7
|
lat: float,
|
|
11
|
-
|
|
12
|
-
edge_size: int = 2_048,
|
|
8
|
+
edge_size: int,
|
|
13
9
|
start: str,
|
|
14
10
|
end: str,
|
|
15
|
-
|
|
16
|
-
|
|
11
|
+
*,
|
|
12
|
+
max_cscore: float = 1,
|
|
13
|
+
min_cscore: float = 0,
|
|
14
|
+
outfolder: str = "raw",
|
|
17
15
|
nworks: int = 4,
|
|
18
|
-
|
|
16
|
+
cache: bool = True,
|
|
17
|
+
show_spinner: bool = True,
|
|
18
|
+
verbose: bool = False
|
|
19
19
|
) -> pd.DataFrame:
|
|
20
20
|
"""
|
|
21
|
-
|
|
21
|
+
"""
|
|
22
|
+
stop_flag = {"v": False}
|
|
22
23
|
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
auto_init_gee Call ee.Initialize() if needed.
|
|
24
|
+
if show_spinner and not verbose:
|
|
25
|
+
def _spin():
|
|
26
|
+
for ch in itertools.cycle("|/-\\"):
|
|
27
|
+
if stop_flag["v"]:
|
|
28
|
+
break
|
|
29
|
+
sys.stdout.write(f"\rDownloading Sentinel-2 imagery & metadata… {ch}")
|
|
30
|
+
sys.stdout.flush()
|
|
31
|
+
time.sleep(0.1)
|
|
32
|
+
sys.stdout.write("\rDownloading Sentinel-2 imagery & metadata ✅\n")
|
|
33
|
+
sys.stdout.flush()
|
|
34
34
|
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
35
|
+
th = threading.Thread(target=_spin, daemon=True)
|
|
36
|
+
th.start()
|
|
37
|
+
else:
|
|
38
|
+
th = None
|
|
39
|
+
|
|
40
|
+
try:
|
|
41
|
+
table = ce.s2_cloud_table(
|
|
42
|
+
lon=lon,
|
|
43
|
+
lat=lat,
|
|
44
|
+
edge_size=edge_size,
|
|
45
|
+
start=start,
|
|
46
|
+
end=end,
|
|
47
|
+
max_cscore=max_cscore,
|
|
48
|
+
min_cscore=min_cscore,
|
|
49
|
+
cache=cache,
|
|
50
|
+
verbose=verbose
|
|
51
|
+
)
|
|
50
52
|
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
53
|
+
ce.get_cube(
|
|
54
|
+
table=table,
|
|
55
|
+
outfolder=outfolder,
|
|
56
|
+
nworks=nworks,
|
|
57
|
+
verbose=verbose,
|
|
58
|
+
cache=cache
|
|
59
|
+
)
|
|
60
|
+
finally:
|
|
61
|
+
stop_flag["v"] = True
|
|
62
|
+
if th is not None:
|
|
63
|
+
th.join()
|
|
54
64
|
|
|
55
|
-
|
|
56
|
-
cubexpress.get_cube(requests, output, nworks)
|
|
57
|
-
return df
|
|
65
|
+
return table
|
satcube/utils.py
CHANGED
|
@@ -1,11 +1,10 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
3
|
import gc
|
|
4
|
+
import itertools
|
|
4
5
|
from typing import Any, Optional
|
|
5
|
-
|
|
6
6
|
import torch
|
|
7
7
|
|
|
8
|
-
|
|
9
8
|
def _reset_gpu() -> None:
|
|
10
9
|
"""Release CUDA memory and reset allocation statistics.
|
|
11
10
|
|
|
@@ -15,6 +14,65 @@ def _reset_gpu() -> None:
|
|
|
15
14
|
torch.cuda.reset_peak_memory_stats()
|
|
16
15
|
|
|
17
16
|
|
|
17
|
+
def define_iteration(dimension: tuple, chunk_size: int, overlap: int = 0):
|
|
18
|
+
"""
|
|
19
|
+
Define the iteration strategy to walk through the image with an overlap.
|
|
20
|
+
|
|
21
|
+
Args:
|
|
22
|
+
dimension (tuple): Dimension of the S2 image.
|
|
23
|
+
chunk_size (int): Size of the chunks.
|
|
24
|
+
overlap (int): Size of the overlap between chunks.
|
|
25
|
+
|
|
26
|
+
Returns:
|
|
27
|
+
list: List of chunk coordinates.
|
|
28
|
+
"""
|
|
29
|
+
dimy, dimx = dimension
|
|
30
|
+
|
|
31
|
+
if chunk_size > max(dimx, dimy):
|
|
32
|
+
return [(0, 0)]
|
|
33
|
+
|
|
34
|
+
# Adjust step to create overlap
|
|
35
|
+
y_step = chunk_size - overlap
|
|
36
|
+
x_step = chunk_size - overlap
|
|
37
|
+
|
|
38
|
+
# Generate initial chunk positions
|
|
39
|
+
iterchunks = list(itertools.product(range(0, dimy, y_step), range(0, dimx, x_step)))
|
|
40
|
+
|
|
41
|
+
# Fix chunks at the edges to stay within bounds
|
|
42
|
+
iterchunks_fixed = fix_lastchunk(
|
|
43
|
+
iterchunks=iterchunks, s2dim=dimension, chunk_size=chunk_size
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
return iterchunks_fixed
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def fix_lastchunk(iterchunks, s2dim, chunk_size):
|
|
50
|
+
"""
|
|
51
|
+
Fix the last chunk of the overlay to ensure it aligns with image boundaries.
|
|
52
|
+
|
|
53
|
+
Args:
|
|
54
|
+
iterchunks (list): List of chunks created by itertools.product.
|
|
55
|
+
s2dim (tuple): Dimension of the S2 images.
|
|
56
|
+
chunk_size (int): Size of the chunks.
|
|
57
|
+
|
|
58
|
+
Returns:
|
|
59
|
+
list: List of adjusted chunk coordinates.
|
|
60
|
+
"""
|
|
61
|
+
itercontainer = []
|
|
62
|
+
|
|
63
|
+
for index_i, index_j in iterchunks:
|
|
64
|
+
# Adjust if the chunk extends beyond bounds
|
|
65
|
+
if index_i + chunk_size > s2dim[0]:
|
|
66
|
+
index_i = max(s2dim[0] - chunk_size, 0)
|
|
67
|
+
if index_j + chunk_size > s2dim[1]:
|
|
68
|
+
index_j = max(s2dim[1] - chunk_size, 0)
|
|
69
|
+
|
|
70
|
+
itercontainer.append((index_i, index_j))
|
|
71
|
+
|
|
72
|
+
return itercontainer
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
|
|
18
76
|
class DeviceManager:
|
|
19
77
|
"""Hold a compiled mlstac model and move it between devices on demand."""
|
|
20
78
|
|
|
@@ -68,3 +126,4 @@ class DeviceManager:
|
|
|
68
126
|
self.model = self._experiment.compiled_model(device=new_device, mode="max")
|
|
69
127
|
self.device = new_device
|
|
70
128
|
return self.model
|
|
129
|
+
|
|
@@ -1,29 +1,24 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: satcube
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.14
|
|
4
4
|
Summary: A Python package to create cloud-free monthly composites by fusing Landsat and Sentinel-2 data.
|
|
5
5
|
Home-page: https://github.com/IPL-UV/satcube
|
|
6
6
|
Author: Cesar Aybar
|
|
7
7
|
Author-email: fcesar.aybar@uv.es
|
|
8
|
-
Requires-Python: >=3.
|
|
8
|
+
Requires-Python: >=3.9
|
|
9
9
|
Classifier: Programming Language :: Python :: 3
|
|
10
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
10
11
|
Classifier: Programming Language :: Python :: 3.10
|
|
11
12
|
Classifier: Programming Language :: Python :: 3.11
|
|
12
13
|
Classifier: Programming Language :: Python :: 3.12
|
|
13
14
|
Provides-Extra: full
|
|
14
|
-
Requires-Dist: cubexpress (>=0.1.
|
|
15
|
-
Requires-Dist: earthengine-api (>=1.5.12)
|
|
15
|
+
Requires-Dist: cubexpress (>=0.1.10)
|
|
16
16
|
Requires-Dist: mlstac (>=0.4.0)
|
|
17
|
-
Requires-Dist: numpy (>=1.25.0)
|
|
18
|
-
Requires-Dist: pandas (>=2.0.0)
|
|
19
17
|
Requires-Dist: phicloudmask (>=0.0.2)
|
|
20
|
-
Requires-Dist: pydantic (>=2.8.0)
|
|
21
|
-
Requires-Dist: rasterio (>=1.3.9)
|
|
22
18
|
Requires-Dist: requests (>=2.26.0)
|
|
23
|
-
Requires-Dist: satalign (>=0.1.
|
|
19
|
+
Requires-Dist: satalign (>=0.1.12)
|
|
24
20
|
Requires-Dist: scikit-learn (>=1.2.0)
|
|
25
21
|
Requires-Dist: segmentation-models-pytorch (>=0.3.0)
|
|
26
|
-
Requires-Dist: utm (>=0.7.0)
|
|
27
22
|
Requires-Dist: xarray (>=2023.7.0)
|
|
28
23
|
Project-URL: Documentation, https://ipl-uv.github.io/satcube/
|
|
29
24
|
Project-URL: Repository, https://github.com/IPL-UV/satcube
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
satcube/__init__.py,sha256=93AVVWcj-sdRjht8uhPOeoCfLMOJRw3G8MsoU7aXCXQ,251
|
|
2
|
+
satcube/align.py,sha256=4Ny1qz1bNcGdTAbQAnFtXiE1yVFIGTSd_CVDl4Kybf8,5461
|
|
3
|
+
satcube/cloud_detection.py,sha256=qeofq5_R-zJYOoTe3ylQS8QIHtdidH43Ii-yFfBGU94,8104
|
|
4
|
+
satcube/cloud_detection_old.py,sha256=7MviF8QlT2tj6QC3seepr8SZU0nHOK9Pji8AU94Z2q0,751
|
|
5
|
+
satcube/dataclass.py,sha256=TAAKouyTts5eMtVXRmwWgJb5EaUXryEtkKMtt1O8TKM,934
|
|
6
|
+
satcube/download.py,sha256=y54zVYATaHyIi2k1Y9dS3iAezeuog4R4kONud_PhMMs,1556
|
|
7
|
+
satcube/download_old.py,sha256=Y7dUgq7Gl7jVKHZl5x9cpxALN9T6dCjtZwNjBiq1CAA,2647
|
|
8
|
+
satcube/main.py,sha256=BpQJbPXl6Ydj6X3pX2lFepH7w1-cfJ3LeTTsTmQih6s,14841
|
|
9
|
+
satcube/utils.py,sha256=QBdmSg6_4Vy-4mXH1Z3Z2AvbIKAXBYi5g3-IgWSE1MY,3660
|
|
10
|
+
satcube/utils_old.py,sha256=UBCI2oaL7E5MEjebobnyqGqgOtK6jU9O3t-c58JqZ0k,35057
|
|
11
|
+
satcube-0.1.14.dist-info/LICENSE,sha256=YdB4BQMkMzWuKvXRIpQR4g91IQ_pwA5PSH2lNM97zFI,1070
|
|
12
|
+
satcube-0.1.14.dist-info/METADATA,sha256=eJpLSXf0_NPVCYLlDnKAY93B0wBRxB1dBe7Y7LIOE90,6528
|
|
13
|
+
satcube-0.1.14.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
|
|
14
|
+
satcube-0.1.14.dist-info/RECORD,,
|
satcube-0.1.12.dist-info/RECORD
DELETED
|
@@ -1,13 +0,0 @@
|
|
|
1
|
-
satcube/__init__.py,sha256=WxfiBldO7YXWJFATRjbhWLOnD1mk0b8Pc6HGQYOJZJA,221
|
|
2
|
-
satcube/cloud_detection.py,sha256=qmC_1hQVxdRzA03r6cML9AuSLQw_P8ZNNFz4Uji1hro,6109
|
|
3
|
-
satcube/cloud_detection_old.py,sha256=7MviF8QlT2tj6QC3seepr8SZU0nHOK9Pji8AU94Z2q0,751
|
|
4
|
-
satcube/dataclass.py,sha256=TAAKouyTts5eMtVXRmwWgJb5EaUXryEtkKMtt1O8TKM,934
|
|
5
|
-
satcube/download.py,sha256=xwPIm6SWN_cev2wM0OzqN18ejPgnjuNVPxfeF7FhI9c,1397
|
|
6
|
-
satcube/download_old.py,sha256=Y7dUgq7Gl7jVKHZl5x9cpxALN9T6dCjtZwNjBiq1CAA,2647
|
|
7
|
-
satcube/main.py,sha256=BpQJbPXl6Ydj6X3pX2lFepH7w1-cfJ3LeTTsTmQih6s,14841
|
|
8
|
-
satcube/utils.py,sha256=wQl4ZSrocSZSSU6hjwCA2CYIv8p3deiX8PUKxJRr3yc,1952
|
|
9
|
-
satcube/utils_old.py,sha256=UBCI2oaL7E5MEjebobnyqGqgOtK6jU9O3t-c58JqZ0k,35057
|
|
10
|
-
satcube-0.1.12.dist-info/LICENSE,sha256=YdB4BQMkMzWuKvXRIpQR4g91IQ_pwA5PSH2lNM97zFI,1070
|
|
11
|
-
satcube-0.1.12.dist-info/METADATA,sha256=ta9-NbSV7U3z72K7fPKoSLtqP-TBWPDrCO0YUOXQ-zU,6686
|
|
12
|
-
satcube-0.1.12.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
|
|
13
|
-
satcube-0.1.12.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|