transformez 0.4.2__tar.gz → 0.5.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {transformez-0.4.2 → transformez-0.5.0}/CHANGELOG.md +15 -0
- {transformez-0.4.2 → transformez-0.5.0}/CITATION.cff +2 -2
- {transformez-0.4.2 → transformez-0.5.0}/PKG-INFO +8 -7
- {transformez-0.4.2 → transformez-0.5.0}/README.md +7 -6
- {transformez-0.4.2 → transformez-0.5.0}/src/transformez/_version.py +1 -1
- {transformez-0.4.2 → transformez-0.5.0}/src/transformez/api.py +84 -2
- {transformez-0.4.2 → transformez-0.5.0}/src/transformez/cli.py +22 -4
- {transformez-0.4.2 → transformez-0.5.0}/src/transformez/definitions.py +3 -3
- {transformez-0.4.2 → transformez-0.5.0}/src/transformez/grid_engine.py +59 -1
- {transformez-0.4.2 → transformez-0.5.0}/src/transformez/htdp.py +74 -6
- {transformez-0.4.2 → transformez-0.5.0}/src/transformez/transform.py +131 -78
- {transformez-0.4.2 → transformez-0.5.0}/src/transformez/utils.py +39 -0
- {transformez-0.4.2 → transformez-0.5.0}/AUTHORS.md +0 -0
- {transformez-0.4.2 → transformez-0.5.0}/CONTRIBUTING.md +0 -0
- {transformez-0.4.2 → transformez-0.5.0}/LICENSE +0 -0
- {transformez-0.4.2 → transformez-0.5.0}/pyproject.toml +0 -0
- {transformez-0.4.2 → transformez-0.5.0}/src/transformez/__init__.py +0 -0
- {transformez-0.4.2 → transformez-0.5.0}/src/transformez/_spatial.py +0 -0
- {transformez-0.4.2 → transformez-0.5.0}/src/transformez/hooks.py +0 -0
- {transformez-0.4.2 → transformez-0.5.0}/src/transformez/modules.py +0 -0
- {transformez-0.4.2 → transformez-0.5.0}/src/transformez/srs.py +0 -0
- {transformez-0.4.2 → transformez-0.5.0}/src/transformez/vdatum.py +0 -0
- {transformez-0.4.2 → transformez-0.5.0}/tests/test_cli.py +0 -0
|
@@ -5,6 +5,21 @@ All notable changes to this project will be documented in this file.
|
|
|
5
5
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
|
6
6
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
7
|
|
|
8
|
+
## [0.4.4] - 2026-06-25
|
|
9
|
+
|
|
10
|
+
### Added
|
|
11
|
+
- support for projected input rasters in the raster command
|
|
12
|
+
- add 'save_shift' to transform_raster api/cli
|
|
13
|
+
|
|
14
|
+
### CHANGED
|
|
15
|
+
- rety failed downloads, such as FES
|
|
16
|
+
|
|
17
|
+
## [0.4.3] - 2026-05-04
|
|
18
|
+
|
|
19
|
+
### Added
|
|
20
|
+
- new logo
|
|
21
|
+
- force htdp 3.5.0
|
|
22
|
+
|
|
8
23
|
## [0.3.5] - 2026-04-08
|
|
9
24
|
|
|
10
25
|
### Added
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: transformez
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.5.0
|
|
4
4
|
Summary: A standalone utility for vertical elevation datum transformations.
|
|
5
5
|
Project-URL: Homepage, https://github.com/continuous-dems/transformez
|
|
6
6
|
Project-URL: Issues, https://github.com/continuous-dems/transformez/issues
|
|
@@ -47,16 +47,16 @@ Provides-Extra: preview
|
|
|
47
47
|
Requires-Dist: matplotlib; extra == 'preview'
|
|
48
48
|
Description-Content-Type: text/markdown
|
|
49
49
|
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
50
|
+
<p align="center">
|
|
51
|
+
<a href="https://github.com/continuous-dems">
|
|
52
|
+
<img src="https://raw.githubusercontent.com/continuous-dems/transformez/refs/heads/main/docs/source/_static/continuous_dems_logo_mini_inf.svg" height="80" alt="Continuous DEMs Logo">
|
|
53
|
+
</a>
|
|
54
|
+
</p>
|
|
55
55
|
<h1 align="center">Transformez</h1>
|
|
56
56
|
<p align="center"><strong>Global vertical datum transformations, simplified.</strong></p>
|
|
57
57
|
|
|
58
58
|
<p align="center">
|
|
59
|
-
<a href="https://github.com/continuous-dems/transformez"><img src="https://img.shields.io/badge/version-0.
|
|
59
|
+
<a href="https://github.com/continuous-dems/transformez"><img src="https://img.shields.io/badge/version-0.5.0-blue.svg" alt="Version"></a>
|
|
60
60
|
<a href="LICENSE"><img src="https://img.shields.io/badge/license-MIT-green.svg" alt="License"></a>
|
|
61
61
|
<a href="https://www.python.org/"><img src="https://img.shields.io/badge/python-3.12+-yellow.svg" alt="Python"></a>
|
|
62
62
|
<a href="https://badge.fury.io/py/transformez"><img src="https://badge.fury.io/py/transformez.svg" alt="PyPI version"></a>
|
|
@@ -127,4 +127,5 @@ Would you like to know more? Check out our [Official Documentation](https://tran
|
|
|
127
127
|
## ⚖ License
|
|
128
128
|
|
|
129
129
|
This project is licensed under the MIT License - see the [LICENSE](https://github.com/ciresdem/transformez/blob/main/LICENSE) file for details.
|
|
130
|
+
|
|
130
131
|
Copyright (c) 2010-2026 Regents of the University of Colorado
|
|
@@ -1,13 +1,13 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
1
|
+
<p align="center">
|
|
2
|
+
<a href="https://github.com/continuous-dems">
|
|
3
|
+
<img src="https://raw.githubusercontent.com/continuous-dems/transformez/refs/heads/main/docs/source/_static/continuous_dems_logo_mini_inf.svg" height="80" alt="Continuous DEMs Logo">
|
|
4
|
+
</a>
|
|
5
|
+
</p>
|
|
6
6
|
<h1 align="center">Transformez</h1>
|
|
7
7
|
<p align="center"><strong>Global vertical datum transformations, simplified.</strong></p>
|
|
8
8
|
|
|
9
9
|
<p align="center">
|
|
10
|
-
<a href="https://github.com/continuous-dems/transformez"><img src="https://img.shields.io/badge/version-0.
|
|
10
|
+
<a href="https://github.com/continuous-dems/transformez"><img src="https://img.shields.io/badge/version-0.5.0-blue.svg" alt="Version"></a>
|
|
11
11
|
<a href="LICENSE"><img src="https://img.shields.io/badge/license-MIT-green.svg" alt="License"></a>
|
|
12
12
|
<a href="https://www.python.org/"><img src="https://img.shields.io/badge/python-3.12+-yellow.svg" alt="Python"></a>
|
|
13
13
|
<a href="https://badge.fury.io/py/transformez"><img src="https://badge.fury.io/py/transformez.svg" alt="PyPI version"></a>
|
|
@@ -78,4 +78,5 @@ Would you like to know more? Check out our [Official Documentation](https://tran
|
|
|
78
78
|
## ⚖ License
|
|
79
79
|
|
|
80
80
|
This project is licensed under the MIT License - see the [LICENSE](https://github.com/ciresdem/transformez/blob/main/LICENSE) file for details.
|
|
81
|
+
|
|
81
82
|
Copyright (c) 2010-2026 Regents of the University of Colorado
|
|
@@ -113,6 +113,8 @@ def generate_grid(
|
|
|
113
113
|
increment: Union[str, float],
|
|
114
114
|
datum_in: str,
|
|
115
115
|
datum_out: str,
|
|
116
|
+
epoch_in: str = "2010.0",
|
|
117
|
+
epoch_out: str = "2010.0",
|
|
116
118
|
decay_pixels: int = 100,
|
|
117
119
|
out_fn: Optional[str] = None,
|
|
118
120
|
cache_dir: Optional[str] = None,
|
|
@@ -126,6 +128,8 @@ def generate_grid(
|
|
|
126
128
|
increment: Resolution (e.g., '3s' or 0.0008333).
|
|
127
129
|
datum_in: Source datum (e.g., 'mllw', '5703').
|
|
128
130
|
datum_out: Target datum (e.g., '4979', '6319').
|
|
131
|
+
epoch_in: Source epoch (e.g., '2010.0')
|
|
132
|
+
epoch_out: Target epoch (e.g., '2010.0')
|
|
129
133
|
decay_pixels: Set the pixel decay in case extrapolation is required.
|
|
130
134
|
out_fn: If provided, saves the grid to this file (.tif or .gtx).
|
|
131
135
|
cache_dir: Path to store downloaded grids.
|
|
@@ -167,6 +171,8 @@ def generate_grid(
|
|
|
167
171
|
epsg_out=epsg_out,
|
|
168
172
|
geoid_in=geoid_in,
|
|
169
173
|
geoid_out=geoid_out,
|
|
174
|
+
epoch_in=epoch_in,
|
|
175
|
+
epoch_out=epoch_out,
|
|
170
176
|
decay_pixels=decay_pixels,
|
|
171
177
|
cache_dir=cache_dir,
|
|
172
178
|
use_stations=use_stations,
|
|
@@ -196,6 +202,7 @@ def transform_raster(
|
|
|
196
202
|
z_unit_in: Optional[str] = "auto",
|
|
197
203
|
z_unit_out: Optional[str] = "auto",
|
|
198
204
|
use_stations: bool = False,
|
|
205
|
+
save_shift: bool = False,
|
|
199
206
|
verbose: bool = False,
|
|
200
207
|
) -> Optional[str]:
|
|
201
208
|
"""Apply a vertical datum transformation directly to an existing raster file.
|
|
@@ -209,6 +216,8 @@ def transform_raster(
|
|
|
209
216
|
cache_dir: Path to store downloaded grids.
|
|
210
217
|
z_unit_in: Input DEM z units.
|
|
211
218
|
z_unit_out: Output DEM z units.
|
|
219
|
+
use_stations: Generate the shift grid from available tide stations,
|
|
220
|
+
safe_shift: Save the generated shift raster to disk.
|
|
212
221
|
verbose: Enable debug logging.
|
|
213
222
|
|
|
214
223
|
Returns:
|
|
@@ -216,16 +225,48 @@ def transform_raster(
|
|
|
216
225
|
"""
|
|
217
226
|
|
|
218
227
|
import rasterio
|
|
228
|
+
from rasterio.warp import transform_bounds, reproject, Resampling
|
|
229
|
+
from rasterio.transform import from_bounds
|
|
230
|
+
from fetchez.spatial import Region
|
|
219
231
|
|
|
220
232
|
if not os.path.exists(input_raster):
|
|
221
233
|
logger.error(f"Input raster not found: {input_raster}")
|
|
222
234
|
return None
|
|
223
235
|
|
|
224
236
|
with rasterio.open(input_raster) as src:
|
|
225
|
-
|
|
226
|
-
|
|
237
|
+
native_crs = src.crs
|
|
238
|
+
native_bounds = src.bounds
|
|
239
|
+
native_transform = src.transform
|
|
227
240
|
nx, ny = src.width, src.height
|
|
228
241
|
|
|
242
|
+
is_projected = native_crs.is_projected if native_crs else False
|
|
243
|
+
if is_projected:
|
|
244
|
+
logger.info(
|
|
245
|
+
f"Projected CRS detected ({native_crs}). Extracting WGS84 envelope..."
|
|
246
|
+
)
|
|
247
|
+
w, s, e, n = transform_bounds(native_crs, "EPSG:4326", *native_bounds)
|
|
248
|
+
|
|
249
|
+
buffer = 0.05
|
|
250
|
+
region_obj = Region(w - buffer, e + buffer, s - buffer, n + buffer)
|
|
251
|
+
logger.info(f"Using WGS84 region: {region_obj}")
|
|
252
|
+
|
|
253
|
+
inc_deg = 3.0 / 3600.0
|
|
254
|
+
vt_nx = int((region_obj.xmax - region_obj.xmin) / inc_deg)
|
|
255
|
+
vt_ny = int((region_obj.ymax - region_obj.ymin) / inc_deg)
|
|
256
|
+
else:
|
|
257
|
+
region_obj = Region(
|
|
258
|
+
native_bounds.left,
|
|
259
|
+
native_bounds.right,
|
|
260
|
+
native_bounds.bottom,
|
|
261
|
+
native_bounds.top,
|
|
262
|
+
)
|
|
263
|
+
vt_nx, vt_ny = nx, ny
|
|
264
|
+
|
|
265
|
+
# with rasterio.open(input_raster) as src:
|
|
266
|
+
# bounds = src.bounds
|
|
267
|
+
# region_obj = Region(bounds.left, bounds.right, bounds.bottom, bounds.top)
|
|
268
|
+
# nx, ny = src.width, src.height
|
|
269
|
+
|
|
229
270
|
epsg_in, geoid_in = _parse_datum(datum_in)
|
|
230
271
|
epsg_out, geoid_out = _parse_datum(datum_out)
|
|
231
272
|
|
|
@@ -266,6 +307,47 @@ def transform_raster(
|
|
|
266
307
|
logger.error("Failed to generate shift array for the raster bounds.")
|
|
267
308
|
return None
|
|
268
309
|
|
|
310
|
+
if is_projected:
|
|
311
|
+
logger.info("Warping shift grid back to native raster projection...")
|
|
312
|
+
wgs_transform = from_bounds(
|
|
313
|
+
region_obj.xmin,
|
|
314
|
+
region_obj.ymin,
|
|
315
|
+
region_obj.xmax,
|
|
316
|
+
region_obj.ymax,
|
|
317
|
+
vt_nx,
|
|
318
|
+
vt_ny,
|
|
319
|
+
)
|
|
320
|
+
native_shift_array = np.zeros((ny, nx), dtype=np.float32)
|
|
321
|
+
|
|
322
|
+
reproject(
|
|
323
|
+
source=shift_array,
|
|
324
|
+
destination=native_shift_array,
|
|
325
|
+
src_transform=wgs_transform,
|
|
326
|
+
src_crs="EPSG:4326",
|
|
327
|
+
dst_transform=native_transform,
|
|
328
|
+
dst_crs=native_crs,
|
|
329
|
+
resampling=Resampling.bilinear,
|
|
330
|
+
)
|
|
331
|
+
|
|
332
|
+
shift_array = native_shift_array
|
|
333
|
+
|
|
334
|
+
if save_shift:
|
|
335
|
+
shift_fn = f"{os.path.splitext(output_raster)[0]}_shiftgrid.tif"
|
|
336
|
+
logger.info(f"Saving aligned shift grid to {shift_fn}...")
|
|
337
|
+
with rasterio.open(
|
|
338
|
+
shift_fn,
|
|
339
|
+
"w",
|
|
340
|
+
driver="GTiff",
|
|
341
|
+
height=ny,
|
|
342
|
+
width=nx,
|
|
343
|
+
count=1,
|
|
344
|
+
dtype=shift_array.dtype,
|
|
345
|
+
crs=native_crs,
|
|
346
|
+
transform=native_transform,
|
|
347
|
+
nodata=-9999.0,
|
|
348
|
+
) as dst:
|
|
349
|
+
dst.write(shift_array, 1)
|
|
350
|
+
|
|
269
351
|
success = GridEngine.apply_vertical_shift(
|
|
270
352
|
input_raster,
|
|
271
353
|
shift_array,
|
|
@@ -19,6 +19,12 @@ from fetchez.cli import setup_logging
|
|
|
19
19
|
|
|
20
20
|
from transformez import api
|
|
21
21
|
|
|
22
|
+
TRANSFORMEZ_COMMANDS = {
|
|
23
|
+
"Execution": ["run", "grid", "raster"],
|
|
24
|
+
"Discovery": ["list"],
|
|
25
|
+
"External": ["htdp", "vdatum"],
|
|
26
|
+
}
|
|
27
|
+
|
|
22
28
|
|
|
23
29
|
class TransformezMainGroup(FetchezMainGroup):
|
|
24
30
|
"""A custom Click Group that handles deprecated aliases."""
|
|
@@ -68,7 +74,7 @@ class TransformezMainGroup(FetchezMainGroup):
|
|
|
68
74
|
@click.group(
|
|
69
75
|
name="transform",
|
|
70
76
|
cls=TransformezMainGroup,
|
|
71
|
-
fetchez_commands=
|
|
77
|
+
fetchez_commands=TRANSFORMEZ_COMMANDS,
|
|
72
78
|
)
|
|
73
79
|
@click.version_option(package_name="transformez")
|
|
74
80
|
@click.option("--verbose", is_flag=True, help="Enable verbose debug logging.")
|
|
@@ -296,6 +302,11 @@ def transform_grid(
|
|
|
296
302
|
is_flag=True,
|
|
297
303
|
help="Force RBF interpolation using live tide stations instead of global satellite models.",
|
|
298
304
|
)
|
|
305
|
+
@click.option(
|
|
306
|
+
"--save-shift",
|
|
307
|
+
is_flag=True,
|
|
308
|
+
help="Save the aligned vertical shift grid to disk alongside the output DEM.",
|
|
309
|
+
)
|
|
299
310
|
def transform_raster(
|
|
300
311
|
input_file,
|
|
301
312
|
input_datum,
|
|
@@ -305,6 +316,7 @@ def transform_raster(
|
|
|
305
316
|
out,
|
|
306
317
|
decay_pixels,
|
|
307
318
|
use_stations,
|
|
319
|
+
save_shift,
|
|
308
320
|
):
|
|
309
321
|
"""Apply a vertical datum shift to an existing DEM."""
|
|
310
322
|
|
|
@@ -320,6 +332,7 @@ def transform_raster(
|
|
|
320
332
|
z_unit_in=in_units,
|
|
321
333
|
z_unit_out=out_units,
|
|
322
334
|
use_stations=use_stations,
|
|
335
|
+
save_shift=save_shift,
|
|
323
336
|
verbose=True,
|
|
324
337
|
)
|
|
325
338
|
|
|
@@ -395,12 +408,17 @@ def htdp_group():
|
|
|
395
408
|
|
|
396
409
|
|
|
397
410
|
@htdp_group.command("install", cls=FetchezMainCommand)
|
|
398
|
-
|
|
399
|
-
""
|
|
411
|
+
@click.option(
|
|
412
|
+
"--version",
|
|
413
|
+
default="3.5.0",
|
|
414
|
+
help="HTDP version to install (e.g., 3.3.0, 3.5.0, 3.6.0)",
|
|
415
|
+
)
|
|
416
|
+
def install_htdp(version):
|
|
417
|
+
"""Downloads and compiles the HTDP executable from github."""
|
|
400
418
|
|
|
401
419
|
from transformez.htdp import install_htdp_binary
|
|
402
420
|
|
|
403
|
-
install_htdp_binary()
|
|
421
|
+
install_htdp_binary(version=version)
|
|
404
422
|
|
|
405
423
|
|
|
406
424
|
# --- VDATUM CLI GROUP ---
|
|
@@ -151,11 +151,11 @@ class Datums:
|
|
|
151
151
|
"epoch": 1997.0,
|
|
152
152
|
},
|
|
153
153
|
4979: {
|
|
154
|
-
"name": "WGS_84(
|
|
154
|
+
"name": "WGS_84(G2139)",
|
|
155
155
|
"description": "(NAD_83(2011) used)",
|
|
156
|
-
"htdp_id":
|
|
156
|
+
"htdp_id": 10,
|
|
157
157
|
"uncertainty": 0,
|
|
158
|
-
"epoch":
|
|
158
|
+
"epoch": 2020.0,
|
|
159
159
|
},
|
|
160
160
|
7815: {
|
|
161
161
|
"name": "WGS_84(original)",
|
|
@@ -22,9 +22,17 @@ from rasterio.transform import from_bounds
|
|
|
22
22
|
from scipy import ndimage
|
|
23
23
|
from scipy.interpolate import Rbf
|
|
24
24
|
|
|
25
|
+
os.environ["HDF5_USE_FILE_LOCKING"] = "FALSE"
|
|
26
|
+
|
|
25
27
|
logger = logging.getLogger(__name__)
|
|
26
28
|
|
|
27
29
|
|
|
30
|
+
class GridCorruptionError(Exception):
|
|
31
|
+
"""Raised when a fetched grid is corrupted and needs re-downloading."""
|
|
32
|
+
|
|
33
|
+
pass
|
|
34
|
+
|
|
35
|
+
|
|
28
36
|
def plot_grid(grid_array, region, title="Vertical Shift Preview"):
|
|
29
37
|
"""Plot the transformation grid using Matplotlib."""
|
|
30
38
|
|
|
@@ -139,6 +147,7 @@ class GridEngine:
|
|
|
139
147
|
|
|
140
148
|
try:
|
|
141
149
|
with rasterio.open(fn) as src:
|
|
150
|
+
# logger.info(f"{fn}: {src}")
|
|
142
151
|
src_data = src.read(1).astype(np.float32)
|
|
143
152
|
src_nodata = src.nodata
|
|
144
153
|
|
|
@@ -166,8 +175,57 @@ class GridEngine:
|
|
|
166
175
|
mosaic[valid_mask] = temp_buffer[valid_mask]
|
|
167
176
|
|
|
168
177
|
except Exception as e:
|
|
169
|
-
|
|
178
|
+
error_msg = str(e)
|
|
179
|
+
|
|
180
|
+
if any(
|
|
181
|
+
err in error_msg
|
|
182
|
+
for err in [
|
|
183
|
+
"-101",
|
|
184
|
+
"HDF error",
|
|
185
|
+
"not recognized as a supported file format",
|
|
186
|
+
"RasterioIOError",
|
|
187
|
+
]
|
|
188
|
+
):
|
|
189
|
+
logger.error(f" CRITICAL: Corrupted grid chunk detected in {fn}!")
|
|
190
|
+
|
|
191
|
+
real_path = fn.split(":")[1] if fn.startswith("netcdf:") else fn
|
|
192
|
+
if os.path.exists(real_path):
|
|
193
|
+
logger.warning(
|
|
194
|
+
f"Auto-deleting corrupted cache file to force re-fetch: {real_path}"
|
|
195
|
+
)
|
|
196
|
+
try:
|
|
197
|
+
os.remove(real_path)
|
|
198
|
+
except OSError:
|
|
199
|
+
pass
|
|
200
|
+
|
|
201
|
+
raise GridCorruptionError(f"Corrupted file deleted: {real_path}")
|
|
202
|
+
|
|
203
|
+
logger.exception(f"Failed to reproject {fn}: {e}")
|
|
170
204
|
continue
|
|
205
|
+
# except Exception as e:
|
|
206
|
+
# error_msg = str(e)
|
|
207
|
+
|
|
208
|
+
# if "-101" in error_msg or "HDF error" in error_msg:
|
|
209
|
+
# logger.error(f" CRITICAL: Corrupted NetCDF chunk detected in {fn}!")
|
|
210
|
+
|
|
211
|
+
# # Extract the real file path from the GDAL netcdf string
|
|
212
|
+
# real_path = fn.split(":")[1] if fn.startswith("netcdf:") else fn
|
|
213
|
+
|
|
214
|
+
# if os.path.exists(real_path):
|
|
215
|
+
# logger.warning(f"Auto-deleting corrupted cache file: {real_path}")
|
|
216
|
+
# os.remove(real_path)
|
|
217
|
+
|
|
218
|
+
# raise RuntimeError(
|
|
219
|
+
# f"Transformation aborted to prevent math corruption. "
|
|
220
|
+
# f"The corrupted file has been deleted. Please re-run your command to fetch a fresh copy!"
|
|
221
|
+
# )
|
|
222
|
+
|
|
223
|
+
# # For all other normal errors, log and continue as usual
|
|
224
|
+
# logger.exception(f"Failed to reproject {fn}: {e}")
|
|
225
|
+
# continue
|
|
226
|
+
# except Exception as e:
|
|
227
|
+
# logger.exception(f"Failed to reproject {fn}: {e}")
|
|
228
|
+
# continue
|
|
171
229
|
|
|
172
230
|
# Fill inland areas (decaying to 0) before we clear the remaining NaNs
|
|
173
231
|
# mosaic = GridEngine.fill_nans(mosaic, decay_pixels=decay_pixels)
|
|
@@ -16,6 +16,7 @@ import sys
|
|
|
16
16
|
import os
|
|
17
17
|
import subprocess
|
|
18
18
|
import tempfile
|
|
19
|
+
import shutil
|
|
19
20
|
import logging
|
|
20
21
|
import urllib.request
|
|
21
22
|
import zipfile
|
|
@@ -40,11 +41,16 @@ HAS_HTDP = utils.cmd_check(f"htdp{ae}", htdp_cmd)
|
|
|
40
41
|
class HTDP:
|
|
41
42
|
"""Wrapper for the NGS HTDP software."""
|
|
42
43
|
|
|
43
|
-
def __init__(
|
|
44
|
+
def __init__(
|
|
45
|
+
self, htdp_bin: str = "htdp", version: str = "3.5.0", verbose: bool = True
|
|
46
|
+
):
|
|
44
47
|
self.htdp_bin = htdp_bin
|
|
48
|
+
self.version = version
|
|
45
49
|
self.verbose = verbose
|
|
46
50
|
if not HAS_HTDP:
|
|
47
|
-
logger.error(
|
|
51
|
+
logger.error(
|
|
52
|
+
f"HTDP {self.version} is not installed or not in PATH. Run 'transformez htdp install --version {self.version}"
|
|
53
|
+
)
|
|
48
54
|
|
|
49
55
|
def run_grid(self, region, nx, ny, epsg_in, epsg_out, epoch_in, epoch_out):
|
|
50
56
|
"""Main entry point called by transform.py.
|
|
@@ -90,8 +96,17 @@ class HTDP:
|
|
|
90
96
|
for y_idx, lat in enumerate(lats):
|
|
91
97
|
for x_idx, lon in enumerate(lons):
|
|
92
98
|
# "Lat Lon Height TextID"
|
|
93
|
-
# We use PNT_x_y tags to
|
|
94
|
-
|
|
99
|
+
# We use PNT_x_y tags to map output back to grid
|
|
100
|
+
# htdp_lon = lon * -1.0
|
|
101
|
+
if lon < 0:
|
|
102
|
+
# Western Hemisphere (e.g., -124.5 -> 124.5 W)
|
|
103
|
+
htdp_lon = abs(lon)
|
|
104
|
+
else:
|
|
105
|
+
# Eastern Hemisphere (e.g., 139.0 E -> 221.0 W)
|
|
106
|
+
htdp_lon = 360.0 - lon
|
|
107
|
+
f.write(
|
|
108
|
+
f'{lat:.9f} {htdp_lon:.9f} 0.000 "PNT_{x_idx}_{y_idx}"\n'
|
|
109
|
+
)
|
|
95
110
|
|
|
96
111
|
# Write Control File
|
|
97
112
|
self._write_control(
|
|
@@ -107,7 +122,6 @@ class HTDP:
|
|
|
107
122
|
return np.zeros((ny, nx))
|
|
108
123
|
|
|
109
124
|
grid = self._read_grid(out_fn, (ny, nx))
|
|
110
|
-
|
|
111
125
|
return grid
|
|
112
126
|
|
|
113
127
|
def _read_grid(self, filename: str, shape: Tuple[int, int]) -> np.ndarray:
|
|
@@ -244,7 +258,61 @@ def download_htdp(target_dir=None):
|
|
|
244
258
|
logger.error(f"Failed to download or extract HTDP source: {e}")
|
|
245
259
|
|
|
246
260
|
|
|
247
|
-
def install_htdp_binary():
|
|
261
|
+
def install_htdp_binary(version="3.5.0"):
|
|
262
|
+
"""Downloads and automatically compiles a specific HTDP version from GitHub."""
|
|
263
|
+
|
|
264
|
+
cache_dir = os.path.join(os.getcwd(), "transformez_cache", "bin")
|
|
265
|
+
os.makedirs(cache_dir, exist_ok=True)
|
|
266
|
+
|
|
267
|
+
# Format the GitHub Tag (e.g., "v3.5.0")
|
|
268
|
+
v_tag = f"v{version}" if not version.startswith("v") else version
|
|
269
|
+
clean_version = v_tag.replace("v", "")
|
|
270
|
+
|
|
271
|
+
if sys.platform == "win32":
|
|
272
|
+
logger.info(f"Downloading Windows HTDP executable ({v_tag})...")
|
|
273
|
+
url = f"https://github.com/noaa-ngs/HTDP/releases/download/{v_tag}/htdp.exe"
|
|
274
|
+
exe_path = os.path.join(cache_dir, f"htdp_{clean_version}.exe")
|
|
275
|
+
|
|
276
|
+
try:
|
|
277
|
+
urllib.request.urlretrieve(url, exe_path)
|
|
278
|
+
logger.info(f"HTDP {clean_version} installed successfully to: {exe_path}")
|
|
279
|
+
except Exception as e:
|
|
280
|
+
logger.error(f"Failed to download {v_tag} from GitHub: {e}")
|
|
281
|
+
|
|
282
|
+
else:
|
|
283
|
+
logger.info(f"Downloading Unix HTDP source code ({v_tag})...")
|
|
284
|
+
url = f"https://github.com/noaa-ngs/HTDP/archive/refs/tags/{v_tag}.zip"
|
|
285
|
+
zip_path = os.path.join(cache_dir, f"htdp_{clean_version}.zip")
|
|
286
|
+
|
|
287
|
+
try:
|
|
288
|
+
urllib.request.urlretrieve(url, zip_path)
|
|
289
|
+
|
|
290
|
+
extract_dir = os.path.join(cache_dir, f"htdp_src_{clean_version}")
|
|
291
|
+
with zipfile.ZipFile(zip_path, "r") as z:
|
|
292
|
+
z.extractall(extract_dir)
|
|
293
|
+
os.remove(zip_path)
|
|
294
|
+
|
|
295
|
+
# GitHub extracts to a folder named Repository-Tag (e.g., HTDP-3.5.0)
|
|
296
|
+
source_folder = os.path.join(extract_dir, f"HTDP-{clean_version}")
|
|
297
|
+
fortran_file = os.path.join(source_folder, "htdp.f")
|
|
298
|
+
out_bin = os.path.join(cache_dir, f"htdp_{clean_version}")
|
|
299
|
+
|
|
300
|
+
logger.info(f"Compiling HTDP {clean_version} with gfortran...")
|
|
301
|
+
subprocess.run(
|
|
302
|
+
["gfortran", "-o", out_bin, fortran_file],
|
|
303
|
+
check=True,
|
|
304
|
+
capture_output=True,
|
|
305
|
+
)
|
|
306
|
+
logger.info(f"HTDP compiled successfully to: {out_bin}")
|
|
307
|
+
|
|
308
|
+
# Clean up the raw source files
|
|
309
|
+
shutil.rmtree(extract_dir)
|
|
310
|
+
|
|
311
|
+
except Exception as e:
|
|
312
|
+
logger.error(f"Failed to install HTDP {clean_version}: {e}")
|
|
313
|
+
|
|
314
|
+
|
|
315
|
+
def _install_htdp_binary():
|
|
248
316
|
"""Downloads and automatically compiles HTDP."""
|
|
249
317
|
|
|
250
318
|
cache_dir = os.path.join(os.getcwd(), "transformez_cache", "bin")
|
|
@@ -110,6 +110,7 @@ class VerticalTransform:
|
|
|
110
110
|
region=self.region.to_list(),
|
|
111
111
|
outdir=self.cache_dir,
|
|
112
112
|
threads=2,
|
|
113
|
+
check_size=True,
|
|
113
114
|
**kwargs,
|
|
114
115
|
)
|
|
115
116
|
|
|
@@ -206,7 +207,9 @@ class VerticalTransform:
|
|
|
206
207
|
|
|
207
208
|
return valid
|
|
208
209
|
|
|
209
|
-
def _get_grid(self, provider, name):
|
|
210
|
+
def _get_grid(self, provider, name, max_retries=3):
|
|
211
|
+
|
|
212
|
+
from .grid_engine import GridCorruptionError
|
|
210
213
|
|
|
211
214
|
if not name:
|
|
212
215
|
return np.zeros((self.ny, self.nx))
|
|
@@ -217,98 +220,119 @@ class VerticalTransform:
|
|
|
217
220
|
if "geoid=" in name:
|
|
218
221
|
name = name.split("=")[1]
|
|
219
222
|
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
def get_vdatum_date(gtx_path):
|
|
226
|
-
"""Finds and parses the release date from VDatum metadata files."""
|
|
223
|
+
for attempt in range(max_retries):
|
|
224
|
+
files = self.fetch_grid(provider, datatype=name, query=name)
|
|
225
|
+
if provider == "vdatum":
|
|
226
|
+
import rasterio
|
|
227
|
+
from datetime import datetime
|
|
227
228
|
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
f for f in os.listdir(dir_name) if f.endswith((".met", ".inf"))
|
|
231
|
-
]
|
|
229
|
+
def get_vdatum_date(gtx_path):
|
|
230
|
+
"""Finds and parses the release date from VDatum metadata files."""
|
|
232
231
|
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
try:
|
|
238
|
-
with open(meta_path, "r") as f:
|
|
239
|
-
content = f.read().splitlines()
|
|
232
|
+
dir_name = os.path.dirname(gtx_path)
|
|
233
|
+
meta_files = [
|
|
234
|
+
f for f in os.listdir(dir_name) if f.endswith((".met", ".inf"))
|
|
235
|
+
]
|
|
240
236
|
|
|
241
|
-
if not
|
|
237
|
+
if not meta_files:
|
|
242
238
|
return datetime(1970, 1, 1)
|
|
243
239
|
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
if
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
240
|
+
meta_path = os.path.join(dir_name, meta_files[0])
|
|
241
|
+
try:
|
|
242
|
+
with open(meta_path, "r") as f:
|
|
243
|
+
content = f.read().splitlines()
|
|
244
|
+
|
|
245
|
+
if not content:
|
|
246
|
+
return datetime(1970, 1, 1)
|
|
247
|
+
|
|
248
|
+
first_line = content[0]
|
|
249
|
+
|
|
250
|
+
# Parse the first line "#Mon Jul 08 10:27:07 EDT 2019"
|
|
251
|
+
if first_line.startswith("#"):
|
|
252
|
+
parts = first_line.replace("#", "").split()
|
|
253
|
+
if len(parts) >= 6:
|
|
254
|
+
year = int(parts[-1])
|
|
255
|
+
day = int(parts[2])
|
|
256
|
+
month_map = {
|
|
257
|
+
"Jan": 1,
|
|
258
|
+
"Feb": 2,
|
|
259
|
+
"Mar": 3,
|
|
260
|
+
"Apr": 4,
|
|
261
|
+
"May": 5,
|
|
262
|
+
"Jun": 6,
|
|
263
|
+
"Jul": 7,
|
|
264
|
+
"Aug": 8,
|
|
265
|
+
"Sep": 9,
|
|
266
|
+
"Oct": 10,
|
|
267
|
+
"Nov": 11,
|
|
268
|
+
"Dec": 12,
|
|
269
|
+
}
|
|
270
|
+
month = month_map.get(parts[1][:3].title(), 1)
|
|
271
|
+
return datetime(year, month, day)
|
|
272
|
+
|
|
273
|
+
# Fallback to scanning for "released_date="
|
|
274
|
+
for line in content:
|
|
275
|
+
if "released_date=" in line:
|
|
276
|
+
date_str = line.split("=")[1].strip()
|
|
277
|
+
m, d, y = map(int, date_str.split("/"))
|
|
278
|
+
return datetime(y, m, d)
|
|
279
|
+
|
|
280
|
+
except Exception as e:
|
|
281
|
+
logger.debug(f"Failed to parse date from {meta_path}: {e}")
|
|
275
282
|
|
|
276
|
-
|
|
277
|
-
logger.debug(f"Failed to parse date from {meta_path}: {e}")
|
|
283
|
+
return datetime(1970, 1, 1)
|
|
278
284
|
|
|
279
|
-
|
|
285
|
+
def sort_key(filepath):
|
|
286
|
+
# Time Sorting (Oldest -> Newest)
|
|
287
|
+
date_val = get_vdatum_date(filepath)
|
|
280
288
|
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
289
|
+
try:
|
|
290
|
+
with rasterio.open(filepath) as src:
|
|
291
|
+
b = src.bounds
|
|
292
|
+
area = (b.right - b.left) * (b.top - b.bottom)
|
|
293
|
+
except Exception:
|
|
294
|
+
area = float("inf")
|
|
284
295
|
|
|
285
|
-
|
|
286
|
-
with rasterio.open(filepath) as src:
|
|
287
|
-
b = src.bounds
|
|
288
|
-
area = (b.right - b.left) * (b.top - b.bottom)
|
|
289
|
-
except Exception:
|
|
290
|
-
area = float("inf")
|
|
296
|
+
return (date_val.timestamp(), -area)
|
|
291
297
|
|
|
292
|
-
|
|
298
|
+
files.sort(key=sort_key, reverse=True)
|
|
293
299
|
|
|
294
|
-
files
|
|
300
|
+
if not files:
|
|
301
|
+
return np.zeros((self.ny, self.nx))
|
|
295
302
|
|
|
296
|
-
|
|
297
|
-
|
|
303
|
+
try:
|
|
304
|
+
if provider == "seanoe" or provider == "fes":
|
|
305
|
+
var_name = (
|
|
306
|
+
"lat_elevation" if "lat" in name.lower() else "msl_elevation"
|
|
307
|
+
)
|
|
308
|
+
nc_path = f"netcdf:{files[0]}:{var_name}"
|
|
309
|
+
return GridEngine.load_and_interpolate(
|
|
310
|
+
[nc_path],
|
|
311
|
+
self.region,
|
|
312
|
+
self.nx,
|
|
313
|
+
self.ny,
|
|
314
|
+
decay_pixels=self.decay_pixels,
|
|
315
|
+
)
|
|
298
316
|
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
317
|
+
return GridEngine.load_and_interpolate(
|
|
318
|
+
files, self.region, self.nx, self.ny, decay_pixels=self.decay_pixels
|
|
319
|
+
)
|
|
320
|
+
except GridCorruptionError:
|
|
321
|
+
if attempt < max_retries - 1:
|
|
322
|
+
logger.warning(
|
|
323
|
+
f"Download corruption detected. Retrying fetch (Attempt {attempt + 2}/{max_retries})..."
|
|
324
|
+
)
|
|
325
|
+
continue
|
|
326
|
+
else:
|
|
327
|
+
logger.error(
|
|
328
|
+
"Max retries reached. Could not secure an uncorrupted grid."
|
|
329
|
+
)
|
|
330
|
+
return np.zeros((self.ny, self.nx))
|
|
305
331
|
|
|
306
|
-
return
|
|
307
|
-
files, self.region, self.nx, self.ny, decay_pixels=self.decay_pixels
|
|
308
|
-
)
|
|
332
|
+
return np.zeros((self.ny, self.nx))
|
|
309
333
|
|
|
310
334
|
def _get_htdp_shift(self, epsg_from, epsg_to, epoch_from, epoch_to):
|
|
311
|
-
"""Calculate Frame Shift via HTDP."""
|
|
335
|
+
"""Calculate Frame Shift via HTDP with Fallback."""
|
|
312
336
|
|
|
313
337
|
if epsg_from == epsg_to and epoch_from == epoch_to:
|
|
314
338
|
return np.zeros((self.ny, self.nx))
|
|
@@ -317,12 +341,41 @@ class VerticalTransform:
|
|
|
317
341
|
|
|
318
342
|
try:
|
|
319
343
|
logger.info(f" [HTDP] Frame Shift: EPSG:{epsg_from} -> EPSG:{epsg_to}")
|
|
320
|
-
tool = htdp.HTDP(verbose=False)
|
|
344
|
+
tool = htdp.HTDP(version="3.5.0", verbose=False)
|
|
345
|
+
|
|
346
|
+
# Attempt 1: Full Cross-Epoch Shift (Datum Shift + Crustal Velocity)
|
|
347
|
+
# This seems to fail more often than not!
|
|
321
348
|
grid = tool.run_grid(
|
|
322
349
|
self.region, self.nx, self.ny, epsg_from, epsg_to, epoch_from, epoch_to
|
|
323
350
|
)
|
|
351
|
+
|
|
352
|
+
# --- FALLBACK ---
|
|
353
|
+
if not np.any(grid):
|
|
354
|
+
logger.warning(
|
|
355
|
+
f" [HTDP WARNING] Cross-epoch shift failed (likely outside modeled velocity region for {epoch_from} -> {epoch_to})."
|
|
356
|
+
)
|
|
357
|
+
logger.warning(
|
|
358
|
+
f" [HTDP WARNING] Falling back to static datum shift at Output Epoch {epoch_to}."
|
|
359
|
+
)
|
|
360
|
+
|
|
361
|
+
# Attempt 2: Static Shift (Datum Shift Only)
|
|
362
|
+
grid = tool.run_grid(
|
|
363
|
+
self.region,
|
|
364
|
+
self.nx,
|
|
365
|
+
self.ny,
|
|
366
|
+
epsg_from,
|
|
367
|
+
epsg_to,
|
|
368
|
+
epoch_to,
|
|
369
|
+
epoch_to,
|
|
370
|
+
)
|
|
371
|
+
|
|
324
372
|
if np.any(grid):
|
|
325
373
|
logger.info(f" [HTDP] Component Shift (Mean: {np.mean(grid):.3f}m)")
|
|
374
|
+
else:
|
|
375
|
+
logger.error(
|
|
376
|
+
" [HTDP FATAL] Both dynamic and static shifts failed returning zeros."
|
|
377
|
+
)
|
|
378
|
+
|
|
326
379
|
return grid
|
|
327
380
|
|
|
328
381
|
except Exception as e:
|
|
@@ -16,6 +16,7 @@ import subprocess
|
|
|
16
16
|
import logging
|
|
17
17
|
import numpy as np
|
|
18
18
|
import rasterio
|
|
19
|
+
import shutil
|
|
19
20
|
|
|
20
21
|
logger = logging.getLogger(__name__)
|
|
21
22
|
|
|
@@ -95,3 +96,41 @@ class RasterQuery:
|
|
|
95
96
|
results[valid] = self.data[rows[valid], cols[valid]]
|
|
96
97
|
|
|
97
98
|
return results
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def export_cache(cache_dir=None, output_name="transformez_offline_cache"):
|
|
102
|
+
"""Packs the local transformez cache into a ZIP file for offline use."""
|
|
103
|
+
|
|
104
|
+
if cache_dir is None:
|
|
105
|
+
cache_dir = os.path.join(os.getcwd(), "transformez_cache")
|
|
106
|
+
|
|
107
|
+
if not os.path.exists(cache_dir):
|
|
108
|
+
logger.error(f"[EXPORT FATAL] Cache directory not found at: {cache_dir}")
|
|
109
|
+
logger.error("Run a transformation to populate the cache before exporting.")
|
|
110
|
+
return None
|
|
111
|
+
|
|
112
|
+
# Determine output path
|
|
113
|
+
out_path = os.path.abspath(output_name)
|
|
114
|
+
|
|
115
|
+
logger.info("-" * 60)
|
|
116
|
+
logger.info(f"Packing offline cache bundle from: {cache_dir}")
|
|
117
|
+
logger.info(
|
|
118
|
+
"This may take a minute depending on the size of your downloaded grids..."
|
|
119
|
+
)
|
|
120
|
+
|
|
121
|
+
try:
|
|
122
|
+
# shutil.make_archive(base_name, format, root_dir)
|
|
123
|
+
zip_path = shutil.make_archive(out_path, "zip", cache_dir)
|
|
124
|
+
|
|
125
|
+
# Get human-readable file size
|
|
126
|
+
size_mb = os.path.getsize(zip_path) / (1024 * 1024)
|
|
127
|
+
|
|
128
|
+
logger.info(
|
|
129
|
+
f"Successfully exported offline cache bundle: {zip_path} ({size_mb:.1f} MB)"
|
|
130
|
+
)
|
|
131
|
+
logger.info("-" * 60)
|
|
132
|
+
return zip_path
|
|
133
|
+
|
|
134
|
+
except Exception as e:
|
|
135
|
+
logger.error(f"[EXPORT FATAL] Failed to export cache: {e}")
|
|
136
|
+
return None
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|