orthoseg 0.8.0.dev1__tar.gz → 0.8.0.dev3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {orthoseg-0.8.0.dev1/orthoseg.egg-info → orthoseg-0.8.0.dev3}/PKG-INFO +1 -2
- {orthoseg-0.8.0.dev1 → orthoseg-0.8.0.dev3}/orthoseg/__init__.py +0 -4
- {orthoseg-0.8.0.dev1 → orthoseg-0.8.0.dev3}/orthoseg/helpers/config_helper.py +38 -8
- {orthoseg-0.8.0.dev1 → orthoseg-0.8.0.dev3}/orthoseg/lib/postprocess_predictions.py +57 -2
- {orthoseg-0.8.0.dev1 → orthoseg-0.8.0.dev3}/orthoseg/lib/predicter.py +16 -12
- {orthoseg-0.8.0.dev1 → orthoseg-0.8.0.dev3}/orthoseg/lib/prepare_traindatasets.py +164 -90
- {orthoseg-0.8.0.dev1 → orthoseg-0.8.0.dev3}/orthoseg/load_sampleprojects.py +10 -36
- {orthoseg-0.8.0.dev1 → orthoseg-0.8.0.dev3}/orthoseg/model/model_factory.py +3 -4
- {orthoseg-0.8.0.dev1 → orthoseg-0.8.0.dev3}/orthoseg/model/model_helper.py +63 -14
- {orthoseg-0.8.0.dev1 → orthoseg-0.8.0.dev3}/orthoseg/model/model_weights_helper.py +0 -1
- {orthoseg-0.8.0.dev1 → orthoseg-0.8.0.dev3}/orthoseg/postprocess.py +12 -3
- {orthoseg-0.8.0.dev1 → orthoseg-0.8.0.dev3}/orthoseg/predict.py +3 -3
- {orthoseg-0.8.0.dev1 → orthoseg-0.8.0.dev3}/orthoseg/project_defaults.ini +12 -5
- {orthoseg-0.8.0.dev1 → orthoseg-0.8.0.dev3}/orthoseg/util/config_util.py +6 -1
- {orthoseg-0.8.0.dev1 → orthoseg-0.8.0.dev3}/orthoseg/util/image_util.py +3 -1
- {orthoseg-0.8.0.dev1 → orthoseg-0.8.0.dev3}/orthoseg/validate.py +1 -1
- orthoseg-0.8.0.dev3/orthoseg/version.txt +1 -0
- {orthoseg-0.8.0.dev1 → orthoseg-0.8.0.dev3/orthoseg.egg-info}/PKG-INFO +1 -2
- {orthoseg-0.8.0.dev1 → orthoseg-0.8.0.dev3}/orthoseg.egg-info/requires.txt +0 -1
- {orthoseg-0.8.0.dev1 → orthoseg-0.8.0.dev3}/pyproject.toml +1 -1
- {orthoseg-0.8.0.dev1 → orthoseg-0.8.0.dev3}/setup.py +0 -1
- {orthoseg-0.8.0.dev1 → orthoseg-0.8.0.dev3}/tests/test_cleanup.py +46 -33
- {orthoseg-0.8.0.dev1 → orthoseg-0.8.0.dev3}/tests/test_config_helper.py +110 -54
- {orthoseg-0.8.0.dev1 → orthoseg-0.8.0.dev3}/tests/test_data.py +3 -3
- {orthoseg-0.8.0.dev1 → orthoseg-0.8.0.dev3}/tests/test_end2end.py +59 -92
- {orthoseg-0.8.0.dev1 → orthoseg-0.8.0.dev3}/tests/test_helper.py +13 -45
- {orthoseg-0.8.0.dev1 → orthoseg-0.8.0.dev3}/tests/test_image_util.py +16 -15
- orthoseg-0.8.0.dev3/tests/test_load_images.py +87 -0
- {orthoseg-0.8.0.dev1 → orthoseg-0.8.0.dev3}/tests/test_load_sampleprojects.py +14 -9
- {orthoseg-0.8.0.dev1 → orthoseg-0.8.0.dev3}/tests/test_model_helper.py +21 -6
- {orthoseg-0.8.0.dev1 → orthoseg-0.8.0.dev3}/tests/test_postprocess.py +2 -3
- orthoseg-0.8.0.dev3/tests/test_postprocess_predictions.py +375 -0
- {orthoseg-0.8.0.dev1 → orthoseg-0.8.0.dev3}/tests/test_predict.py +30 -27
- {orthoseg-0.8.0.dev1 → orthoseg-0.8.0.dev3}/tests/test_prepare_traindata.py +87 -0
- {orthoseg-0.8.0.dev1 → orthoseg-0.8.0.dev3}/tests/test_train.py +2 -2
- {orthoseg-0.8.0.dev1 → orthoseg-0.8.0.dev3}/tests/test_validate.py +15 -13
- orthoseg-0.8.0.dev1/orthoseg/version.txt +0 -1
- orthoseg-0.8.0.dev1/tests/test_load_images.py +0 -73
- orthoseg-0.8.0.dev1/tests/test_postprocess_predictions.py +0 -178
- {orthoseg-0.8.0.dev1 → orthoseg-0.8.0.dev3}/LICENSE.MD +0 -0
- {orthoseg-0.8.0.dev1 → orthoseg-0.8.0.dev3}/MANIFEST.in +0 -0
- {orthoseg-0.8.0.dev1 → orthoseg-0.8.0.dev3}/README.md +0 -0
- {orthoseg-0.8.0.dev1 → orthoseg-0.8.0.dev3}/orthoseg/_compat.py +0 -0
- {orthoseg-0.8.0.dev1 → orthoseg-0.8.0.dev3}/orthoseg/helpers/__init__.py +0 -0
- {orthoseg-0.8.0.dev1 → orthoseg-0.8.0.dev3}/orthoseg/helpers/email_helper.py +0 -0
- {orthoseg-0.8.0.dev1 → orthoseg-0.8.0.dev3}/orthoseg/helpers/vectorfile_helper.py +0 -0
- {orthoseg-0.8.0.dev1 → orthoseg-0.8.0.dev3}/orthoseg/lib/__init__.py +0 -0
- {orthoseg-0.8.0.dev1 → orthoseg-0.8.0.dev3}/orthoseg/lib/cleanup.py +0 -0
- {orthoseg-0.8.0.dev1 → orthoseg-0.8.0.dev3}/orthoseg/lib/trainer.py +0 -0
- {orthoseg-0.8.0.dev1 → orthoseg-0.8.0.dev3}/orthoseg/load_images.py +0 -0
- {orthoseg-0.8.0.dev1 → orthoseg-0.8.0.dev3}/orthoseg/model/__init__.py +0 -0
- {orthoseg-0.8.0.dev1 → orthoseg-0.8.0.dev3}/orthoseg/py.typed +0 -0
- {orthoseg-0.8.0.dev1 → orthoseg-0.8.0.dev3}/orthoseg/scriptrunner.py +0 -0
- {orthoseg-0.8.0.dev1 → orthoseg-0.8.0.dev3}/orthoseg/scriptrunner_defaults.ini +0 -0
- {orthoseg-0.8.0.dev1 → orthoseg-0.8.0.dev3}/orthoseg/train.py +0 -0
- {orthoseg-0.8.0.dev1 → orthoseg-0.8.0.dev3}/orthoseg/util/__init__.py +0 -0
- {orthoseg-0.8.0.dev1 → orthoseg-0.8.0.dev3}/orthoseg/util/_processing_util.py +0 -0
- {orthoseg-0.8.0.dev1 → orthoseg-0.8.0.dev3}/orthoseg/util/data.py +0 -0
- {orthoseg-0.8.0.dev1 → orthoseg-0.8.0.dev3}/orthoseg/util/general_util.py +0 -0
- {orthoseg-0.8.0.dev1 → orthoseg-0.8.0.dev3}/orthoseg/util/log_util.py +0 -0
- {orthoseg-0.8.0.dev1 → orthoseg-0.8.0.dev3}/orthoseg/util/progress_util.py +0 -0
- {orthoseg-0.8.0.dev1 → orthoseg-0.8.0.dev3}/orthoseg/util/vector_util.py +0 -0
- {orthoseg-0.8.0.dev1 → orthoseg-0.8.0.dev3}/orthoseg.egg-info/SOURCES.txt +0 -0
- {orthoseg-0.8.0.dev1 → orthoseg-0.8.0.dev3}/orthoseg.egg-info/dependency_links.txt +0 -0
- {orthoseg-0.8.0.dev1 → orthoseg-0.8.0.dev3}/orthoseg.egg-info/entry_points.txt +0 -0
- {orthoseg-0.8.0.dev1 → orthoseg-0.8.0.dev3}/orthoseg.egg-info/top_level.txt +0 -0
- {orthoseg-0.8.0.dev1 → orthoseg-0.8.0.dev3}/setup.cfg +0 -0
- {orthoseg-0.8.0.dev1 → orthoseg-0.8.0.dev3}/tests/__init__.py +0 -0
- {orthoseg-0.8.0.dev1 → orthoseg-0.8.0.dev3}/tests/test_config_util.py +0 -0
- {orthoseg-0.8.0.dev1 → orthoseg-0.8.0.dev3}/tests/test_model_factory.py +0 -0
- {orthoseg-0.8.0.dev1 → orthoseg-0.8.0.dev3}/tests/test_model_weights_helper.py +0 -0
- {orthoseg-0.8.0.dev1 → orthoseg-0.8.0.dev3}/tests/test_predicter.py +0 -0
- {orthoseg-0.8.0.dev1 → orthoseg-0.8.0.dev3}/tests/test_trainer.py +0 -0
- {orthoseg-0.8.0.dev1 → orthoseg-0.8.0.dev3}/tests/test_vector_util.py +0 -0
- {orthoseg-0.8.0.dev1 → orthoseg-0.8.0.dev3}/tests/test_vectorfile_helper.py +0 -0
- {orthoseg-0.8.0.dev1 → orthoseg-0.8.0.dev3}/tests/test_version.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: orthoseg
|
|
3
|
-
Version: 0.8.0.
|
|
3
|
+
Version: 0.8.0.dev3
|
|
4
4
|
Summary: Package to make it easier to segment orthophotos.
|
|
5
5
|
Home-page: https://github.com/orthoseg/orthoseg
|
|
6
6
|
Author: Pieter Roggemans
|
|
@@ -11,7 +11,6 @@ Requires-Python: >=3.10
|
|
|
11
11
|
Description-Content-Type: text/markdown
|
|
12
12
|
License-File: LICENSE.MD
|
|
13
13
|
Requires-Dist: gdal
|
|
14
|
-
Requires-Dist: gdown
|
|
15
14
|
Requires-Dist: geofileops>=0.10
|
|
16
15
|
Requires-Dist: geopandas>=1.0
|
|
17
16
|
Requires-Dist: matplotlib
|
|
@@ -7,8 +7,6 @@ from pathlib import Path
|
|
|
7
7
|
if "KERAS_BACKEND" not in os.environ:
|
|
8
8
|
os.environ["KERAS_BACKEND"] = "tensorflow"
|
|
9
9
|
|
|
10
|
-
import keras
|
|
11
|
-
|
|
12
10
|
# ruff: noqa: F401
|
|
13
11
|
from orthoseg._compat import __version__
|
|
14
12
|
from orthoseg.load_images import load_images
|
|
@@ -16,5 +14,3 @@ from orthoseg.postprocess import postprocess
|
|
|
16
14
|
from orthoseg.predict import predict
|
|
17
15
|
from orthoseg.train import train
|
|
18
16
|
from orthoseg.validate import validate
|
|
19
|
-
|
|
20
|
-
print(f"Using Keras backend: {keras.backend.backend()}")
|
|
@@ -30,6 +30,9 @@ from orthoseg.util.image_util import (
|
|
|
30
30
|
has_switched_axes,
|
|
31
31
|
)
|
|
32
32
|
|
|
33
|
+
# Activate gdal to raise exceptions instead of printing/returning errors/errorcodes
|
|
34
|
+
gdal.UseExceptions()
|
|
35
|
+
|
|
33
36
|
# Get a logger...
|
|
34
37
|
logger = logging.getLogger(__name__)
|
|
35
38
|
|
|
@@ -182,6 +185,33 @@ def read_orthoseg_config(config_path: Path, overrules: list[str] | None = None):
|
|
|
182
185
|
)
|
|
183
186
|
dirs["projects_dir"] = projects_dir_absolute.as_posix()
|
|
184
187
|
|
|
188
|
+
# If postprocess.output_style_path is a relative path, resolve it towards the
|
|
189
|
+
# location of the project config file.
|
|
190
|
+
output_style_path_is_default_value = (
|
|
191
|
+
True
|
|
192
|
+
if postprocess.get("output_style_path") == f"{segment_subject}.qml"
|
|
193
|
+
else False
|
|
194
|
+
)
|
|
195
|
+
output_style_path = postprocess.getpath("output_style_path")
|
|
196
|
+
if output_style_path is not None and not output_style_path.is_absolute():
|
|
197
|
+
output_style_path_absolute = (config_path.parent / output_style_path).resolve()
|
|
198
|
+
postprocess["output_style_path"] = output_style_path_absolute.as_posix()
|
|
199
|
+
output_style_path = output_style_path_absolute
|
|
200
|
+
|
|
201
|
+
if output_style_path is not None and not output_style_path.exists():
|
|
202
|
+
if output_style_path_is_default_value:
|
|
203
|
+
logger.warning(
|
|
204
|
+
"postprocess.output_style_path is set to the default value and file "
|
|
205
|
+
f"doesn't exist (warning only): {output_style_path}"
|
|
206
|
+
)
|
|
207
|
+
# Set to None so that the postprocess will not try to use it.
|
|
208
|
+
postprocess["output_style_path"] = None
|
|
209
|
+
else:
|
|
210
|
+
raise FileNotFoundError(
|
|
211
|
+
"postprocess.output_style_path is configured explicitly, but file "
|
|
212
|
+
f"doesn't exist: {output_style_path}"
|
|
213
|
+
)
|
|
214
|
+
|
|
185
215
|
# Some version-specific defaults
|
|
186
216
|
if train.get("save_format") is None:
|
|
187
217
|
train["save_format"] = "keras" if KERAS_GTE_3 else "h5"
|
|
@@ -368,9 +398,13 @@ def _read_layer_config(layer_config_filepath: Path) -> dict:
|
|
|
368
398
|
].getint("nb_concurrent_calls", fallback=1)
|
|
369
399
|
|
|
370
400
|
# Check if a region of interest is specified as file or bbox
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
401
|
+
roi_filepath = layer_config[image_layer].getpath("roi_filepath", fallback=None)
|
|
402
|
+
if roi_filepath is not None and not roi_filepath.is_absolute():
|
|
403
|
+
# path is relative, so resolve towards the location of imagelayers file
|
|
404
|
+
roi_filepath = layer_config_filepath.parent / roi_filepath
|
|
405
|
+
roi_filepath = roi_filepath.resolve()
|
|
406
|
+
image_layers[image_layer]["roi_filepath"] = roi_filepath
|
|
407
|
+
|
|
374
408
|
bbox_tuple = None
|
|
375
409
|
if layer_config.has_option(image_layer, "bbox"):
|
|
376
410
|
bbox_list = layer_config[image_layer].getlist("bbox")
|
|
@@ -698,10 +732,6 @@ def _gdal_virtual_file_path(layersource) -> Path:
|
|
|
698
732
|
)
|
|
699
733
|
gdal_options = gdal.TranslateOptions(format="VRT")
|
|
700
734
|
|
|
701
|
-
gdal.Translate(
|
|
702
|
-
str(output_path),
|
|
703
|
-
input_url,
|
|
704
|
-
options=gdal_options,
|
|
705
|
-
)
|
|
735
|
+
gdal.Translate(destName=str(output_path), srcDS=input_url, options=gdal_options)
|
|
706
736
|
|
|
707
737
|
return output_path
|
|
@@ -41,6 +41,7 @@ def postprocess_predictions(
|
|
|
41
41
|
simplify_algorithm: str | None = None,
|
|
42
42
|
simplify_tolerance: float = 1,
|
|
43
43
|
simplify_lookahead: int = 8,
|
|
44
|
+
output_style_path: Path | None = None,
|
|
44
45
|
keep_original_file: bool = True,
|
|
45
46
|
keep_intermediary_files: bool = True,
|
|
46
47
|
nb_parallel: int = -1,
|
|
@@ -63,6 +64,8 @@ def postprocess_predictions(
|
|
|
63
64
|
simplify_tolerance (float): Tolerance to use for the simplification.
|
|
64
65
|
Defaults to 1.
|
|
65
66
|
simplify_lookahead (int): Lookahead to use for simplification. Default to 8.
|
|
67
|
+
output_style_path (Path, optional): Path to a QGIS .qml style file. If
|
|
68
|
+
specified and output is a GeoPackage, the style is added to the layer.
|
|
66
69
|
nb_parallel (int, optional): number of cpu's to use for postprocessing.
|
|
67
70
|
Use all cpu's if it is -1. Defaults to -1.
|
|
68
71
|
force: False to skip results that already exist, true to
|
|
@@ -184,9 +187,13 @@ def postprocess_predictions(
|
|
|
184
187
|
# (input_path) is renamed to ..._orig.gpkg
|
|
185
188
|
if dissolve or reclassify_to_neighbour_query or simplify_algorithm:
|
|
186
189
|
original_file = input_path.parent / f"{input_path.stem}_orig.gpkg"
|
|
190
|
+
if original_file.exists():
|
|
191
|
+
gfo.remove(original_file)
|
|
187
192
|
input_path.rename(original_file)
|
|
188
193
|
shutil.copy(src=curr_output_path, dst=input_path)
|
|
189
194
|
|
|
195
|
+
_add_output_layer_style(output_path=input_path, output_style_path=output_style_path)
|
|
196
|
+
|
|
190
197
|
# Cleanup original file
|
|
191
198
|
if not keep_original_file:
|
|
192
199
|
original_file.unlink()
|
|
@@ -199,6 +206,34 @@ def postprocess_predictions(
|
|
|
199
206
|
return output_paths
|
|
200
207
|
|
|
201
208
|
|
|
209
|
+
def _add_output_layer_style(output_path: Path, output_style_path: Path | None) -> None:
|
|
210
|
+
"""Add a QML layer style to a GeoPackage output if configured."""
|
|
211
|
+
if output_style_path is None:
|
|
212
|
+
return
|
|
213
|
+
|
|
214
|
+
if output_path.suffix.lower() != ".gpkg":
|
|
215
|
+
raise ValueError(
|
|
216
|
+
"postprocess.output_style_path is set but output is not a GeoPackage: "
|
|
217
|
+
f"{output_path}"
|
|
218
|
+
)
|
|
219
|
+
|
|
220
|
+
if not output_style_path.exists():
|
|
221
|
+
raise FileNotFoundError(
|
|
222
|
+
f"postprocess.output_style_path doesn't exist: {output_style_path}"
|
|
223
|
+
)
|
|
224
|
+
|
|
225
|
+
qml_style = output_style_path.read_text(encoding="utf-8")
|
|
226
|
+
layer_name = gfo.get_only_layer(output_path)
|
|
227
|
+
|
|
228
|
+
gfo.add_layerstyle(
|
|
229
|
+
path=output_path,
|
|
230
|
+
layer=layer_name,
|
|
231
|
+
name=output_style_path.stem,
|
|
232
|
+
qml=qml_style,
|
|
233
|
+
use_as_default=True,
|
|
234
|
+
)
|
|
235
|
+
|
|
236
|
+
|
|
202
237
|
def read_prediction_file(filepath: Path) -> gpd.GeoDataFrame | None:
|
|
203
238
|
"""Read the prediction file specified.
|
|
204
239
|
|
|
@@ -243,7 +278,7 @@ def postprocess_prediction_to_file(
|
|
|
243
278
|
output_vector_path (Path | None, optional): The path to write the polygonized
|
|
244
279
|
prediction to. If None, no polygonized result is written. Defaults to None.
|
|
245
280
|
output_image_dir (Path | None, optional): The directory to write the prediction
|
|
246
|
-
to as an image. If None, no image of
|
|
281
|
+
to as an image. If None, no image of the prediction is written.
|
|
247
282
|
Defaults to None.
|
|
248
283
|
input_image_filepath (Path | None, optional): The path to the image that was
|
|
249
284
|
predicted image. Is mandatory if output_image_dir is not None.
|
|
@@ -351,6 +386,7 @@ def postprocess_for_evaluation(
|
|
|
351
386
|
output_suffix: str | None = None,
|
|
352
387
|
input_image_dir: Path | None = None,
|
|
353
388
|
input_mask_dir: Path | None = None,
|
|
389
|
+
max_similarity_to_save: float | None = 0.999,
|
|
354
390
|
border_pixels_to_ignore: int = 0,
|
|
355
391
|
):
|
|
356
392
|
"""This function postprocesses a prediction for manual evaluation.
|
|
@@ -380,6 +416,9 @@ def postprocess_for_evaluation(
|
|
|
380
416
|
output_suffix (Optional[str], optional): _description_. Defaults to None.
|
|
381
417
|
input_image_dir (Optional[Path], optional): _description_. Defaults to None.
|
|
382
418
|
input_mask_dir (Optional[Path], optional): _description_. Defaults to None.
|
|
419
|
+
max_similarity_to_save (float | None, optional): Maximum similarity
|
|
420
|
+
for which evaluation outputs should be saved. If the measured
|
|
421
|
+
similarity is higher, evaluation outputs are skipped. Defaults to 0.999.
|
|
383
422
|
border_pixels_to_ignore (int, optional): number of pixels at all borders that
|
|
384
423
|
should be ignored. Defaults to 0.
|
|
385
424
|
|
|
@@ -394,6 +433,11 @@ def postprocess_for_evaluation(
|
|
|
394
433
|
logger.debug(f"Start postprocess for {image_pred_filepath}")
|
|
395
434
|
all_black = False
|
|
396
435
|
try:
|
|
436
|
+
if max_similarity_to_save is not None and (
|
|
437
|
+
max_similarity_to_save < 0 or max_similarity_to_save > 1
|
|
438
|
+
):
|
|
439
|
+
raise ValueError("max_similarity_to_save should be between 0 and 1")
|
|
440
|
+
|
|
397
441
|
# If the image wasn't saved, it must have been all black
|
|
398
442
|
if image_pred_filepath is None:
|
|
399
443
|
all_black = True
|
|
@@ -466,6 +510,17 @@ def postprocess_for_evaluation(
|
|
|
466
510
|
np.array(np.equal(mask_arr, image_pred_uint8_cleaned_bin)).sum()
|
|
467
511
|
/ image_pred_uint8_cleaned_bin.size
|
|
468
512
|
)
|
|
513
|
+
|
|
514
|
+
# If the prediction is already very accurate, skip saving evaluation
|
|
515
|
+
# artifacts to keep output focused on less accurate predictions.
|
|
516
|
+
if (
|
|
517
|
+
max_similarity_to_save is not None
|
|
518
|
+
and similarity > max_similarity_to_save
|
|
519
|
+
):
|
|
520
|
+
if image_pred_filepath is not None and image_pred_filepath.exists():
|
|
521
|
+
image_pred_filepath.unlink()
|
|
522
|
+
return
|
|
523
|
+
|
|
469
524
|
pred_prefix_str = f"{similarity:0.3f}_"
|
|
470
525
|
|
|
471
526
|
# Write mask
|
|
@@ -1046,10 +1101,10 @@ def clean_and_save_prediction(
|
|
|
1046
1101
|
# If the cleaned result contains useful values or in evaluate mode... save
|
|
1047
1102
|
if (
|
|
1048
1103
|
min_probability == 0
|
|
1104
|
+
or evaluate_mode
|
|
1049
1105
|
or np.any(
|
|
1050
1106
|
image_pred_uint8_cleaned_curr >= math.floor(min_probability * 255)
|
|
1051
1107
|
)
|
|
1052
|
-
or evaluate_mode is True
|
|
1053
1108
|
):
|
|
1054
1109
|
# Find the class name in the classes list
|
|
1055
1110
|
class_name = None
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
"""Module with high-level operations to segment images."""
|
|
2
2
|
|
|
3
3
|
import csv
|
|
4
|
-
import datetime
|
|
5
4
|
import json
|
|
6
5
|
import logging
|
|
7
6
|
import multiprocessing
|
|
@@ -11,7 +10,9 @@ import time
|
|
|
11
10
|
import traceback
|
|
12
11
|
from collections.abc import Callable
|
|
13
12
|
from concurrent import futures
|
|
13
|
+
from datetime import datetime
|
|
14
14
|
from pathlib import Path
|
|
15
|
+
from time import perf_counter
|
|
15
16
|
from typing import Any
|
|
16
17
|
|
|
17
18
|
import geofileops as gfo
|
|
@@ -445,7 +446,7 @@ def _predict_layer(
|
|
|
445
446
|
nb_to_predict = nb_images
|
|
446
447
|
nb_done = 0
|
|
447
448
|
nb_errors = 0
|
|
448
|
-
|
|
449
|
+
read_sleep_last_logged = None
|
|
449
450
|
progress = None
|
|
450
451
|
read_queue: dict[futures.Future, Path] = {}
|
|
451
452
|
postp_queue: dict[futures.Future, Path] = {}
|
|
@@ -467,7 +468,7 @@ def _predict_layer(
|
|
|
467
468
|
):
|
|
468
469
|
# Start looping.
|
|
469
470
|
# If ready to stop, the code below will break
|
|
470
|
-
perf_time_start = datetime.
|
|
471
|
+
perf_time_start = datetime.now()
|
|
471
472
|
while True:
|
|
472
473
|
# If we are ready, stop!
|
|
473
474
|
if (
|
|
@@ -593,9 +594,13 @@ def _predict_layer(
|
|
|
593
594
|
break
|
|
594
595
|
|
|
595
596
|
# Wait a bit for images to be read, then try finding images again
|
|
596
|
-
|
|
597
|
+
cur_counter = perf_counter()
|
|
598
|
+
if (
|
|
599
|
+
not read_sleep_last_logged
|
|
600
|
+
or (read_sleep_last_logged - cur_counter) > 5
|
|
601
|
+
):
|
|
597
602
|
logger.info("Wait for images to be read")
|
|
598
|
-
|
|
603
|
+
read_sleep_last_logged = cur_counter
|
|
599
604
|
time.sleep(0.01)
|
|
600
605
|
|
|
601
606
|
# If sufficient images in predict queue -> predict
|
|
@@ -603,22 +608,21 @@ def _predict_layer(
|
|
|
603
608
|
if len(predict_queue) == batch_size or (
|
|
604
609
|
last_image_reached and len(predict_queue) > 0
|
|
605
610
|
):
|
|
606
|
-
|
|
607
|
-
perf_time_now = datetime.datetime.now()
|
|
611
|
+
perf_time_now = datetime.now()
|
|
608
612
|
perfinfo += f"waiting for read took {perf_time_now - perf_time_start}"
|
|
609
613
|
perf_time_start = perf_time_now
|
|
610
614
|
|
|
611
615
|
# Predict!
|
|
612
616
|
# --------
|
|
613
617
|
logger.debug(f"Start prediction for {len(predict_queue)} images")
|
|
614
|
-
perf_time_start = datetime.
|
|
618
|
+
perf_time_start = datetime.now()
|
|
615
619
|
curr_batch_image_list = [
|
|
616
620
|
batch_image_info["image_data"] for batch_image_info in predict_queue
|
|
617
621
|
]
|
|
618
622
|
batch_image_arr = np.stack(curr_batch_image_list)
|
|
619
623
|
batch_pred_arr = model.predict_on_batch(batch_image_arr)
|
|
620
624
|
|
|
621
|
-
perf_time_now = datetime.
|
|
625
|
+
perf_time_now = datetime.now()
|
|
622
626
|
perfinfo += f", predict took {perf_time_now - perf_time_start}"
|
|
623
627
|
perf_time_start = perf_time_now
|
|
624
628
|
|
|
@@ -679,7 +683,7 @@ def _predict_layer(
|
|
|
679
683
|
predict_queue = []
|
|
680
684
|
|
|
681
685
|
# Collect performance debugging info
|
|
682
|
-
time_taken = datetime.
|
|
686
|
+
time_taken = datetime.now() - perf_time_start
|
|
683
687
|
perfinfo += f", scheduling postprocessings took {time_taken}"
|
|
684
688
|
perf_time_start = perf_time_now
|
|
685
689
|
|
|
@@ -741,7 +745,7 @@ def _predict_layer(
|
|
|
741
745
|
if postp_sleep_logged:
|
|
742
746
|
logger.info("Waited enough for postprocessing to catch up...")
|
|
743
747
|
|
|
744
|
-
perf_time_now = datetime.
|
|
748
|
+
perf_time_now = datetime.now()
|
|
745
749
|
perfinfo += (
|
|
746
750
|
f", after postproces: took {perf_time_now - perf_time_start}"
|
|
747
751
|
)
|
|
@@ -773,7 +777,7 @@ def _predict_layer(
|
|
|
773
777
|
# Wait till number below thresshold to avoid huge waiting list (and
|
|
774
778
|
# memory issues)
|
|
775
779
|
if len(write_queue) > nb_parallel_postprocess * 2: # pragma: no cover
|
|
776
|
-
if
|
|
780
|
+
if write_sleep_logged:
|
|
777
781
|
logger.info("Writing takes longer than prediction, so wait")
|
|
778
782
|
write_sleep_logged = True
|
|
779
783
|
time.sleep(0.01)
|
|
@@ -5,7 +5,9 @@ import math
|
|
|
5
5
|
import pprint
|
|
6
6
|
import shutil
|
|
7
7
|
import warnings
|
|
8
|
+
from concurrent import futures
|
|
8
9
|
from pathlib import Path
|
|
10
|
+
from typing import Any
|
|
9
11
|
|
|
10
12
|
import geofileops as gfo
|
|
11
13
|
import geopandas as gpd
|
|
@@ -18,7 +20,7 @@ import shapely
|
|
|
18
20
|
import shapely.geometry as sh_geom
|
|
19
21
|
from PIL import Image
|
|
20
22
|
|
|
21
|
-
from orthoseg.util import image_util, vector_util
|
|
23
|
+
from orthoseg.util import _processing_util, image_util, vector_util
|
|
22
24
|
from orthoseg.util.progress_util import ProgressLogger
|
|
23
25
|
|
|
24
26
|
# Get a logger...
|
|
@@ -144,6 +146,7 @@ def prepare_traindatasets(
|
|
|
144
146
|
ssl_verify: bool | str = True,
|
|
145
147
|
only_validate: bool = False,
|
|
146
148
|
force: bool = False,
|
|
149
|
+
nb_concurrent_calls: int = 4,
|
|
147
150
|
) -> tuple[Path, int]:
|
|
148
151
|
"""This function prepares training data for the vector labels provided.
|
|
149
152
|
|
|
@@ -181,13 +184,27 @@ def prepare_traindatasets(
|
|
|
181
184
|
only_validate (bool, optional): True to only validate the input label data.
|
|
182
185
|
force (bool, opitional): True to force recreation of output files.
|
|
183
186
|
Defaults to False.
|
|
187
|
+
nb_concurrent_calls (int, optional): Number of concurrent worker threads used
|
|
188
|
+
to prepare images/masks per traindata type. A conservative upper bound is
|
|
189
|
+
enforced to avoid overwhelming remote image services. Defaults to 4.
|
|
184
190
|
"""
|
|
191
|
+
if nb_concurrent_calls < 1:
|
|
192
|
+
raise ValueError("nb_concurrent_calls should be >= 1")
|
|
193
|
+
|
|
194
|
+
# Keep concurrency bounded to reduce the risk of overloading remote WMS services.
|
|
195
|
+
nb_concurrent_calls_effective = min(nb_concurrent_calls, 8)
|
|
196
|
+
if nb_concurrent_calls_effective != nb_concurrent_calls:
|
|
197
|
+
logger.info(
|
|
198
|
+
"nb_concurrent_calls capped to 8 for service safety: "
|
|
199
|
+
f"{nb_concurrent_calls} -> {nb_concurrent_calls_effective}"
|
|
200
|
+
)
|
|
201
|
+
|
|
185
202
|
# Check if the first class is named "background"
|
|
186
203
|
if len(classes) == 0:
|
|
187
|
-
raise
|
|
204
|
+
raise ValueError("No classes specified")
|
|
188
205
|
elif next(iter(classes)).lower() != "background":
|
|
189
206
|
classes_str = pprint.pformat(classes, sort_dicts=False, width=50)
|
|
190
|
-
raise
|
|
207
|
+
raise ValueError(
|
|
191
208
|
f"By convention, the first class must be called background!\n{classes_str}"
|
|
192
209
|
)
|
|
193
210
|
|
|
@@ -295,6 +312,7 @@ def prepare_traindatasets(
|
|
|
295
312
|
logger.info(f"Get images for {nb_todo} labels")
|
|
296
313
|
|
|
297
314
|
for traindata_type in traindata_types:
|
|
315
|
+
previous_imagedata_image_dir = None
|
|
298
316
|
if dataversion_mostrecent is not None:
|
|
299
317
|
previous_dataversion_dir = training_dir / f"{dataversion_mostrecent:02d}"
|
|
300
318
|
previous_imagedata_image_dir = (
|
|
@@ -312,100 +330,67 @@ def prepare_traindatasets(
|
|
|
312
330
|
if output_dir and not output_dir.exists():
|
|
313
331
|
output_dir.mkdir(parents=True, exist_ok=True)
|
|
314
332
|
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
image_filepath: Path | None = Path(
|
|
344
|
-
shutil.copy(
|
|
345
|
-
src=previous_imagedata_image_dir / output_filename,
|
|
346
|
-
dst=output_imagedata_image_dir / output_filename,
|
|
347
|
-
)
|
|
348
|
-
)
|
|
349
|
-
pgw_filename = output_filename.replace(".png", ".pgw")
|
|
350
|
-
if (previous_imagedata_image_dir / pgw_filename).exists():
|
|
351
|
-
shutil.copy(
|
|
352
|
-
src=previous_imagedata_image_dir / pgw_filename,
|
|
353
|
-
dst=output_imagedata_image_dir / pgw_filename,
|
|
354
|
-
)
|
|
355
|
-
else:
|
|
356
|
-
# Get the image from the WMS service.
|
|
357
|
-
image_filepath = image_util.load_image_to_file(
|
|
358
|
-
layersources=image_layers[image_layer]["layersources"],
|
|
359
|
-
output_dir=output_imagedata_image_dir,
|
|
360
|
-
crs=labellocations_gdf.crs,
|
|
361
|
-
bbox=img_bbox.bounds,
|
|
362
|
-
size=(image_pixel_width, image_pixel_height),
|
|
333
|
+
future_queue: dict[futures.Future, tuple[str, int]] = {}
|
|
334
|
+
with _processing_util.PooledExecutorFactory(
|
|
335
|
+
worker_type="processes",
|
|
336
|
+
max_workers=nb_concurrent_calls_effective,
|
|
337
|
+
) as pool:
|
|
338
|
+
for labeldata_idx, (labellocations_gdf, labels_to_burn_gdf) in enumerate(
|
|
339
|
+
labeldata
|
|
340
|
+
):
|
|
341
|
+
# Get the label locations for this traindata type
|
|
342
|
+
labellocations_curr_gdf = labellocations_gdf[
|
|
343
|
+
labellocations_gdf["traindata_type"] == traindata_type
|
|
344
|
+
]
|
|
345
|
+
|
|
346
|
+
# Loop trough all locations labels to get an image for each of them
|
|
347
|
+
for label_tuple in labellocations_curr_gdf.itertuples():
|
|
348
|
+
assert labellocations_gdf.crs is not None
|
|
349
|
+
future = pool.submit(
|
|
350
|
+
_prepare_image_and_mask,
|
|
351
|
+
img_bbox=label_tuple.geometry,
|
|
352
|
+
image_layer=label_tuple.image_layer,
|
|
353
|
+
image_crs=labellocations_gdf.crs,
|
|
354
|
+
labels_to_burn_gdf=labels_to_burn_gdf,
|
|
355
|
+
image_layers=image_layers,
|
|
356
|
+
output_imagedata_image_dir=output_imagedata_image_dir,
|
|
357
|
+
output_imagedata_mask_dir=output_imagedata_mask_dir,
|
|
358
|
+
previous_imagedata_image_dir=previous_imagedata_image_dir,
|
|
359
|
+
image_pixel_width=image_pixel_width,
|
|
360
|
+
image_pixel_height=image_pixel_height,
|
|
363
361
|
ssl_verify=ssl_verify,
|
|
364
|
-
|
|
365
|
-
# image_format_save=image_util.FORMAT_TIFF,
|
|
366
|
-
image_pixels_ignore_border=image_layers[image_layer][
|
|
367
|
-
"image_pixels_ignore_border"
|
|
368
|
-
],
|
|
369
|
-
transparent=False,
|
|
370
|
-
layername_in_filename=True,
|
|
371
|
-
output_filename=output_filename,
|
|
362
|
+
force=force,
|
|
372
363
|
)
|
|
364
|
+
future_queue[future] = (traindata_type, labeldata_idx)
|
|
373
365
|
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
366
|
+
future_errors = []
|
|
367
|
+
for future in futures.as_completed(future_queue):
|
|
368
|
+
# ProgressLogger is not thread-safe, so only update it in the main
|
|
369
|
+
# thread when a future is completed.
|
|
370
|
+
progress.step()
|
|
371
|
+
|
|
372
|
+
try:
|
|
373
|
+
future.result()
|
|
374
|
+
except Exception as ex:
|
|
375
|
+
current_traindata_type, labeldata_idx = future_queue[future]
|
|
376
|
+
future_errors.append(
|
|
377
|
+
(
|
|
378
|
+
current_traindata_type,
|
|
379
|
+
labeldata_idx,
|
|
380
|
+
ex,
|
|
381
|
+
)
|
|
385
382
|
)
|
|
386
|
-
.replace(".jpg", ".png")
|
|
387
|
-
)
|
|
388
383
|
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
if len(labels_for_layer_gdf) == 0:
|
|
398
|
-
logger.info(f"No polygons to burn for {image_layer=}!")
|
|
399
|
-
_create_mask(
|
|
400
|
-
input_image_filepath=image_filepath,
|
|
401
|
-
output_mask_filepath=mask_filepath,
|
|
402
|
-
labels_to_burn_gdf=labels_for_layer_gdf,
|
|
403
|
-
force=force,
|
|
384
|
+
if len(future_errors) > 0:
|
|
385
|
+
error_messages = [
|
|
386
|
+
(f"[{traindata_type_error}] labeldata_idx={labeldata_idx}: {error}")
|
|
387
|
+
for traindata_type_error, labeldata_idx, error in future_errors
|
|
388
|
+
]
|
|
389
|
+
raise RuntimeError(
|
|
390
|
+
"Errors while preparing training images/masks:\n"
|
|
391
|
+
+ "\n".join(error_messages)
|
|
404
392
|
)
|
|
405
393
|
|
|
406
|
-
# Log the progress estimated time remaining.
|
|
407
|
-
progress.step()
|
|
408
|
-
|
|
409
394
|
# If everything went fine, rename output_tmp_dir to the final output_dir
|
|
410
395
|
output_tmp_dir.rename(training_dataversion_dir)
|
|
411
396
|
|
|
@@ -770,6 +755,95 @@ def create_tmp_dir(
|
|
|
770
755
|
return tmp_dir
|
|
771
756
|
|
|
772
757
|
|
|
758
|
+
def _prepare_image_and_mask(
|
|
759
|
+
img_bbox: sh_geom.base.BaseGeometry,
|
|
760
|
+
image_layer: str,
|
|
761
|
+
image_crs: Any,
|
|
762
|
+
labels_to_burn_gdf: gpd.GeoDataFrame,
|
|
763
|
+
image_layers: dict,
|
|
764
|
+
output_imagedata_image_dir: Path,
|
|
765
|
+
output_imagedata_mask_dir: Path,
|
|
766
|
+
previous_imagedata_image_dir: Path | None,
|
|
767
|
+
image_pixel_width: int,
|
|
768
|
+
image_pixel_height: int,
|
|
769
|
+
ssl_verify: bool | str,
|
|
770
|
+
force: bool,
|
|
771
|
+
) -> None:
|
|
772
|
+
"""Prepare one training image and its mask."""
|
|
773
|
+
# Prepare file name for the image
|
|
774
|
+
output_filename = image_util.create_filename(
|
|
775
|
+
crs=image_crs,
|
|
776
|
+
bbox=img_bbox.bounds,
|
|
777
|
+
size=(image_pixel_width, image_pixel_height),
|
|
778
|
+
image_format=image_util.FORMAT_PNG,
|
|
779
|
+
layername="_".join(image_layers[image_layer]["layersources"][0].layernames),
|
|
780
|
+
)
|
|
781
|
+
|
|
782
|
+
# If the image exists already in the previous version, reuse it.
|
|
783
|
+
if (
|
|
784
|
+
previous_imagedata_image_dir is not None
|
|
785
|
+
and (previous_imagedata_image_dir / output_filename).exists()
|
|
786
|
+
):
|
|
787
|
+
image_filepath: Path | None = Path(
|
|
788
|
+
shutil.copy(
|
|
789
|
+
src=previous_imagedata_image_dir / output_filename,
|
|
790
|
+
dst=output_imagedata_image_dir / output_filename,
|
|
791
|
+
)
|
|
792
|
+
)
|
|
793
|
+
pgw_filename = output_filename.replace(".png", ".pgw")
|
|
794
|
+
if (previous_imagedata_image_dir / pgw_filename).exists():
|
|
795
|
+
shutil.copy(
|
|
796
|
+
src=previous_imagedata_image_dir / pgw_filename,
|
|
797
|
+
dst=output_imagedata_image_dir / pgw_filename,
|
|
798
|
+
)
|
|
799
|
+
else:
|
|
800
|
+
# Get the image from the WMS service.
|
|
801
|
+
image_filepath = image_util.load_image_to_file(
|
|
802
|
+
layersources=image_layers[image_layer]["layersources"],
|
|
803
|
+
output_dir=output_imagedata_image_dir,
|
|
804
|
+
crs=image_crs,
|
|
805
|
+
bbox=img_bbox.bounds,
|
|
806
|
+
size=(image_pixel_width, image_pixel_height),
|
|
807
|
+
ssl_verify=ssl_verify,
|
|
808
|
+
image_format=image_util.FORMAT_PNG,
|
|
809
|
+
# image_format_save=image_util.FORMAT_TIFF,
|
|
810
|
+
image_pixels_ignore_border=image_layers[image_layer][
|
|
811
|
+
"image_pixels_ignore_border"
|
|
812
|
+
],
|
|
813
|
+
transparent=False,
|
|
814
|
+
layername_in_filename=True,
|
|
815
|
+
output_filename=output_filename,
|
|
816
|
+
)
|
|
817
|
+
|
|
818
|
+
if image_filepath is None:
|
|
819
|
+
logger.warning(f"Could not get image at {img_bbox.wkt}")
|
|
820
|
+
return
|
|
821
|
+
|
|
822
|
+
# Create a mask corresponding with the image file
|
|
823
|
+
# Mask should never be in a lossy format -> png!
|
|
824
|
+
mask_filepath = Path(
|
|
825
|
+
str(image_filepath)
|
|
826
|
+
.replace(
|
|
827
|
+
str(output_imagedata_image_dir),
|
|
828
|
+
str(output_imagedata_mask_dir),
|
|
829
|
+
)
|
|
830
|
+
.replace(".jpg", ".png")
|
|
831
|
+
)
|
|
832
|
+
|
|
833
|
+
# Only keep the labels that are meant for this image layer
|
|
834
|
+
labels_for_layer_gdf = (
|
|
835
|
+
labels_to_burn_gdf.loc[labels_to_burn_gdf["image_layer"] == image_layer]
|
|
836
|
+
).copy()
|
|
837
|
+
if len(labels_for_layer_gdf) == 0:
|
|
838
|
+
logger.info(f"No polygons to burn for {image_layer=}!")
|
|
839
|
+
_create_mask(
|
|
840
|
+
input_image_filepath=image_filepath,
|
|
841
|
+
output_mask_filepath=mask_filepath,
|
|
842
|
+
labels_to_burn_gdf=labels_for_layer_gdf,
|
|
843
|
+
force=force,
|
|
844
|
+
)
|
|
845
|
+
|
|
846
|
+
|
|
773
847
|
def _create_mask(
|
|
774
848
|
input_image_filepath: Path,
|
|
775
849
|
output_mask_filepath: Path,
|