orthoseg 0.8.0.dev2__tar.gz → 0.8.0.dev3__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. {orthoseg-0.8.0.dev2/orthoseg.egg-info → orthoseg-0.8.0.dev3}/PKG-INFO +1 -2
  2. {orthoseg-0.8.0.dev2 → orthoseg-0.8.0.dev3}/orthoseg/__init__.py +0 -4
  3. {orthoseg-0.8.0.dev2 → orthoseg-0.8.0.dev3}/orthoseg/helpers/config_helper.py +38 -8
  4. {orthoseg-0.8.0.dev2 → orthoseg-0.8.0.dev3}/orthoseg/lib/postprocess_predictions.py +57 -2
  5. {orthoseg-0.8.0.dev2 → orthoseg-0.8.0.dev3}/orthoseg/lib/predicter.py +16 -12
  6. {orthoseg-0.8.0.dev2 → orthoseg-0.8.0.dev3}/orthoseg/lib/prepare_traindatasets.py +164 -90
  7. {orthoseg-0.8.0.dev2 → orthoseg-0.8.0.dev3}/orthoseg/load_sampleprojects.py +10 -36
  8. {orthoseg-0.8.0.dev2 → orthoseg-0.8.0.dev3}/orthoseg/model/model_factory.py +1 -2
  9. {orthoseg-0.8.0.dev2 → orthoseg-0.8.0.dev3}/orthoseg/model/model_helper.py +52 -13
  10. {orthoseg-0.8.0.dev2 → orthoseg-0.8.0.dev3}/orthoseg/model/model_weights_helper.py +0 -1
  11. {orthoseg-0.8.0.dev2 → orthoseg-0.8.0.dev3}/orthoseg/postprocess.py +12 -3
  12. {orthoseg-0.8.0.dev2 → orthoseg-0.8.0.dev3}/orthoseg/predict.py +3 -3
  13. {orthoseg-0.8.0.dev2 → orthoseg-0.8.0.dev3}/orthoseg/project_defaults.ini +12 -5
  14. {orthoseg-0.8.0.dev2 → orthoseg-0.8.0.dev3}/orthoseg/util/config_util.py +6 -1
  15. {orthoseg-0.8.0.dev2 → orthoseg-0.8.0.dev3}/orthoseg/util/image_util.py +3 -1
  16. {orthoseg-0.8.0.dev2 → orthoseg-0.8.0.dev3}/orthoseg/validate.py +1 -1
  17. orthoseg-0.8.0.dev3/orthoseg/version.txt +1 -0
  18. {orthoseg-0.8.0.dev2 → orthoseg-0.8.0.dev3/orthoseg.egg-info}/PKG-INFO +1 -2
  19. {orthoseg-0.8.0.dev2 → orthoseg-0.8.0.dev3}/orthoseg.egg-info/requires.txt +0 -1
  20. {orthoseg-0.8.0.dev2 → orthoseg-0.8.0.dev3}/pyproject.toml +1 -1
  21. {orthoseg-0.8.0.dev2 → orthoseg-0.8.0.dev3}/setup.py +0 -1
  22. {orthoseg-0.8.0.dev2 → orthoseg-0.8.0.dev3}/tests/test_cleanup.py +46 -33
  23. {orthoseg-0.8.0.dev2 → orthoseg-0.8.0.dev3}/tests/test_config_helper.py +110 -54
  24. {orthoseg-0.8.0.dev2 → orthoseg-0.8.0.dev3}/tests/test_data.py +3 -3
  25. {orthoseg-0.8.0.dev2 → orthoseg-0.8.0.dev3}/tests/test_end2end.py +59 -92
  26. {orthoseg-0.8.0.dev2 → orthoseg-0.8.0.dev3}/tests/test_helper.py +13 -45
  27. {orthoseg-0.8.0.dev2 → orthoseg-0.8.0.dev3}/tests/test_image_util.py +16 -15
  28. orthoseg-0.8.0.dev3/tests/test_load_images.py +87 -0
  29. {orthoseg-0.8.0.dev2 → orthoseg-0.8.0.dev3}/tests/test_load_sampleprojects.py +14 -9
  30. {orthoseg-0.8.0.dev2 → orthoseg-0.8.0.dev3}/tests/test_model_helper.py +3 -6
  31. {orthoseg-0.8.0.dev2 → orthoseg-0.8.0.dev3}/tests/test_postprocess.py +2 -3
  32. orthoseg-0.8.0.dev3/tests/test_postprocess_predictions.py +375 -0
  33. {orthoseg-0.8.0.dev2 → orthoseg-0.8.0.dev3}/tests/test_predict.py +30 -27
  34. {orthoseg-0.8.0.dev2 → orthoseg-0.8.0.dev3}/tests/test_prepare_traindata.py +87 -0
  35. {orthoseg-0.8.0.dev2 → orthoseg-0.8.0.dev3}/tests/test_train.py +2 -2
  36. {orthoseg-0.8.0.dev2 → orthoseg-0.8.0.dev3}/tests/test_validate.py +15 -13
  37. orthoseg-0.8.0.dev2/orthoseg/version.txt +0 -1
  38. orthoseg-0.8.0.dev2/tests/test_load_images.py +0 -73
  39. orthoseg-0.8.0.dev2/tests/test_postprocess_predictions.py +0 -178
  40. {orthoseg-0.8.0.dev2 → orthoseg-0.8.0.dev3}/LICENSE.MD +0 -0
  41. {orthoseg-0.8.0.dev2 → orthoseg-0.8.0.dev3}/MANIFEST.in +0 -0
  42. {orthoseg-0.8.0.dev2 → orthoseg-0.8.0.dev3}/README.md +0 -0
  43. {orthoseg-0.8.0.dev2 → orthoseg-0.8.0.dev3}/orthoseg/_compat.py +0 -0
  44. {orthoseg-0.8.0.dev2 → orthoseg-0.8.0.dev3}/orthoseg/helpers/__init__.py +0 -0
  45. {orthoseg-0.8.0.dev2 → orthoseg-0.8.0.dev3}/orthoseg/helpers/email_helper.py +0 -0
  46. {orthoseg-0.8.0.dev2 → orthoseg-0.8.0.dev3}/orthoseg/helpers/vectorfile_helper.py +0 -0
  47. {orthoseg-0.8.0.dev2 → orthoseg-0.8.0.dev3}/orthoseg/lib/__init__.py +0 -0
  48. {orthoseg-0.8.0.dev2 → orthoseg-0.8.0.dev3}/orthoseg/lib/cleanup.py +0 -0
  49. {orthoseg-0.8.0.dev2 → orthoseg-0.8.0.dev3}/orthoseg/lib/trainer.py +0 -0
  50. {orthoseg-0.8.0.dev2 → orthoseg-0.8.0.dev3}/orthoseg/load_images.py +0 -0
  51. {orthoseg-0.8.0.dev2 → orthoseg-0.8.0.dev3}/orthoseg/model/__init__.py +0 -0
  52. {orthoseg-0.8.0.dev2 → orthoseg-0.8.0.dev3}/orthoseg/py.typed +0 -0
  53. {orthoseg-0.8.0.dev2 → orthoseg-0.8.0.dev3}/orthoseg/scriptrunner.py +0 -0
  54. {orthoseg-0.8.0.dev2 → orthoseg-0.8.0.dev3}/orthoseg/scriptrunner_defaults.ini +0 -0
  55. {orthoseg-0.8.0.dev2 → orthoseg-0.8.0.dev3}/orthoseg/train.py +0 -0
  56. {orthoseg-0.8.0.dev2 → orthoseg-0.8.0.dev3}/orthoseg/util/__init__.py +0 -0
  57. {orthoseg-0.8.0.dev2 → orthoseg-0.8.0.dev3}/orthoseg/util/_processing_util.py +0 -0
  58. {orthoseg-0.8.0.dev2 → orthoseg-0.8.0.dev3}/orthoseg/util/data.py +0 -0
  59. {orthoseg-0.8.0.dev2 → orthoseg-0.8.0.dev3}/orthoseg/util/general_util.py +0 -0
  60. {orthoseg-0.8.0.dev2 → orthoseg-0.8.0.dev3}/orthoseg/util/log_util.py +0 -0
  61. {orthoseg-0.8.0.dev2 → orthoseg-0.8.0.dev3}/orthoseg/util/progress_util.py +0 -0
  62. {orthoseg-0.8.0.dev2 → orthoseg-0.8.0.dev3}/orthoseg/util/vector_util.py +0 -0
  63. {orthoseg-0.8.0.dev2 → orthoseg-0.8.0.dev3}/orthoseg.egg-info/SOURCES.txt +0 -0
  64. {orthoseg-0.8.0.dev2 → orthoseg-0.8.0.dev3}/orthoseg.egg-info/dependency_links.txt +0 -0
  65. {orthoseg-0.8.0.dev2 → orthoseg-0.8.0.dev3}/orthoseg.egg-info/entry_points.txt +0 -0
  66. {orthoseg-0.8.0.dev2 → orthoseg-0.8.0.dev3}/orthoseg.egg-info/top_level.txt +0 -0
  67. {orthoseg-0.8.0.dev2 → orthoseg-0.8.0.dev3}/setup.cfg +0 -0
  68. {orthoseg-0.8.0.dev2 → orthoseg-0.8.0.dev3}/tests/__init__.py +0 -0
  69. {orthoseg-0.8.0.dev2 → orthoseg-0.8.0.dev3}/tests/test_config_util.py +0 -0
  70. {orthoseg-0.8.0.dev2 → orthoseg-0.8.0.dev3}/tests/test_model_factory.py +0 -0
  71. {orthoseg-0.8.0.dev2 → orthoseg-0.8.0.dev3}/tests/test_model_weights_helper.py +0 -0
  72. {orthoseg-0.8.0.dev2 → orthoseg-0.8.0.dev3}/tests/test_predicter.py +0 -0
  73. {orthoseg-0.8.0.dev2 → orthoseg-0.8.0.dev3}/tests/test_trainer.py +0 -0
  74. {orthoseg-0.8.0.dev2 → orthoseg-0.8.0.dev3}/tests/test_vector_util.py +0 -0
  75. {orthoseg-0.8.0.dev2 → orthoseg-0.8.0.dev3}/tests/test_vectorfile_helper.py +0 -0
  76. {orthoseg-0.8.0.dev2 → orthoseg-0.8.0.dev3}/tests/test_version.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: orthoseg
3
- Version: 0.8.0.dev2
3
+ Version: 0.8.0.dev3
4
4
  Summary: Package to make it easier to segment orthophotos.
5
5
  Home-page: https://github.com/orthoseg/orthoseg
6
6
  Author: Pieter Roggemans
@@ -11,7 +11,6 @@ Requires-Python: >=3.10
11
11
  Description-Content-Type: text/markdown
12
12
  License-File: LICENSE.MD
13
13
  Requires-Dist: gdal
14
- Requires-Dist: gdown
15
14
  Requires-Dist: geofileops>=0.10
16
15
  Requires-Dist: geopandas>=1.0
17
16
  Requires-Dist: matplotlib
@@ -7,8 +7,6 @@ from pathlib import Path
7
7
  if "KERAS_BACKEND" not in os.environ:
8
8
  os.environ["KERAS_BACKEND"] = "tensorflow"
9
9
 
10
- import keras
11
-
12
10
  # ruff: noqa: F401
13
11
  from orthoseg._compat import __version__
14
12
  from orthoseg.load_images import load_images
@@ -16,5 +14,3 @@ from orthoseg.postprocess import postprocess
16
14
  from orthoseg.predict import predict
17
15
  from orthoseg.train import train
18
16
  from orthoseg.validate import validate
19
-
20
- print(f"Using Keras backend: {keras.backend.backend()}")
@@ -30,6 +30,9 @@ from orthoseg.util.image_util import (
30
30
  has_switched_axes,
31
31
  )
32
32
 
33
+ # Activate gdal to raise exceptions instead of printing/returning errors/errorcodes
34
+ gdal.UseExceptions()
35
+
33
36
  # Get a logger...
34
37
  logger = logging.getLogger(__name__)
35
38
 
@@ -182,6 +185,33 @@ def read_orthoseg_config(config_path: Path, overrules: list[str] | None = None):
182
185
  )
183
186
  dirs["projects_dir"] = projects_dir_absolute.as_posix()
184
187
 
188
+ # If postprocess.output_style_path is a relative path, resolve it towards the
189
+ # location of the project config file.
190
+ output_style_path_is_default_value = (
191
+ True
192
+ if postprocess.get("output_style_path") == f"{segment_subject}.qml"
193
+ else False
194
+ )
195
+ output_style_path = postprocess.getpath("output_style_path")
196
+ if output_style_path is not None and not output_style_path.is_absolute():
197
+ output_style_path_absolute = (config_path.parent / output_style_path).resolve()
198
+ postprocess["output_style_path"] = output_style_path_absolute.as_posix()
199
+ output_style_path = output_style_path_absolute
200
+
201
+ if output_style_path is not None and not output_style_path.exists():
202
+ if output_style_path_is_default_value:
203
+ logger.warning(
204
+ "postprocess.output_style_path is set to the default value and file "
205
+ f"doesn't exist (warning only): {output_style_path}"
206
+ )
207
+ # Set to None so that the postprocess will not try to use it.
208
+ postprocess["output_style_path"] = None
209
+ else:
210
+ raise FileNotFoundError(
211
+ "postprocess.output_style_path is configured explicitly, but file "
212
+ f"doesn't exist: {output_style_path}"
213
+ )
214
+
185
215
  # Some version-specific defaults
186
216
  if train.get("save_format") is None:
187
217
  train["save_format"] = "keras" if KERAS_GTE_3 else "h5"
@@ -368,9 +398,13 @@ def _read_layer_config(layer_config_filepath: Path) -> dict:
368
398
  ].getint("nb_concurrent_calls", fallback=1)
369
399
 
370
400
  # Check if a region of interest is specified as file or bbox
371
- image_layers[image_layer]["roi_filepath"] = layer_config[image_layer].getpath(
372
- "roi_filepath", fallback=None
373
- )
401
+ roi_filepath = layer_config[image_layer].getpath("roi_filepath", fallback=None)
402
+ if roi_filepath is not None and not roi_filepath.is_absolute():
403
+ # path is relative, so resolve towards the location of imagelayers file
404
+ roi_filepath = layer_config_filepath.parent / roi_filepath
405
+ roi_filepath = roi_filepath.resolve()
406
+ image_layers[image_layer]["roi_filepath"] = roi_filepath
407
+
374
408
  bbox_tuple = None
375
409
  if layer_config.has_option(image_layer, "bbox"):
376
410
  bbox_list = layer_config[image_layer].getlist("bbox")
@@ -698,10 +732,6 @@ def _gdal_virtual_file_path(layersource) -> Path:
698
732
  )
699
733
  gdal_options = gdal.TranslateOptions(format="VRT")
700
734
 
701
- gdal.Translate(
702
- str(output_path),
703
- input_url,
704
- options=gdal_options,
705
- )
735
+ gdal.Translate(destName=str(output_path), srcDS=input_url, options=gdal_options)
706
736
 
707
737
  return output_path
@@ -41,6 +41,7 @@ def postprocess_predictions(
41
41
  simplify_algorithm: str | None = None,
42
42
  simplify_tolerance: float = 1,
43
43
  simplify_lookahead: int = 8,
44
+ output_style_path: Path | None = None,
44
45
  keep_original_file: bool = True,
45
46
  keep_intermediary_files: bool = True,
46
47
  nb_parallel: int = -1,
@@ -63,6 +64,8 @@ def postprocess_predictions(
63
64
  simplify_tolerance (float): Tolerance to use for the simplification.
64
65
  Defaults to 1.
65
66
  simplify_lookahead (int): Lookahead to use for simplification. Default to 8.
67
+ output_style_path (Path, optional): Path to a QGIS .qml style file. If
68
+ specified and output is a GeoPackage, the style is added to the layer.
66
69
  nb_parallel (int, optional): number of cpu's to use for postprocessing.
67
70
  Use all cpu's if it is -1. Defaults to -1.
68
71
  force: False to skip results that already exist, true to
@@ -184,9 +187,13 @@ def postprocess_predictions(
184
187
  # (input_path) is renamed to ..._orig.gpkg
185
188
  if dissolve or reclassify_to_neighbour_query or simplify_algorithm:
186
189
  original_file = input_path.parent / f"{input_path.stem}_orig.gpkg"
190
+ if original_file.exists():
191
+ gfo.remove(original_file)
187
192
  input_path.rename(original_file)
188
193
  shutil.copy(src=curr_output_path, dst=input_path)
189
194
 
195
+ _add_output_layer_style(output_path=input_path, output_style_path=output_style_path)
196
+
190
197
  # Cleanup original file
191
198
  if not keep_original_file:
192
199
  original_file.unlink()
@@ -199,6 +206,34 @@ def postprocess_predictions(
199
206
  return output_paths
200
207
 
201
208
 
209
+ def _add_output_layer_style(output_path: Path, output_style_path: Path | None) -> None:
210
+ """Add a QML layer style to a GeoPackage output if configured."""
211
+ if output_style_path is None:
212
+ return
213
+
214
+ if output_path.suffix.lower() != ".gpkg":
215
+ raise ValueError(
216
+ "postprocess.output_style_path is set but output is not a GeoPackage: "
217
+ f"{output_path}"
218
+ )
219
+
220
+ if not output_style_path.exists():
221
+ raise FileNotFoundError(
222
+ f"postprocess.output_style_path doesn't exist: {output_style_path}"
223
+ )
224
+
225
+ qml_style = output_style_path.read_text(encoding="utf-8")
226
+ layer_name = gfo.get_only_layer(output_path)
227
+
228
+ gfo.add_layerstyle(
229
+ path=output_path,
230
+ layer=layer_name,
231
+ name=output_style_path.stem,
232
+ qml=qml_style,
233
+ use_as_default=True,
234
+ )
235
+
236
+
202
237
  def read_prediction_file(filepath: Path) -> gpd.GeoDataFrame | None:
203
238
  """Read the prediction file specified.
204
239
 
@@ -243,7 +278,7 @@ def postprocess_prediction_to_file(
243
278
  output_vector_path (Path | None, optional): The path to write the polygonized
244
279
  prediction to. If None, no polygonized result is written. Defaults to None.
245
280
  output_image_dir (Path | None, optional): The directory to write the prediction
246
- to as an image. If None, no image of tyhe prediction is written.
281
+ to as an image. If None, no image of the prediction is written.
247
282
  Defaults to None.
248
283
  input_image_filepath (Path | None, optional): The path to the image that was
249
284
  predicted image. Is mandatory if output_image_dir is not None.
@@ -351,6 +386,7 @@ def postprocess_for_evaluation(
351
386
  output_suffix: str | None = None,
352
387
  input_image_dir: Path | None = None,
353
388
  input_mask_dir: Path | None = None,
389
+ max_similarity_to_save: float | None = 0.999,
354
390
  border_pixels_to_ignore: int = 0,
355
391
  ):
356
392
  """This function postprocesses a prediction for manual evaluation.
@@ -380,6 +416,9 @@ def postprocess_for_evaluation(
380
416
  output_suffix (Optional[str], optional): _description_. Defaults to None.
381
417
  input_image_dir (Optional[Path], optional): _description_. Defaults to None.
382
418
  input_mask_dir (Optional[Path], optional): _description_. Defaults to None.
419
+ max_similarity_to_save (float | None, optional): Maximum similarity
420
+ for which evaluation outputs should be saved. If the measured
421
+ similarity is higher, evaluation outputs are skipped. Defaults to 0.999.
383
422
  border_pixels_to_ignore (int, optional): number of pixels at all borders that
384
423
  should be ignored. Defaults to 0.
385
424
 
@@ -394,6 +433,11 @@ def postprocess_for_evaluation(
394
433
  logger.debug(f"Start postprocess for {image_pred_filepath}")
395
434
  all_black = False
396
435
  try:
436
+ if max_similarity_to_save is not None and (
437
+ max_similarity_to_save < 0 or max_similarity_to_save > 1
438
+ ):
439
+ raise ValueError("max_similarity_to_save should be between 0 and 1")
440
+
397
441
  # If the image wasn't saved, it must have been all black
398
442
  if image_pred_filepath is None:
399
443
  all_black = True
@@ -466,6 +510,17 @@ def postprocess_for_evaluation(
466
510
  np.array(np.equal(mask_arr, image_pred_uint8_cleaned_bin)).sum()
467
511
  / image_pred_uint8_cleaned_bin.size
468
512
  )
513
+
514
+ # If the prediction is already very accurate, skip saving evaluation
515
+ # artifacts to keep output focused on less accurate predictions.
516
+ if (
517
+ max_similarity_to_save is not None
518
+ and similarity > max_similarity_to_save
519
+ ):
520
+ if image_pred_filepath is not None and image_pred_filepath.exists():
521
+ image_pred_filepath.unlink()
522
+ return
523
+
469
524
  pred_prefix_str = f"{similarity:0.3f}_"
470
525
 
471
526
  # Write mask
@@ -1046,10 +1101,10 @@ def clean_and_save_prediction(
1046
1101
  # If the cleaned result contains useful values or in evaluate mode... save
1047
1102
  if (
1048
1103
  min_probability == 0
1104
+ or evaluate_mode
1049
1105
  or np.any(
1050
1106
  image_pred_uint8_cleaned_curr >= math.floor(min_probability * 255)
1051
1107
  )
1052
- or evaluate_mode is True
1053
1108
  ):
1054
1109
  # Find the class name in the classes list
1055
1110
  class_name = None
@@ -1,7 +1,6 @@
1
1
  """Module with high-level operations to segment images."""
2
2
 
3
3
  import csv
4
- import datetime
5
4
  import json
6
5
  import logging
7
6
  import multiprocessing
@@ -11,7 +10,9 @@ import time
11
10
  import traceback
12
11
  from collections.abc import Callable
13
12
  from concurrent import futures
13
+ from datetime import datetime
14
14
  from pathlib import Path
15
+ from time import perf_counter
15
16
  from typing import Any
16
17
 
17
18
  import geofileops as gfo
@@ -445,7 +446,7 @@ def _predict_layer(
445
446
  nb_to_predict = nb_images
446
447
  nb_done = 0
447
448
  nb_errors = 0
448
- read_sleep_logged = False
449
+ read_sleep_last_logged = None
449
450
  progress = None
450
451
  read_queue: dict[futures.Future, Path] = {}
451
452
  postp_queue: dict[futures.Future, Path] = {}
@@ -467,7 +468,7 @@ def _predict_layer(
467
468
  ):
468
469
  # Start looping.
469
470
  # If ready to stop, the code below will break
470
- perf_time_start = datetime.datetime.now()
471
+ perf_time_start = datetime.now()
471
472
  while True:
472
473
  # If we are ready, stop!
473
474
  if (
@@ -593,9 +594,13 @@ def _predict_layer(
593
594
  break
594
595
 
595
596
  # Wait a bit for images to be read, then try finding images again
596
- if not read_sleep_logged and not last_image_reached:
597
+ cur_counter = perf_counter()
598
+ if (
599
+ not read_sleep_last_logged
600
+ or (read_sleep_last_logged - cur_counter) > 5
601
+ ):
597
602
  logger.info("Wait for images to be read")
598
- read_sleep_logged = True
603
+ read_sleep_last_logged = cur_counter
599
604
  time.sleep(0.01)
600
605
 
601
606
  # If sufficient images in predict queue -> predict
@@ -603,22 +608,21 @@ def _predict_layer(
603
608
  if len(predict_queue) == batch_size or (
604
609
  last_image_reached and len(predict_queue) > 0
605
610
  ):
606
- read_sleep_logged = False
607
- perf_time_now = datetime.datetime.now()
611
+ perf_time_now = datetime.now()
608
612
  perfinfo += f"waiting for read took {perf_time_now - perf_time_start}"
609
613
  perf_time_start = perf_time_now
610
614
 
611
615
  # Predict!
612
616
  # --------
613
617
  logger.debug(f"Start prediction for {len(predict_queue)} images")
614
- perf_time_start = datetime.datetime.now()
618
+ perf_time_start = datetime.now()
615
619
  curr_batch_image_list = [
616
620
  batch_image_info["image_data"] for batch_image_info in predict_queue
617
621
  ]
618
622
  batch_image_arr = np.stack(curr_batch_image_list)
619
623
  batch_pred_arr = model.predict_on_batch(batch_image_arr)
620
624
 
621
- perf_time_now = datetime.datetime.now()
625
+ perf_time_now = datetime.now()
622
626
  perfinfo += f", predict took {perf_time_now - perf_time_start}"
623
627
  perf_time_start = perf_time_now
624
628
 
@@ -679,7 +683,7 @@ def _predict_layer(
679
683
  predict_queue = []
680
684
 
681
685
  # Collect performance debugging info
682
- time_taken = datetime.datetime.now() - perf_time_start
686
+ time_taken = datetime.now() - perf_time_start
683
687
  perfinfo += f", scheduling postprocessings took {time_taken}"
684
688
  perf_time_start = perf_time_now
685
689
 
@@ -741,7 +745,7 @@ def _predict_layer(
741
745
  if postp_sleep_logged:
742
746
  logger.info("Waited enough for postprocessing to catch up...")
743
747
 
744
- perf_time_now = datetime.datetime.now()
748
+ perf_time_now = datetime.now()
745
749
  perfinfo += (
746
750
  f", after postproces: took {perf_time_now - perf_time_start}"
747
751
  )
@@ -773,7 +777,7 @@ def _predict_layer(
773
777
  # Wait till number below thresshold to avoid huge waiting list (and
774
778
  # memory issues)
775
779
  if len(write_queue) > nb_parallel_postprocess * 2: # pragma: no cover
776
- if not write_sleep_logged:
780
+ if write_sleep_logged:
777
781
  logger.info("Writing takes longer than prediction, so wait")
778
782
  write_sleep_logged = True
779
783
  time.sleep(0.01)
@@ -5,7 +5,9 @@ import math
5
5
  import pprint
6
6
  import shutil
7
7
  import warnings
8
+ from concurrent import futures
8
9
  from pathlib import Path
10
+ from typing import Any
9
11
 
10
12
  import geofileops as gfo
11
13
  import geopandas as gpd
@@ -18,7 +20,7 @@ import shapely
18
20
  import shapely.geometry as sh_geom
19
21
  from PIL import Image
20
22
 
21
- from orthoseg.util import image_util, vector_util
23
+ from orthoseg.util import _processing_util, image_util, vector_util
22
24
  from orthoseg.util.progress_util import ProgressLogger
23
25
 
24
26
  # Get a logger...
@@ -144,6 +146,7 @@ def prepare_traindatasets(
144
146
  ssl_verify: bool | str = True,
145
147
  only_validate: bool = False,
146
148
  force: bool = False,
149
+ nb_concurrent_calls: int = 4,
147
150
  ) -> tuple[Path, int]:
148
151
  """This function prepares training data for the vector labels provided.
149
152
 
@@ -181,13 +184,27 @@ def prepare_traindatasets(
181
184
  only_validate (bool, optional): True to only validate the input label data.
182
185
  force (bool, opitional): True to force recreation of output files.
183
186
  Defaults to False.
187
+ nb_concurrent_calls (int, optional): Number of concurrent worker threads used
188
+ to prepare images/masks per traindata type. A conservative upper bound is
189
+ enforced to avoid overwhelming remote image services. Defaults to 4.
184
190
  """
191
+ if nb_concurrent_calls < 1:
192
+ raise ValueError("nb_concurrent_calls should be >= 1")
193
+
194
+ # Keep concurrency bounded to reduce the risk of overloading remote WMS services.
195
+ nb_concurrent_calls_effective = min(nb_concurrent_calls, 8)
196
+ if nb_concurrent_calls_effective != nb_concurrent_calls:
197
+ logger.info(
198
+ "nb_concurrent_calls capped to 8 for service safety: "
199
+ f"{nb_concurrent_calls} -> {nb_concurrent_calls_effective}"
200
+ )
201
+
185
202
  # Check if the first class is named "background"
186
203
  if len(classes) == 0:
187
- raise Exception("No classes specified")
204
+ raise ValueError("No classes specified")
188
205
  elif next(iter(classes)).lower() != "background":
189
206
  classes_str = pprint.pformat(classes, sort_dicts=False, width=50)
190
- raise Exception(
207
+ raise ValueError(
191
208
  f"By convention, the first class must be called background!\n{classes_str}"
192
209
  )
193
210
 
@@ -295,6 +312,7 @@ def prepare_traindatasets(
295
312
  logger.info(f"Get images for {nb_todo} labels")
296
313
 
297
314
  for traindata_type in traindata_types:
315
+ previous_imagedata_image_dir = None
298
316
  if dataversion_mostrecent is not None:
299
317
  previous_dataversion_dir = training_dir / f"{dataversion_mostrecent:02d}"
300
318
  previous_imagedata_image_dir = (
@@ -312,100 +330,67 @@ def prepare_traindatasets(
312
330
  if output_dir and not output_dir.exists():
313
331
  output_dir.mkdir(parents=True, exist_ok=True)
314
332
 
315
- for labellocations_gdf, labels_to_burn_gdf in labeldata:
316
- # Get the label locations for this traindata type
317
- labellocations_curr_gdf = labellocations_gdf[
318
- labellocations_gdf["traindata_type"] == traindata_type
319
- ]
320
-
321
- # Loop trough all locations labels to get an image for each of them
322
- for label_tuple in labellocations_curr_gdf.itertuples():
323
- img_bbox = label_tuple.geometry
324
- image_layer = label_tuple.image_layer
325
-
326
- # Prepare file name for the image
327
- assert labellocations_gdf.crs is not None
328
- output_filename = image_util.create_filename(
329
- crs=labellocations_gdf.crs,
330
- bbox=img_bbox.bounds,
331
- size=(image_pixel_width, image_pixel_height),
332
- image_format=image_util.FORMAT_PNG,
333
- layername="_".join(
334
- image_layers[image_layer]["layersources"][0].layernames
335
- ),
336
- )
337
-
338
- # If the image exists already in the previous version, reuse it.
339
- if (
340
- dataversion_mostrecent is not None
341
- and (previous_imagedata_image_dir / output_filename).exists()
342
- ):
343
- image_filepath: Path | None = Path(
344
- shutil.copy(
345
- src=previous_imagedata_image_dir / output_filename,
346
- dst=output_imagedata_image_dir / output_filename,
347
- )
348
- )
349
- pgw_filename = output_filename.replace(".png", ".pgw")
350
- if (previous_imagedata_image_dir / pgw_filename).exists():
351
- shutil.copy(
352
- src=previous_imagedata_image_dir / pgw_filename,
353
- dst=output_imagedata_image_dir / pgw_filename,
354
- )
355
- else:
356
- # Get the image from the WMS service.
357
- image_filepath = image_util.load_image_to_file(
358
- layersources=image_layers[image_layer]["layersources"],
359
- output_dir=output_imagedata_image_dir,
360
- crs=labellocations_gdf.crs,
361
- bbox=img_bbox.bounds,
362
- size=(image_pixel_width, image_pixel_height),
333
+ future_queue: dict[futures.Future, tuple[str, int]] = {}
334
+ with _processing_util.PooledExecutorFactory(
335
+ worker_type="processes",
336
+ max_workers=nb_concurrent_calls_effective,
337
+ ) as pool:
338
+ for labeldata_idx, (labellocations_gdf, labels_to_burn_gdf) in enumerate(
339
+ labeldata
340
+ ):
341
+ # Get the label locations for this traindata type
342
+ labellocations_curr_gdf = labellocations_gdf[
343
+ labellocations_gdf["traindata_type"] == traindata_type
344
+ ]
345
+
346
+ # Loop trough all locations labels to get an image for each of them
347
+ for label_tuple in labellocations_curr_gdf.itertuples():
348
+ assert labellocations_gdf.crs is not None
349
+ future = pool.submit(
350
+ _prepare_image_and_mask,
351
+ img_bbox=label_tuple.geometry,
352
+ image_layer=label_tuple.image_layer,
353
+ image_crs=labellocations_gdf.crs,
354
+ labels_to_burn_gdf=labels_to_burn_gdf,
355
+ image_layers=image_layers,
356
+ output_imagedata_image_dir=output_imagedata_image_dir,
357
+ output_imagedata_mask_dir=output_imagedata_mask_dir,
358
+ previous_imagedata_image_dir=previous_imagedata_image_dir,
359
+ image_pixel_width=image_pixel_width,
360
+ image_pixel_height=image_pixel_height,
363
361
  ssl_verify=ssl_verify,
364
- image_format=image_util.FORMAT_PNG,
365
- # image_format_save=image_util.FORMAT_TIFF,
366
- image_pixels_ignore_border=image_layers[image_layer][
367
- "image_pixels_ignore_border"
368
- ],
369
- transparent=False,
370
- layername_in_filename=True,
371
- output_filename=output_filename,
362
+ force=force,
372
363
  )
364
+ future_queue[future] = (traindata_type, labeldata_idx)
373
365
 
374
- if image_filepath is None:
375
- logger.warning(f"Could not get image at {img_bbox.wkt}")
376
- continue
377
-
378
- # Create a mask corresponding with the image file
379
- # Mask should never be in a lossy format -> png!
380
- mask_filepath = Path(
381
- str(image_filepath)
382
- .replace(
383
- str(output_imagedata_image_dir),
384
- str(output_imagedata_mask_dir),
366
+ future_errors = []
367
+ for future in futures.as_completed(future_queue):
368
+ # ProgressLogger is not thread-safe, so only update it in the main
369
+ # thread when a future is completed.
370
+ progress.step()
371
+
372
+ try:
373
+ future.result()
374
+ except Exception as ex:
375
+ current_traindata_type, labeldata_idx = future_queue[future]
376
+ future_errors.append(
377
+ (
378
+ current_traindata_type,
379
+ labeldata_idx,
380
+ ex,
381
+ )
385
382
  )
386
- .replace(".jpg", ".png")
387
- )
388
383
 
389
- # Only keep the labels that are meant for this image layer
390
- if "image_layer" not in labels_to_burn_gdf.columns:
391
- print("odd")
392
- labels_for_layer_gdf = (
393
- labels_to_burn_gdf.loc[
394
- labels_to_burn_gdf["image_layer"] == image_layer
395
- ]
396
- ).copy()
397
- if len(labels_for_layer_gdf) == 0:
398
- logger.info(f"No polygons to burn for {image_layer=}!")
399
- _create_mask(
400
- input_image_filepath=image_filepath,
401
- output_mask_filepath=mask_filepath,
402
- labels_to_burn_gdf=labels_for_layer_gdf,
403
- force=force,
384
+ if len(future_errors) > 0:
385
+ error_messages = [
386
+ (f"[{traindata_type_error}] labeldata_idx={labeldata_idx}: {error}")
387
+ for traindata_type_error, labeldata_idx, error in future_errors
388
+ ]
389
+ raise RuntimeError(
390
+ "Errors while preparing training images/masks:\n"
391
+ + "\n".join(error_messages)
404
392
  )
405
393
 
406
- # Log the progress estimated time remaining.
407
- progress.step()
408
-
409
394
  # If everything went fine, rename output_tmp_dir to the final output_dir
410
395
  output_tmp_dir.rename(training_dataversion_dir)
411
396
 
@@ -770,6 +755,95 @@ def create_tmp_dir(
770
755
  return tmp_dir
771
756
 
772
757
 
758
+ def _prepare_image_and_mask(
759
+ img_bbox: sh_geom.base.BaseGeometry,
760
+ image_layer: str,
761
+ image_crs: Any,
762
+ labels_to_burn_gdf: gpd.GeoDataFrame,
763
+ image_layers: dict,
764
+ output_imagedata_image_dir: Path,
765
+ output_imagedata_mask_dir: Path,
766
+ previous_imagedata_image_dir: Path | None,
767
+ image_pixel_width: int,
768
+ image_pixel_height: int,
769
+ ssl_verify: bool | str,
770
+ force: bool,
771
+ ) -> None:
772
+ """Prepare one training image and its mask."""
773
+ # Prepare file name for the image
774
+ output_filename = image_util.create_filename(
775
+ crs=image_crs,
776
+ bbox=img_bbox.bounds,
777
+ size=(image_pixel_width, image_pixel_height),
778
+ image_format=image_util.FORMAT_PNG,
779
+ layername="_".join(image_layers[image_layer]["layersources"][0].layernames),
780
+ )
781
+
782
+ # If the image exists already in the previous version, reuse it.
783
+ if (
784
+ previous_imagedata_image_dir is not None
785
+ and (previous_imagedata_image_dir / output_filename).exists()
786
+ ):
787
+ image_filepath: Path | None = Path(
788
+ shutil.copy(
789
+ src=previous_imagedata_image_dir / output_filename,
790
+ dst=output_imagedata_image_dir / output_filename,
791
+ )
792
+ )
793
+ pgw_filename = output_filename.replace(".png", ".pgw")
794
+ if (previous_imagedata_image_dir / pgw_filename).exists():
795
+ shutil.copy(
796
+ src=previous_imagedata_image_dir / pgw_filename,
797
+ dst=output_imagedata_image_dir / pgw_filename,
798
+ )
799
+ else:
800
+ # Get the image from the WMS service.
801
+ image_filepath = image_util.load_image_to_file(
802
+ layersources=image_layers[image_layer]["layersources"],
803
+ output_dir=output_imagedata_image_dir,
804
+ crs=image_crs,
805
+ bbox=img_bbox.bounds,
806
+ size=(image_pixel_width, image_pixel_height),
807
+ ssl_verify=ssl_verify,
808
+ image_format=image_util.FORMAT_PNG,
809
+ # image_format_save=image_util.FORMAT_TIFF,
810
+ image_pixels_ignore_border=image_layers[image_layer][
811
+ "image_pixels_ignore_border"
812
+ ],
813
+ transparent=False,
814
+ layername_in_filename=True,
815
+ output_filename=output_filename,
816
+ )
817
+
818
+ if image_filepath is None:
819
+ logger.warning(f"Could not get image at {img_bbox.wkt}")
820
+ return
821
+
822
+ # Create a mask corresponding with the image file
823
+ # Mask should never be in a lossy format -> png!
824
+ mask_filepath = Path(
825
+ str(image_filepath)
826
+ .replace(
827
+ str(output_imagedata_image_dir),
828
+ str(output_imagedata_mask_dir),
829
+ )
830
+ .replace(".jpg", ".png")
831
+ )
832
+
833
+ # Only keep the labels that are meant for this image layer
834
+ labels_for_layer_gdf = (
835
+ labels_to_burn_gdf.loc[labels_to_burn_gdf["image_layer"] == image_layer]
836
+ ).copy()
837
+ if len(labels_for_layer_gdf) == 0:
838
+ logger.info(f"No polygons to burn for {image_layer=}!")
839
+ _create_mask(
840
+ input_image_filepath=image_filepath,
841
+ output_mask_filepath=mask_filepath,
842
+ labels_to_burn_gdf=labels_for_layer_gdf,
843
+ force=force,
844
+ )
845
+
846
+
773
847
  def _create_mask(
774
848
  input_image_filepath: Path,
775
849
  output_mask_filepath: Path,