grdl-runtime 0.1.2__tar.gz → 0.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (122) hide show
  1. {grdl_runtime-0.1.2/grdl_runtime.egg-info → grdl_runtime-0.2.0}/PKG-INFO +23 -2
  2. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/README.md +21 -0
  3. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/grdl_rt/execution/band_adaptation.py +24 -6
  4. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/grdl_rt/execution/builder.py +225 -12
  5. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/grdl_rt/execution/config.py +6 -1
  6. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/grdl_rt/execution/memory_sampler.py +13 -2
  7. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/grdl_rt/execution/metrics.py +19 -9
  8. grdl_runtime-0.2.0/grdl_rt/execution/tiled.py +773 -0
  9. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0/grdl_runtime.egg-info}/PKG-INFO +23 -2
  10. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/grdl_runtime.egg-info/SOURCES.txt +3 -0
  11. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/grdl_runtime.egg-info/requires.txt +1 -1
  12. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/pyproject.toml +2 -2
  13. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/tests/test_band_adaptation.py +18 -0
  14. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/tests/test_config.py +7 -0
  15. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/tests/test_metrics.py +61 -0
  16. grdl_runtime-0.2.0/tests/test_nisar_refined_lee_pauli.py +210 -0
  17. grdl_runtime-0.2.0/tests/test_tiled_execution.py +506 -0
  18. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/LICENSE +0 -0
  19. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/MANIFEST.in +0 -0
  20. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/grdl_rt/__init__.py +0 -0
  21. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/grdl_rt/__main__.py +0 -0
  22. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/grdl_rt/api.py +0 -0
  23. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/grdl_rt/catalog/__init__.py +0 -0
  24. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/grdl_rt/catalog/base.py +0 -0
  25. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/grdl_rt/catalog/database.py +0 -0
  26. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/grdl_rt/catalog/federated.py +0 -0
  27. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/grdl_rt/catalog/models.py +0 -0
  28. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/grdl_rt/catalog/pool.py +0 -0
  29. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/grdl_rt/catalog/resolver.py +0 -0
  30. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/grdl_rt/catalog/schema.py +0 -0
  31. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/grdl_rt/catalog/updater.py +0 -0
  32. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/grdl_rt/catalog/yaml_catalog.py +0 -0
  33. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/grdl_rt/cli.py +0 -0
  34. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/grdl_rt/execution/__init__.py +0 -0
  35. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/grdl_rt/execution/checkpoint.py +0 -0
  36. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/grdl_rt/execution/chip.py +0 -0
  37. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/grdl_rt/execution/context.py +0 -0
  38. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/grdl_rt/execution/dag.py +0 -0
  39. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/grdl_rt/execution/dag_executor.py +0 -0
  40. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/grdl_rt/execution/discovery.py +0 -0
  41. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/grdl_rt/execution/dispatch.py +0 -0
  42. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/grdl_rt/execution/dsl.py +0 -0
  43. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/grdl_rt/execution/errors.py +0 -0
  44. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/grdl_rt/execution/executor.py +0 -0
  45. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/grdl_rt/execution/gpu.py +0 -0
  46. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/grdl_rt/execution/graph.py +0 -0
  47. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/grdl_rt/execution/hardware.py +0 -0
  48. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/grdl_rt/execution/history.py +0 -0
  49. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/grdl_rt/execution/instrumentation/__init__.py +0 -0
  50. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/grdl_rt/execution/instrumentation/prometheus.py +0 -0
  51. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/grdl_rt/execution/instrumentation/tracing.py +0 -0
  52. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/grdl_rt/execution/lineage.py +0 -0
  53. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/grdl_rt/execution/operators.py +0 -0
  54. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/grdl_rt/execution/plan.py +0 -0
  55. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/grdl_rt/execution/project.py +0 -0
  56. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/grdl_rt/execution/quota.py +0 -0
  57. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/grdl_rt/execution/resilience.py +0 -0
  58. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/grdl_rt/execution/resolver.py +0 -0
  59. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/grdl_rt/execution/result.py +0 -0
  60. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/grdl_rt/execution/tags.py +0 -0
  61. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/grdl_rt/execution/validation.py +0 -0
  62. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/grdl_rt/execution/workflow.py +0 -0
  63. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/grdl_rt/ui/__init__.py +0 -0
  64. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/grdl_rt/ui/__main__.py +0 -0
  65. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/grdl_rt/ui/_accuracy.py +0 -0
  66. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/grdl_rt/ui/_app.py +0 -0
  67. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/grdl_rt/ui/_config_dialog.py +0 -0
  68. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/grdl_rt/ui/_importer.py +0 -0
  69. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/grdl_rt/ui/_metrics_panel.py +0 -0
  70. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/grdl_rt/ui/_runner.py +0 -0
  71. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/grdl_rt/ui/_widgets.py +0 -0
  72. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/grdl_runtime.egg-info/dependency_links.txt +0 -0
  73. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/grdl_runtime.egg-info/entry_points.txt +0 -0
  74. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/grdl_runtime.egg-info/top_level.txt +0 -0
  75. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/setup.cfg +0 -0
  76. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/tests/test_api.py +0 -0
  77. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/tests/test_builder.py +0 -0
  78. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/tests/test_catalog_alternatives.py +0 -0
  79. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/tests/test_catalog_base.py +0 -0
  80. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/tests/test_catalog_database.py +0 -0
  81. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/tests/test_catalog_federated.py +0 -0
  82. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/tests/test_catalog_pool.py +0 -0
  83. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/tests/test_catalog_resolver.py +0 -0
  84. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/tests/test_catalog_updater.py +0 -0
  85. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/tests/test_catalog_yaml.py +0 -0
  86. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/tests/test_checkpoint.py +0 -0
  87. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/tests/test_chip.py +0 -0
  88. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/tests/test_cli.py +0 -0
  89. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/tests/test_context.py +0 -0
  90. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/tests/test_dag.py +0 -0
  91. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/tests/test_dag_executor.py +0 -0
  92. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/tests/test_discovery.py +0 -0
  93. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/tests/test_dispatch.py +0 -0
  94. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/tests/test_dsl.py +0 -0
  95. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/tests/test_executor.py +0 -0
  96. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/tests/test_feature_set_dispatch.py +0 -0
  97. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/tests/test_global_pass.py +0 -0
  98. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/tests/test_gpu.py +0 -0
  99. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/tests/test_grdl_integration.py +0 -0
  100. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/tests/test_hardware_context.py +0 -0
  101. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/tests/test_headless.py +0 -0
  102. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/tests/test_history.py +0 -0
  103. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/tests/test_lineage.py +0 -0
  104. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/tests/test_operators.py +0 -0
  105. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/tests/test_otel_tracing.py +0 -0
  106. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/tests/test_package.py +0 -0
  107. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/tests/test_param_schema.py +0 -0
  108. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/tests/test_project.py +0 -0
  109. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/tests/test_prometheus_metrics.py +0 -0
  110. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/tests/test_quota.py +0 -0
  111. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/tests/test_resilience.py +0 -0
  112. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/tests/test_resolver.py +0 -0
  113. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/tests/test_runtime_fallback.py +0 -0
  114. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/tests/test_tags.py +0 -0
  115. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/tests/test_tap_out.py +0 -0
  116. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/tests/test_type_validation.py +0 -0
  117. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/tests/test_ui_accuracy.py +0 -0
  118. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/tests/test_ui_importer.py +0 -0
  119. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/tests/test_ui_runner.py +0 -0
  120. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/tests/test_validation.py +0 -0
  121. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/tests/test_workflow.py +0 -0
  122. {grdl_runtime-0.1.2 → grdl_runtime-0.2.0}/tests/test_workflow_graph.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: grdl-runtime
3
- Version: 0.1.2
3
+ Version: 0.2.0
4
4
  Summary: GEOINT Rapid Development Library Runtime — headless execution engine for GRDL workflow orchestration
5
5
  Author: Claude Code (Anthropic)
6
6
  Maintainer: Steven Siebert
@@ -19,7 +19,7 @@ Classifier: Topic :: Scientific/Engineering :: GIS
19
19
  Requires-Python: >=3.11
20
20
  Description-Content-Type: text/markdown
21
21
  License-File: LICENSE
22
- Requires-Dist: grdl>=0.5.1
22
+ Requires-Dist: grdl>=0.8.0
23
23
  Requires-Dist: pyyaml>=6.0
24
24
  Requires-Dist: requests>=2.25
25
25
  Requires-Dist: packaging>=21.0
@@ -118,6 +118,27 @@ wf = (
118
118
  result = wf.execute(my_array)
119
119
  ```
120
120
 
121
+ ### Memory timeline profiling
122
+
123
+ Memory time-series collection is **disabled by default** for all execution
124
+ paths, including the NISAR example. No sampler thread or tracemalloc session
125
+ is started, and JSON metrics omit `memory_timeline` (including its large
126
+ `timestamps` and `values` arrays). Sampled memory peaks are reported as zero
127
+ when profiling is disabled; run start/end times and elapsed durations are
128
+ still recorded.
129
+
130
+ To opt in, set `GRDL_RT_MEMORY_TIMELINE_ENABLED=true` before launching Python,
131
+ or add this to `~/.grdl/runtime.toml`:
132
+
133
+ ```toml
134
+ [memory]
135
+ timeline_enabled = true
136
+ ```
137
+
138
+ When enabled, the timeline is sampled every millisecond and included in JSON
139
+ metrics. Memory estimation and safety thresholds are independent of profiling
140
+ and remain active.
141
+
121
142
  ### Batch Execution
122
143
 
123
144
  ```python
@@ -64,6 +64,27 @@ wf = (
64
64
  result = wf.execute(my_array)
65
65
  ```
66
66
 
67
+ ### Memory timeline profiling
68
+
69
+ Memory time-series collection is **disabled by default** for all execution
70
+ paths, including the NISAR example. No sampler thread or tracemalloc session
71
+ is started, and JSON metrics omit `memory_timeline` (including its large
72
+ `timestamps` and `values` arrays). Sampled memory peaks are reported as zero
73
+ when profiling is disabled; run start/end times and elapsed durations are
74
+ still recorded.
75
+
76
+ To opt in, set `GRDL_RT_MEMORY_TIMELINE_ENABLED=true` before launching Python,
77
+ or add this to `~/.grdl/runtime.toml`:
78
+
79
+ ```toml
80
+ [memory]
81
+ timeline_enabled = true
82
+ ```
83
+
84
+ When enabled, the timeline is sampled every millisecond and included in JSON
85
+ metrics. Memory estimation and safety thresholds are independent of profiling
86
+ and remain active.
87
+
67
88
  ### Batch Execution
68
89
 
69
90
  ```python
@@ -54,13 +54,19 @@ logger = get_logger(__name__)
54
54
  # ------------------------------------------------------------------
55
55
 
56
56
 
57
- def detect_band_axis(source: np.ndarray) -> int | None:
57
+ def detect_band_axis(
58
+ source: np.ndarray,
59
+ axis_order: str | None = None,
60
+ ) -> int | None:
58
61
  """Auto-detect the band axis of an image array.
59
62
 
60
63
  Parameters
61
64
  ----------
62
65
  source : np.ndarray
63
66
  2-D or 3-D image array.
67
+ axis_order : str, optional
68
+ Metadata ``axis_order`` (``'CYX'`` or ``'YXC'``). When given for
69
+ a 3-D array it is used instead of the shape heuristic.
64
70
 
65
71
  Returns
66
72
  -------
@@ -69,34 +75,43 @@ def detect_band_axis(source: np.ndarray) -> int | None:
69
75
 
70
76
  Notes
71
77
  -----
72
- Heuristic for 3-D arrays:
78
+ Heuristic for 3-D arrays without ``axis_order``:
73
79
 
74
80
  - ``shape[0] <= shape[2]`` → CHW (bands-first, e.g. ``(3, 512, 512)``)
75
81
  - ``shape[2] < shape[0]`` → HWC (bands-last, e.g. ``(512, 512, 3)``)
82
+
83
+ The heuristic misreads small tiles (e.g. ``(4, 3, 3)``), so pass
84
+ ``axis_order`` whenever metadata is available.
76
85
  """
77
86
  if source.ndim == 2:
78
87
  return None
79
88
  if source.ndim != 3:
80
89
  return None
90
+ if axis_order == "CYX":
91
+ return 0
92
+ if axis_order == "YXC":
93
+ return 2
81
94
  if source.shape[0] <= source.shape[2]:
82
95
  return 0 # CHW
83
96
  return 2 # HWC
84
97
 
85
98
 
86
- def get_band_count(source: np.ndarray) -> int:
99
+ def get_band_count(source: np.ndarray, axis_order: str | None = None) -> int:
87
100
  """Return the number of spectral bands in an image array.
88
101
 
89
102
  Parameters
90
103
  ----------
91
104
  source : np.ndarray
92
105
  2-D or 3-D image array.
106
+ axis_order : str, optional
107
+ Metadata ``axis_order``; see :func:`detect_band_axis`.
93
108
 
94
109
  Returns
95
110
  -------
96
111
  int
97
112
  Number of bands (1 for 2-D arrays).
98
113
  """
99
- axis = detect_band_axis(source)
114
+ axis = detect_band_axis(source, axis_order)
100
115
  if axis is None:
101
116
  return 1
102
117
  return source.shape[axis]
@@ -112,6 +127,7 @@ def adapt_bands(
112
127
  required_bands: int,
113
128
  expansion: BandExpansion = BandExpansion.REPEAT,
114
129
  reduction: BandReduction = BandReduction.FIRST_N,
130
+ axis_order: str | None = None,
115
131
  ) -> np.ndarray:
116
132
  """Adapt *source* to have exactly *required_bands* bands.
117
133
 
@@ -127,6 +143,8 @@ def adapt_bands(
127
143
  Strategy when the source has fewer bands than required.
128
144
  reduction : BandReduction
129
145
  Strategy when the source has more bands than required.
146
+ axis_order : str, optional
147
+ Metadata ``axis_order``; see :func:`detect_band_axis`.
130
148
 
131
149
  Returns
132
150
  -------
@@ -138,12 +156,12 @@ def adapt_bands(
138
156
  ValueError
139
157
  If LUMINANCE reduction is requested with a non-3-band input.
140
158
  """
141
- current_bands = get_band_count(source)
159
+ current_bands = get_band_count(source, axis_order)
142
160
 
143
161
  if current_bands == required_bands:
144
162
  return source
145
163
 
146
- axis = detect_band_axis(source)
164
+ axis = detect_band_axis(source, axis_order)
147
165
 
148
166
  logger.warning(
149
167
  "band_adaptation",
@@ -68,6 +68,7 @@ from grdl_rt.execution.resilience import (
68
68
  )
69
69
  from grdl_rt.execution.result import WorkflowResult
70
70
  from grdl_rt.execution.tags import ExecutionPhase, ImageModality, OutputFormat, WorkflowTags
71
+ from grdl_rt.execution.tiled import ReaderSource, TiledExecutor, TilingConfig
71
72
 
72
73
  logger = get_logger(__name__)
73
74
 
@@ -382,6 +383,9 @@ class Workflow:
382
383
  self._steps: list[WorkflowStep | DeferredStep | TapOutStep] = []
383
384
  self._source: Callable[[], np.ndarray] | None = None
384
385
  self._reader_cls: type | None = None
386
+ self._reader_kwargs: dict[str, Any] = {}
387
+ self._tiling: TilingConfig | None = None
388
+ self._collection_checks: dict[str, Any] | None = None
385
389
  self._chip_strategy: str | None = None
386
390
  self._chip_kwargs: dict[str, Any] = {}
387
391
  self._current_phase: str | None = None
@@ -441,7 +445,7 @@ class Workflow:
441
445
  # Builder
442
446
  # ------------------------------------------------------------------
443
447
 
444
- def reader(self, reader_cls: type) -> "Workflow":
448
+ def reader(self, reader_cls: type, **reader_kwargs: Any) -> "Workflow":
445
449
  """Declare the reader type for this workflow.
446
450
 
447
451
  When :meth:`execute` is called with a filepath, the framework
@@ -451,6 +455,11 @@ class Workflow:
451
455
  ----------
452
456
  reader_cls : type
453
457
  An ``ImageReader`` subclass (e.g., ``SICDReader``).
458
+ **reader_kwargs
459
+ Extra constructor arguments, e.g. ``polarizations='all'`` for
460
+ ``NISARReader``. For collection readers such as
461
+ ``SICDCollectionReader``, pass a list of paths to
462
+ :meth:`execute` as the source.
454
463
 
455
464
  Returns
456
465
  -------
@@ -458,6 +467,76 @@ class Workflow:
458
467
  Self, for fluent chaining.
459
468
  """
460
469
  self._reader_cls = reader_cls
470
+ self._reader_kwargs = reader_kwargs
471
+ return self
472
+
473
+ def tiling(
474
+ self,
475
+ tile_size: int | tuple[int, int] = 1024,
476
+ *,
477
+ stride: int | tuple[int, int] | None = None,
478
+ workers: int | None = None,
479
+ backend: str = "process",
480
+ overlap_method: str = "weighted",
481
+ scratch_dir: str | Path | None = None,
482
+ ) -> "Workflow":
483
+ """Run tile-safe steps in parallel on halo tiles.
484
+
485
+ Consecutive steps whose processors declare a ``tile_halo`` run
486
+ tile by tile; any other step runs on the assembled full array, and
487
+ later tile-safe steps tile again. Only file mode with a linear
488
+ workflow is supported, and not together with ``chip('center')``.
489
+ See :class:`~grdl_rt.execution.tiled.TilingConfig` for options.
490
+
491
+ Returns
492
+ -------
493
+ Workflow
494
+ Self, for fluent chaining.
495
+ """
496
+ self._tiling = TilingConfig(
497
+ tile_size=tile_size,
498
+ stride=stride,
499
+ workers=workers,
500
+ backend=backend,
501
+ overlap_method=overlap_method,
502
+ scratch_dir=scratch_dir,
503
+ )
504
+ return self
505
+
506
+ def validate_collection(
507
+ self,
508
+ *,
509
+ strict: bool = True,
510
+ reciprocity: str | None = None,
511
+ **thresholds: Any,
512
+ ) -> "Workflow":
513
+ """Check that a multi-channel input is a consistent collection.
514
+
515
+ Runs ``grdl.IO.sar.validate_collection`` after the reader opens and
516
+ before any pixels are processed.
517
+
518
+ Parameters
519
+ ----------
520
+ strict : bool
521
+ Raise on validation errors; otherwise only log them.
522
+ reciprocity : {None, 'warn', 'strict'}
523
+ Also test HV/VH reciprocity on sampled chips of a quad-pol
524
+ input; ``'strict'`` raises on failure, ``'warn'`` warns.
525
+ **thresholds
526
+ Forwarded to ``validate_collection``.
527
+
528
+ Returns
529
+ -------
530
+ Workflow
531
+ Self, for fluent chaining.
532
+ """
533
+ if reciprocity not in (None, "warn", "strict"):
534
+ raise ValueError("reciprocity must be None, 'warn' or 'strict'")
535
+ self._collection_checks = {
536
+ "strict": strict,
537
+ "reciprocity": reciprocity,
538
+ "thresholds": thresholds,
539
+ }
461
540
  return self
462
541
 
463
542
  def chip(self, strategy: str = "center", **kwargs: Any) -> "Workflow":
@@ -868,7 +947,7 @@ class Workflow:
868
947
 
869
948
  def execute(
870
949
  self,
871
- source: np.ndarray | str | Path | None = None,
950
+ source: np.ndarray | str | Path | list[str | Path] | tuple[str | Path, ...] | None = None,
872
951
  *,
873
952
  prefer_gpu: bool = False,
874
953
  progress_callback: Callable[[float], None] | None = None,
@@ -900,8 +979,9 @@ class Workflow:
900
979
 
901
980
  Parameters
902
981
  ----------
903
- source : np.ndarray, str, Path, or None
904
- Input data.
982
+ source : np.ndarray, str, Path, list of paths, or None
983
+ Input data. A list of paths is passed to the configured
984
+ reader as one argument (multi-file collections).
905
985
  prefer_gpu : bool
906
986
  If ``True``, attempt GPU acceleration for compatible steps.
907
987
  progress_callback : callable, optional
@@ -929,13 +1009,16 @@ class Workflow:
929
1009
  RuntimeError
930
1010
  If any step fails.
931
1011
  """
932
- source_path: Path | None = None
933
-
934
1012
  # File mode → framework orchestration
935
- if isinstance(source, (str, Path)):
936
- source_path = Path(source)
1013
+ if isinstance(source, (str, Path, list, tuple)):
1014
+ if isinstance(source, (list, tuple)):
1015
+ if not source:
1016
+ raise ValueError("source path list is empty")
1017
+ paths: Path | list[Path] = [Path(p) for p in source]
1018
+ else:
1019
+ paths = Path(source)
937
1020
  return self._execute_from_file(
938
- source_path,
1021
+ paths,
939
1022
  prefer_gpu=prefer_gpu,
940
1023
  progress_callback=progress_callback,
941
1024
  metadata_override=metadata,
@@ -1069,7 +1152,7 @@ class Workflow:
1069
1152
 
1070
1153
  def _execute_from_file(
1071
1154
  self,
1072
- filepath: Path,
1155
+ filepath: Path | list[Path],
1073
1156
  *,
1074
1157
  prefer_gpu: bool,
1075
1158
  progress_callback: Callable[[float], None] | None,
@@ -1084,10 +1167,26 @@ class Workflow:
1084
1167
  f"Workflow '{self._name}' received a filepath but no reader "
1085
1168
  f"is configured. Call .reader(ReaderClass) first."
1086
1169
  )
1170
+ if self._tiling is not None:
1171
+ self._check_tiling_supported()
1087
1172
 
1088
- with self._reader_cls(filepath) as rdr:
1173
+ reader_cls, reader_kwargs = self._reader_cls, self._reader_kwargs
1174
+ with reader_cls(filepath, **reader_kwargs) as rdr:
1089
1175
  meta = metadata_override if metadata_override is not None else rdr.metadata
1090
1176
 
1177
+ if self._collection_checks is not None:
1178
+ self._run_collection_checks(rdr)
1179
+
1180
+ if self._tiling is not None:
1181
+ return self._execute_tiled(
1182
+ functools.partial(reader_cls, filepath, **reader_kwargs),
1183
+ meta,
1184
+ prefer_gpu=prefer_gpu,
1185
+ progress_callback=progress_callback,
1186
+ )
1187
+
1188
+ source_path = filepath[0] if isinstance(filepath, list) else filepath
1189
+
1091
1190
  # Read chip according to strategy
1092
1191
  chip = self._read_chip(rdr)
1093
1192
 
@@ -1109,7 +1208,7 @@ class Workflow:
1109
1208
  resolved,
1110
1209
  prefer_gpu=prefer_gpu,
1111
1210
  progress_callback=progress_callback,
1112
- source_path=filepath,
1211
+ source_path=source_path,
1113
1212
  tap_out_format=tap_out_format,
1114
1213
  tap_out_dir=tap_out_dir,
1115
1214
  )
@@ -1121,6 +1220,119 @@ class Workflow:
1121
1220
  metadata=meta,
1122
1221
  )
1123
1222
 
1223
+ def _check_tiling_supported(self) -> None:
1224
+ if self._is_dag:
1225
+ raise ValueError("tiling() supports linear workflows only, not DAGs")
1226
+ if self._chip_strategy == "center":
1227
+ raise ValueError("tiling() cannot be combined with chip('center')")
1228
+
1229
+ def _run_collection_checks(self, rdr: Any) -> None:
1230
+ """Validate the opened reader as a collection and test reciprocity."""
1231
+ try:
1232
+ from grdl.IO.sar import check_reciprocity_reader, validate_collection
1233
+ except ImportError as e:
1234
+ raise ImportError(
1235
+ "validate_collection() requires grdl with " "grdl.IO.sar.collection_validation"
1236
+ ) from e
1237
+
1238
+ checks = self._collection_checks or {}
1239
+ report = validate_collection(rdr, **checks.get("thresholds", {}))
1240
+ for issue in report.issues:
1241
+ logger.warning(
1242
+ "collection_issue",
1243
+ level=issue.level,
1244
+ code=issue.code,
1245
+ message=issue.message,
1246
+ )
1247
+ if checks.get("strict", True):
1248
+ report.raise_if_invalid()
1249
+
1250
+ mode = checks.get("reciprocity")
1251
+ if mode is not None and report.is_quad_pol:
1252
+ check_reciprocity_reader(rdr, strict=(mode == "strict"))
1253
+
1254
+ def _execute_tiled(
1255
+ self,
1256
+ reader_factory: Callable[[], Any],
1257
+ meta: Any,
1258
+ *,
1259
+ prefer_gpu: bool,
1260
+ progress_callback: Callable[[float], None] | None,
1261
+ ) -> WorkflowResult:
1262
+ """Run the workflow with :class:`TiledExecutor`."""
1263
+ assert self._tiling is not None
1264
+ ctx = ExecutionContext(
1265
+ workflow_id=f"{self._name}:{self._version}",
1266
+ workflow_name=self._name,
1267
+ )
1268
+ hardware = _capture_hardware()
1269
+ sampler = MemorySampler()
1270
+ sampler.start()
1271
+ started_at = datetime.now(UTC).isoformat()
1272
+ wall_t0 = time.perf_counter()
1273
+ cpu_t0 = time.process_time()
1274
+
1275
+ resolved = self._resolve_steps(meta)
1276
+ gpu = GpuBackend(prefer_gpu=prefer_gpu)
1277
+ n_steps = len(resolved)
1278
+
1279
+ def run_global(ws: Any, data: Any, step_meta: Any, index: int) -> tuple[Any, Any, bool]:
1280
+ if gpu.is_gpu_array(data):
1281
+ data = gpu.to_cpu(data)
1282
+ return self._execute_step_gpu_aware(
1283
+ ws,
1284
+ data,
1285
+ gpu,
1286
+ index,
1287
+ n_steps,
1288
+ metadata=step_meta,
1289
+ )
1290
+
1291
+ run = TiledExecutor(self._tiling).run(
1292
+ ReaderSource(reader_factory),
1293
+ (meta.rows, meta.cols),
1294
+ resolved,
1295
+ meta,
1296
+ run_global,
1297
+ self._execute_tap_out,
1298
+ progress_callback=progress_callback,
1299
+ )
1300
+
1301
+ step_metrics = [
1302
+ StepMetrics(
1303
+ step_index=i,
1304
+ processor_name=ws.name,
1305
+ wall_time_s=timing.wall_time_s,
1306
+ cpu_time_s=timing.cpu_time_s,
1307
+ peak_rss_bytes=0,
1308
+ gpu_used=gpu_used,
1309
+ concurrent=tiled,
1310
+ wall_start=timing.wall_start,
1311
+ wall_end=timing.wall_end,
1312
+ )
1313
+ for i, (ws, timing, tiled, gpu_used) in enumerate(run.step_times)
1314
+ ]
1315
+ timeline = sampler.stop()
1316
+ metrics = WorkflowMetrics(
1317
+ workflow_id=ctx.workflow_id,
1318
+ run_id=ctx.run_id,
1319
+ workflow_name=self._name,
1320
+ workflow_version=self._version,
1321
+ total_wall_time_s=time.perf_counter() - wall_t0,
1322
+ total_cpu_time_s=time.process_time() - cpu_t0 + run.worker_cpu_time_s,
1323
+ peak_rss_bytes=timeline.peak(),
1324
+ step_metrics=step_metrics,
1325
+ started_at=started_at,
1326
+ completed_at=datetime.now(UTC).isoformat(),
1327
+ hardware=hardware,
1328
+ memory_timeline=timeline,
1329
+ )
1330
+ return WorkflowResult(
1331
+ result=run.result,
1332
+ metrics=metrics,
1333
+ metadata_trace=run.metadata_trace,
1334
+ )
1335
+
1124
1336
  def _read_chip(self, reader: Any) -> np.ndarray:
1125
1337
  """Plan and read a chip from the reader.
1126
1338
 
@@ -1371,6 +1583,7 @@ class Workflow:
1371
1583
  ws.required_bands,
1372
1584
  expansion=exp,
1373
1585
  reduction=red,
1586
+ axis_order=getattr(current_meta, "axis_order", None),
1374
1587
  )
1375
1588
 
1376
1589
  # Capture step-level metrics
@@ -92,6 +92,10 @@ class RetryDefaults(BaseModel):
92
92
  class MemoryConfig(BaseModel):
93
93
  """Memory estimation and threshold configuration."""
94
94
 
95
+ timeline_enabled: bool = Field(
96
+ default=False,
97
+ description="Collect and serialize the sampled memory timeline (opt-in profiling)",
98
+ )
95
99
  warn_threshold: float = Field(
96
100
  default=0.80,
97
101
  ge=0,
@@ -240,6 +244,7 @@ _ENV_MAP: dict[str, tuple] = {
240
244
  "GRDL_RT_MEMORY_WARN_THRESHOLD": ("memory", "warn_threshold"),
241
245
  "GRDL_RT_MEMORY_ABORT_THRESHOLD": ("memory", "abort_threshold"),
242
246
  "GRDL_RT_MEMORY_ESTIMATION_MULTIPLIER": ("memory", "estimation_multiplier"),
247
+ "GRDL_RT_MEMORY_TIMELINE_ENABLED": ("memory", "timeline_enabled"),
243
248
  "GRDL_RT_GPU_PREFER_GPU": ("gpu", "prefer_gpu"),
244
249
  "GRDL_RT_GPU_MEMORY_FRACTION": ("gpu", "memory_fraction"),
245
250
  "GRDL_RT_CATALOG_PATH": (None, "catalog_path"),
@@ -266,7 +271,7 @@ _BOOL_FALSY = {"0", "false", "no", "off", ""}
266
271
  def _coerce_env_value(raw: str, section: str | None, key: str) -> Any:
267
272
  """Coerce a string env var value to the appropriate Python type."""
268
273
  # Boolean fields
269
- if key in ("prefer_gpu", "enabled"):
274
+ if key in ("prefer_gpu", "enabled", "timeline_enabled"):
270
275
  return raw.lower() in _BOOL_TRUTHY
271
276
 
272
277
  # Float fields
@@ -11,7 +11,7 @@ recorded values.
11
11
 
12
12
  Typical usage::
13
13
 
14
- sampler = MemorySampler(interval_s=0.001)
14
+ sampler = MemorySampler(interval_s=0.001, enabled=True)
15
15
  sampler.start()
16
16
 
17
17
  # … run pipeline …
@@ -50,6 +50,8 @@ import time
50
50
  import tracemalloc
51
51
  from typing import Any
52
52
 
53
+ from grdl_rt.execution.config import get_runtime_config
54
+
53
55
 
54
56
  class MemoryTimeline:
55
57
  """Immutable snapshot of a completed memory sampling session.
@@ -173,10 +175,15 @@ class MemorySampler:
173
175
  ----------
174
176
  interval_s : float
175
177
  Sampling interval in seconds. Default ``0.001`` (1 ms).
178
+ enabled : bool, optional
179
+ Override ``memory.timeline_enabled`` in runtime configuration.
180
+ Disabled by default: no tracing, thread, or timestamp buffers are
181
+ started, and :meth:`stop` returns an empty timeline with peak 0.
176
182
  """
177
183
 
178
- def __init__(self, interval_s: float = 0.001) -> None:
184
+ def __init__(self, interval_s: float = 0.1, *, enabled: bool | None = None) -> None:
179
185
  self.interval_s = interval_s
186
+ self.enabled = get_runtime_config().memory.timeline_enabled if enabled is None else enabled
180
187
  self._timestamps: array.array[float] | None = None
181
188
  self._mem_values: array.array[float] | None = None
182
189
  self._stop_event = threading.Event()
@@ -189,6 +196,8 @@ class MemorySampler:
189
196
 
190
197
  def start(self) -> None:
191
198
  """Start ``tracemalloc``, allocate buffers, and begin sampling."""
199
+ if not self.enabled:
200
+ return
192
201
  self._tracemalloc_was_running = tracemalloc.is_tracing()
193
202
  if not self._tracemalloc_was_running:
194
203
  tracemalloc.start()
@@ -225,6 +234,8 @@ class MemorySampler:
225
234
  MemoryTimeline
226
235
  Immutable timeline of corrected memory samples.
227
236
  """
237
+ if not self.enabled:
238
+ return MemoryTimeline(timestamps=[], values=[], t0=0.0)
228
239
  self._stop_event.set()
229
240
  if self._thread is not None:
230
241
  self._thread.join(timeout=2.0)
@@ -2,11 +2,12 @@
2
2
  Execution Metrics — Structured timing and resource usage data.
3
3
 
4
4
  Provides ``StepMetrics`` and ``WorkflowMetrics`` dataclasses that capture
5
- wall-clock time, CPU time, peak memory, GPU usage, and a continuous
6
- memory timeline for every workflow execution. These are returned as
5
+ wall-clock time, CPU time, peak memory, GPU usage, and an optional
6
+ memory timeline for workflow executions. These are returned as
7
7
  part of ``WorkflowResult`` from all execution paths.
8
8
 
9
- Memory is measured by a background ``MemorySampler`` thread that polls
9
+ When opted in via ``memory.timeline_enabled``, memory is measured by a
10
+ background ``MemorySampler`` thread that polls
10
11
  ``tracemalloc`` at 1 ms intervals. The resulting ``MemoryTimeline``
11
12
  on ``WorkflowMetrics`` can be overlaid with each step's
12
13
  ``wall_start`` / ``wall_end`` timestamps for visualization.
@@ -64,8 +65,13 @@ class StepMetrics:
64
65
  Fully-qualified or short name of the processor executed.
65
66
  wall_time_s : float
66
67
  Wall-clock duration in seconds (``time.perf_counter`` delta).
68
+ For tiled steps, elapsed span from the first tile invocation to
69
+ the last completion, not the sum of worker durations. Steps in
70
+ one tiled segment overlap, so their wall times are not additive.
67
71
  cpu_time_s : float
68
- CPU time in seconds (``time.process_time`` delta).
72
+ Measured CPU seconds, not estimated from wall time. Tiled steps
73
+ sum worker process CPU deltas (serial/process backend) or calling
74
+ thread CPU deltas (thread backend, excluding native helper threads).
69
75
  peak_rss_bytes : int
70
76
  Peak memory in bytes. For single-component benchmarks this
71
77
  is the timeline peak for the step. For pipeline steps this
@@ -166,7 +172,8 @@ class StepMetrics:
166
172
  class WorkflowMetrics:
167
173
  """Aggregate metrics for a complete workflow execution.
168
174
 
169
- Memory is measured via a ``MemorySampler`` background thread that
175
+ When ``memory.timeline_enabled`` is enabled, memory is measured via
176
+ a ``MemorySampler`` background thread that
170
177
  polls ``tracemalloc.get_traced_memory()`` at 1 ms intervals and
171
178
  subtracts its own storage overhead from every reading. The
172
179
  resulting ``memory_timeline`` is a continuous time-series of
@@ -186,10 +193,12 @@ class WorkflowMetrics:
186
193
  total_wall_time_s : float
187
194
  Total wall-clock duration in seconds.
188
195
  total_cpu_time_s : float
189
- Total CPU time in seconds.
196
+ Total CPU time in seconds. Tiled process execution includes measured
197
+ worker-job CPU in addition to parent process CPU; worker startup and
198
+ interprocess serialization outside tile jobs are not included.
190
199
  peak_rss_bytes : int
191
200
  Peak memory usage in bytes across the entire run, derived from
192
- the ``memory_timeline``.
201
+ the ``memory_timeline``. Zero when memory profiling is disabled.
193
202
  step_metrics : List[StepMetrics]
194
203
  Per-step metrics, one entry per processing step.
195
204
  started_at : str
@@ -215,7 +224,8 @@ class WorkflowMetrics:
215
224
  readings sampled at 1 ms intervals over the pipeline lifetime.
216
225
  Correlate with each step's ``wall_start`` / ``wall_end`` for
217
226
  per-step memory visualization. ``None`` for runs produced
218
- before this field was added.
227
+ before this field was added. Empty when profiling is disabled;
228
+ empty timelines are omitted from serialization.
219
229
  """
220
230
 
221
231
  workflow_id: str
@@ -259,7 +269,7 @@ class WorkflowMetrics:
259
269
  d["hardware"] = self.hardware
260
270
  if self.step_depends_on is not None:
261
271
  d["step_depends_on"] = self.step_depends_on
262
- if self.memory_timeline is not None:
272
+ if self.memory_timeline is not None and len(self.memory_timeline) > 0:
263
273
  d["memory_timeline"] = self.memory_timeline.to_dict()
264
274
  return d
265
275