flimanalyzer 0.6.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (110) hide show
  1. flim/__init__.py +2 -0
  2. flim/_version.py +5 -0
  3. flim/analysis/__init__.py +1 -0
  4. flim/analysis/ml/__init__.py +0 -0
  5. flim/analysis/ml/autoencoder.py +188 -0
  6. flim/analyzerapp.py +131 -0
  7. flim/batch.py +235 -0
  8. flim/configs/__init__.py +77 -0
  9. flim/configs/analysis.py +554 -0
  10. flim/configs/data.py +278 -0
  11. flim/configs/plot.py +543 -0
  12. flim/configs/workflow.py +204 -0
  13. flim/core/__init__.py +0 -0
  14. flim/core/calculator.py +0 -0
  15. flim/core/config.py +305 -0
  16. flim/core/execution.py +110 -0
  17. flim/core/fields.py +226 -0
  18. flim/core/filter.py +179 -0
  19. flim/core/graph.py +111 -0
  20. flim/core/headers.py +72 -0
  21. flim/core/importer.py +269 -0
  22. flim/core/parser.py +305 -0
  23. flim/core/paths.py +66 -0
  24. flim/core/prefect_server.py +152 -0
  25. flim/core/preprocessor.py +287 -0
  26. flim/core/registry.py +108 -0
  27. flim/core/runner.py +703 -0
  28. flim/core/schema.py +114 -0
  29. flim/core/settings.py +419 -0
  30. flim/core/tool.py +284 -0
  31. flim/core/tools.py +76 -0
  32. flim/core/types.py +113 -0
  33. flim/core/validation.py +272 -0
  34. flim/core/workflow.py +148 -0
  35. flim/gui/__init__.py +12 -0
  36. flim/gui/app.py +953 -0
  37. flim/gui/configdialog.py +495 -0
  38. flim/gui/datapanel.py +841 -0
  39. flim/gui/delimpanel.py +56 -0
  40. flim/gui/dialogs/__init__.py +473 -0
  41. flim/gui/dialogs/autoencoder.py +89 -0
  42. flim/gui/dialogs/categoryorder.py +130 -0
  43. flim/gui/dialogs/filter.py +86 -0
  44. flim/gui/dialogs/unpivot.py +72 -0
  45. flim/gui/dicttablepanel.py +178 -0
  46. flim/gui/editors.py +1578 -0
  47. flim/gui/events.py +162 -0
  48. flim/gui/icons.py +16 -0
  49. flim/gui/importdlg.py +391 -0
  50. flim/gui/listcontrol.py +472 -0
  51. flim/gui/macos.py +77 -0
  52. flim/gui/plotframe.py +73 -0
  53. flim/gui/progressdlg.py +290 -0
  54. flim/resources/aeaugment.png +0 -0
  55. flim/resources/aefeatureworkflow.png +0 -0
  56. flim/resources/aerun.png +0 -0
  57. flim/resources/aetrain.png +0 -0
  58. flim/resources/aetuneworkflow.png +0 -0
  59. flim/resources/barplot.png +0 -0
  60. flim/resources/boxplot.png +0 -0
  61. flim/resources/categorize.png +0 -0
  62. flim/resources/concatenate.png +0 -0
  63. flim/resources/fileopen.png +0 -0
  64. flim/resources/filter.png +0 -0
  65. flim/resources/flimanalyzer.ico +0 -0
  66. flim/resources/flimanalyzer.png +0 -0
  67. flim/resources/flirrplot.png +0 -0
  68. flim/resources/flirrworkflow.png +0 -0
  69. flim/resources/gmm.png +0 -0
  70. flim/resources/heatmap.png +0 -0
  71. flim/resources/histogram.png +0 -0
  72. flim/resources/import.png +0 -0
  73. flim/resources/kde.png +0 -0
  74. flim/resources/kmeans.png +0 -0
  75. flim/resources/ks.png +0 -0
  76. flim/resources/lineplot.png +0 -0
  77. flim/resources/merge.png +0 -0
  78. flim/resources/order.png +0 -0
  79. flim/resources/pairplot.png +0 -0
  80. flim/resources/pca.png +0 -0
  81. flim/resources/pivot.png +0 -0
  82. flim/resources/randomforest.png +0 -0
  83. flim/resources/relchange.png +0 -0
  84. flim/resources/scatter.png +0 -0
  85. flim/resources/seriesanalysis.png +0 -0
  86. flim/resources/sort.png +0 -0
  87. flim/resources/split.png +0 -0
  88. flim/resources/summary.png +0 -0
  89. flim/resources/swarmplot.png +0 -0
  90. flim/resources/unpivot.png +0 -0
  91. flim/resources/violinplot.png +0 -0
  92. flim/resources/workflow.png +0 -0
  93. flim/results.py +70 -0
  94. flim/tools/__init__.py +7 -0
  95. flim/tools/autoencoder.py +662 -0
  96. flim/tools/common.py +62 -0
  97. flim/tools/data.py +301 -0
  98. flim/tools/ml.py +311 -0
  99. flim/tools/plot.py +1439 -0
  100. flim/tools/statistics.py +348 -0
  101. flim/utils.py +43 -0
  102. flim/workflow/__init__.py +3 -0
  103. flim/workflow/aefeature.py +145 -0
  104. flim/workflow/aetune.py +163 -0
  105. flim/workflow/basicflow.py +99 -0
  106. flimanalyzer-0.6.0.dist-info/METADATA +262 -0
  107. flimanalyzer-0.6.0.dist-info/RECORD +110 -0
  108. flimanalyzer-0.6.0.dist-info/WHEEL +4 -0
  109. flimanalyzer-0.6.0.dist-info/entry_points.txt +5 -0
  110. flimanalyzer-0.6.0.dist-info/licenses/LICENSE +19 -0
@@ -0,0 +1,204 @@
1
+ """Configs of the workflows: their own fields plus the configs of their steps.
2
+
3
+ A workflow config owns the fields its steps share (grouping, features, ...); each step
4
+ config holds only the settings of that step. ``bound_fields`` names the fields the
5
+ workflow sets in each step: they are copied from the workflow config when it is
6
+ validated, hidden in dialogs and not saved.
7
+ """
8
+
9
+ import copy
10
+ from typing import Annotated, ClassVar, Optional
11
+
12
+ from pydantic import Field, model_validator
13
+
14
+ from flim.configs.analysis import (
15
+ AE_FEATURES,
16
+ AEAugmentConfig,
17
+ AERunConfig,
18
+ AETrainConfig,
19
+ AutoencoderResources,
20
+ )
21
+ from flim.core.config import CategoryColumn, FeatureColumn, TaskConfig, ValueOf
22
+
23
+
24
+ class WorkflowConfig(TaskConfig):
25
+ legacy_keys = {"strict": None}
26
+ legacy_dropped = frozenset({"executor"}) # an execution option
27
+
28
+ # step name -> fields the workflow sets in that step's config. Fields the workflow
29
+ # config also has are copied from it; the workflow's code sets the others.
30
+ bound_fields: ClassVar[dict[str, frozenset[str]]] = {}
31
+
32
+ # False: step settings naming columns that changed upstream are replaced by the
33
+ # closest match (and reported); True: they are errors to fix before running.
34
+ # None: the mode's default, False in the GUI and True in batch runs.
35
+ strict: Optional[bool] = Field(
36
+ default=None,
37
+ title="Strict",
38
+ description=(
39
+ "Columns that changed upstream are errors to fix; otherwise they are"
40
+ " replaced by the closest match. Automatic: strict in batch runs only."
41
+ ),
42
+ )
43
+
44
+ @model_validator(mode="after")
45
+ def _share_fields(self):
46
+ for step, names in self.bound_fields.items():
47
+ config = getattr(self, step)
48
+ for name in names & set(type(self).model_fields):
49
+ value = getattr(self, name)
50
+ if getattr(config, name) != value:
51
+ setattr(config, name, copy.deepcopy(value))
52
+ return self
53
+
54
+ def _after_dump(self, data):
55
+ for step, names in self.bound_fields.items():
56
+ if isinstance(data.get(step), dict):
57
+ for name in names:
58
+ data[step].pop(name, None)
59
+ return data
60
+
61
+
62
+ class BasicFLIRRConfig(WorkflowConfig):
63
+ grouping: list[CategoryColumn] = ["Treatment", "FOV", "Cell"]
64
+ # values the workflow used to take from the data, now settings with that default
65
+ reference_value: Annotated[Optional[str], ValueOf("grouping")] = Field(
66
+ default=None,
67
+ title="Reference value",
68
+ description=(
69
+ "Relative changes: the features are divided by their mean in the rows with"
70
+ " this value of the first grouping column. Empty: its first value (in the"
71
+ " order of its categories)."
72
+ ),
73
+ )
74
+ fad_a1: Optional[FeatureColumn] = Field(
75
+ default=None,
76
+ title="Scatter x (FAD a1 %)",
77
+ description="Empty: found by name (FAD, a1, %).",
78
+ )
79
+ nadph_a2: Optional[FeatureColumn] = Field(
80
+ default=None,
81
+ title="Scatter y (NAD(P)H a2 %)",
82
+ description="Empty: found by name (NAD, a2, %).",
83
+ )
84
+ features: list[FeatureColumn] = [
85
+ "FAD a1",
86
+ "FAD a1%",
87
+ "FAD a2",
88
+ "FAD a2%",
89
+ "FAD t1",
90
+ "FAD t2",
91
+ "FAD photons",
92
+ "NAD(P)H a1",
93
+ "NAD(P)H a1%",
94
+ "NAD(P)H a2",
95
+ "NAD(P)H a2%",
96
+ "NAD(P)H t1",
97
+ "NAD(P)H t2",
98
+ "NAD(P)H photons",
99
+ ]
100
+
101
+
102
+ # legacy workflow parameters that are settings of the training step
103
+ _TRAINING_KEYS = (
104
+ "epoches",
105
+ "learning_rate",
106
+ "weight_decay",
107
+ "batch_size",
108
+ "model",
109
+ "rescale",
110
+ "checkpoint_interval",
111
+ )
112
+ _SHARED = frozenset({"grouping", "features", "resources"})
113
+
114
+
115
+ def _workflow_training(**changes) -> AETrainConfig:
116
+ """Training step defaults of the autoencoder workflows."""
117
+ return AETrainConfig(
118
+ epochs=6,
119
+ learning_rates=[1e-5, 2e-5],
120
+ weight_decays=[1e-7, 1e-8],
121
+ batch_sizes=[128, 64],
122
+ rescale=True,
123
+ checkpoint_interval=2,
124
+ **changes,
125
+ )
126
+
127
+
128
+ class _AEWorkflowConfig(WorkflowConfig):
129
+ legacy_keys = {**WorkflowConfig.legacy_keys, "training": None}
130
+ bound_fields = {"training": _SHARED | {"timeseries"}}
131
+
132
+ grouping: list[CategoryColumn] = ["Treatment", "FOV", "Cell"]
133
+ features: list[FeatureColumn] = Field(default_factory=lambda: list(AE_FEATURES))
134
+ timeseries: CategoryColumn = Field(default="Treatment", title="Time series")
135
+ resources: AutoencoderResources = Field(default_factory=AutoencoderResources)
136
+ training: AETrainConfig
137
+
138
+ @classmethod
139
+ def _from_legacy(cls, params):
140
+ values = {}
141
+ if "device" in params:
142
+ values["resources"] = {"preferred_device": params.pop("device")}
143
+ legacy = {k: params.pop(k) for k in _TRAINING_KEYS if k in params}
144
+ default = cls.model_fields["training"].default_factory()
145
+ values["training"] = AETrainConfig.from_legacy_params(
146
+ {**default.to_legacy_params(), **legacy}
147
+ )
148
+ return values
149
+
150
+ def _to_legacy(self):
151
+ params = self.training.to_legacy_params()
152
+ return {
153
+ **{k: params[k] for k in _TRAINING_KEYS},
154
+ "device": self.resources.preferred_device,
155
+ }
156
+
157
+
158
+ class AETuneConfig(_AEWorkflowConfig):
159
+ """FLIM Data Augmentation Tuning: trains autoencoders, then augments the data."""
160
+
161
+ legacy_keys = {**_AEWorkflowConfig.legacy_keys, "augment": None}
162
+ bound_fields = {**_AEWorkflowConfig.bound_fields, "augment": _SHARED}
163
+
164
+ training: AETrainConfig = Field(
165
+ default_factory=lambda: _workflow_training(architecture="AE Simulator 1-6")
166
+ )
167
+ # the workflow simulates without noise
168
+ augment: AEAugmentConfig = Field(
169
+ default_factory=lambda: AEAugmentConfig(sets=4, add_noise=False)
170
+ )
171
+
172
+ @classmethod
173
+ def _from_legacy(cls, params):
174
+ values = super()._from_legacy(params)
175
+ augment = cls.model_fields["augment"].default_factory()
176
+ if "sets" in params:
177
+ augment.sets = params.pop("sets")
178
+ return {**values, "augment": augment}
179
+
180
+ def _to_legacy(self):
181
+ return {**super()._to_legacy(), "sets": self.augment.sets}
182
+
183
+
184
+ class AEFeatureConfig(_AEWorkflowConfig):
185
+ """FLIM Feature Analysis: trains autoencoders, then runs them on the data."""
186
+
187
+ legacy_keys = {
188
+ **_AEWorkflowConfig.legacy_keys,
189
+ "run": None,
190
+ "control_group": "ctrl_group",
191
+ }
192
+ # the run step uses the models the training step saves
193
+ bound_fields = {
194
+ **_AEWorkflowConfig.bound_fields,
195
+ "run": _SHARED | {"model_files"},
196
+ }
197
+
198
+ training: AETrainConfig = Field(default_factory=_workflow_training)
199
+ run: AERunConfig = Field(default_factory=AERunConfig)
200
+ control_group: Annotated[str, ValueOf("timeseries")] = Field(
201
+ default="Ctrl",
202
+ title="Control",
203
+ description="The value of the time series column used as control.",
204
+ )
flim/core/__init__.py ADDED
File without changes
File without changes
flim/core/config.py ADDED
@@ -0,0 +1,305 @@
1
+ """Configuration models of tools: *how* a task processes its input.
2
+
3
+ Configs are Pydantic models: validated when created and when a field changes, saved as
4
+ JSON, and the source from which the GUI builds configuration dialogs.
5
+ """
6
+
7
+ import copy
8
+ import logging
9
+ import typing
10
+ from dataclasses import dataclass
11
+ from typing import Annotated, Any, ClassVar, Literal, Optional, Self, Union
12
+
13
+ from pydantic import BaseModel, ConfigDict, Field, model_serializer, model_validator
14
+ from pydantic.fields import FieldInfo
15
+
16
+
17
+ class BaseConfig(BaseModel):
18
+ """Base of all configuration models."""
19
+
20
+ model_config = ConfigDict(
21
+ extra="forbid", validate_assignment=True, arbitrary_types_allowed=True
22
+ )
23
+
24
+ classname: Optional[str] = (
25
+ None # tool class, "module.Class", for loading/reproducing
26
+ )
27
+ version: Optional[str] = None # FLIMAnalyzer version that wrote the config
28
+
29
+ @classmethod
30
+ def allowed_values(cls, values: dict) -> dict[str, tuple]:
31
+ """Fields that allow only some of their values, given the other ``values``:
32
+ name -> the allowed values (the first: the default).
33
+
34
+ E.g. a lower half of a grid only exists if the grid is symmetric; legends of
35
+ a single figure are not inside a plot. Dialogs offer only the allowed values,
36
+ and hide a field with one. Configs that define this apply it with
37
+ ``_apply_allowed_values`` at the end of their validator.
38
+ """
39
+ return {}
40
+
41
+ @classmethod
42
+ def fixed_fields(cls, values: dict) -> dict:
43
+ """Fields with one allowed value: name -> the value."""
44
+ return {n: v[0] for n, v in cls.allowed_values(values).items() if len(v) == 1}
45
+
46
+ def _apply_allowed_values(self) -> None:
47
+ # through __dict__: assignments would run the validators again
48
+ for name, allowed in self.allowed_values(self.__dict__).items():
49
+ if self.__dict__.get(name) not in allowed:
50
+ self.__dict__[name] = allowed[0]
51
+
52
+ @model_serializer(mode="wrap")
53
+ def _serialize(self, handler):
54
+ data = handler(self)
55
+ # the version is set once per settings file; classname names the tool of a
56
+ # task config (see TaskConfig) and means nothing in nested models such as
57
+ # Resources
58
+ if data.get("version") is None:
59
+ data.pop("version", None)
60
+ if data.get("classname") is None:
61
+ if isinstance(self, TaskConfig):
62
+ # created before its tool was registered
63
+ data["classname"] = self._registered_classname()
64
+ else:
65
+ data.pop("classname", None)
66
+ return self._after_dump(data)
67
+
68
+ def _after_dump(self, data: dict) -> dict:
69
+ """Adjusts the dumped fields of this model (subclasses)."""
70
+ return data
71
+
72
+
73
+ # --- annotated field types: meaning for validation and for the GUI's widgets ---
74
+
75
+
76
+ @dataclass(frozen=True)
77
+ class ColumnRole:
78
+ """Marks a field as naming table columns of a kind: "category" or "feature"."""
79
+
80
+ role: str
81
+
82
+
83
+ @dataclass(frozen=True)
84
+ class ValueOf:
85
+ """Marks a field as holding a value of the category column that the field
86
+ ``column_field`` of the same config names (of a list of columns: the first)."""
87
+
88
+ column_field: str
89
+
90
+
91
+ CategoryColumn = Annotated[str, ColumnRole("category")]
92
+ FeatureColumn = Annotated[str, ColumnRole("feature")]
93
+ AllFeatures = Literal["all"]
94
+ Device = Literal["auto", "cpu", "cuda", "mps"]
95
+ Probability = Annotated[float, Field(gt=0, lt=1)]
96
+
97
+
98
+ def column_role(field: FieldInfo) -> Optional[str]:
99
+ """The ColumnRole of a field, also inside list[...] or unions; None if it has none."""
100
+ for meta in field.metadata:
101
+ if isinstance(meta, ColumnRole):
102
+ return meta.role
103
+ return _column_role_of(field.annotation)
104
+
105
+
106
+ def _column_role_of(annotation) -> Optional[str]:
107
+ if typing.get_origin(annotation) is Annotated:
108
+ for meta in typing.get_args(annotation)[1:]:
109
+ if isinstance(meta, ColumnRole):
110
+ return meta.role
111
+ for arg in typing.get_args(annotation):
112
+ role = _column_role_of(arg)
113
+ if role:
114
+ return role
115
+ return None
116
+
117
+
118
+ def fixed_value(model: type[BaseModel], name: str) -> tuple[bool, Any]:
119
+ """(True, value) if the field allows only one value, e.g. ``Literal["cpu"]``.
120
+
121
+ Such fields are not shown in configuration dialogs.
122
+ """
123
+ annotation = model.model_fields[name].annotation
124
+ if typing.get_origin(annotation) is Literal:
125
+ values = typing.get_args(annotation)
126
+ if len(values) == 1:
127
+ return True, values[0]
128
+ return False, None
129
+
130
+
131
+ # --- resources and the task config base ---
132
+
133
+
134
+ class Resources(BaseConfig):
135
+ """What a task requests; the executor decides what it actually gets.
136
+
137
+ Defaults are conservative (CPU, one thread); tools that can use more widen the
138
+ allowed values in a subclass.
139
+ """
140
+
141
+ # shown collapsed in dialogs (rarely changed)
142
+ advanced: ClassVar[bool] = True
143
+
144
+ preferred_device: Literal["cpu"] = Field(
145
+ default="cpu",
146
+ title="Device",
147
+ description="The device to compute on, if available where the task runs.",
148
+ )
149
+ # threads/processes inside one task; -1 = all cores allocated to the task
150
+ preferred_threads: Literal[1] = Field(
151
+ default=1,
152
+ title="Threads",
153
+ description="Threads within the task; -1: all cores allocated to the task.",
154
+ )
155
+
156
+
157
+ # legacy parameters of every plugin that are not part of a config: the input (1),
158
+ # execution (3) and the dialogs' "save configuration" option
159
+ LEGACY_IGNORED = frozenset(
160
+ {"input", "autosave", "working_dir", "saveconfig", "config_file", "_config"}
161
+ )
162
+ LEGACY_ALL_FEATURES = "all_features"
163
+ _NOT_LEGACY = ("classname", "version", "resources")
164
+
165
+
166
+ class TaskConfig(BaseConfig):
167
+ """Base of the configs of all tools and workflows.
168
+
169
+ ``from_legacy_params`` and ``to_legacy_params`` convert from and to the parameter
170
+ dicts of the 0.5 plugins (as stored in 0.5 settings files). Fields are matched by
171
+ name; ``legacy_keys`` names the legacy key where it differs, or None for fields the
172
+ hooks ``_from_legacy`` and ``_to_legacy`` convert (or that have no legacy key).
173
+ """
174
+
175
+ legacy_keys: ClassVar[dict[str, Optional[str]]] = {}
176
+ # legacy keys that are dropped: unused, or meaningless for tasks
177
+ legacy_dropped: ClassVar[frozenset[str]] = frozenset()
178
+ # inherited fields the tool does not use; not shown in dialogs
179
+ hidden_fields: ClassVar[frozenset[str]] = frozenset()
180
+
181
+ grouping: list[CategoryColumn] = Field(
182
+ default=[],
183
+ title="Grouping",
184
+ description="Category columns to group the data by, in this order.",
185
+ )
186
+ features: Union[list[FeatureColumn], AllFeatures] = Field(
187
+ default=[], title="Features", description="The feature columns to use."
188
+ )
189
+ prefix: Optional[str] = Field(
190
+ default=None,
191
+ title="Result prefix",
192
+ description="Text put in front of the names of the results.",
193
+ )
194
+ resources: Resources = Field(default_factory=Resources, title="Resources")
195
+
196
+ @model_validator(mode="after")
197
+ def _set_classname(self):
198
+ """Names the registered tool of this config, unless a classname is given."""
199
+ if self.classname is None:
200
+ # not setattr: that would run the validators again
201
+ self.__dict__["classname"] = self._registered_classname()
202
+ return self
203
+
204
+ def _registered_classname(self) -> Optional[str]:
205
+ """``module.Class`` of the registered tool of this config, None if none is.
206
+
207
+ Tools are registered when their module is imported, so configs are best
208
+ created after that (the application imports the tools at startup).
209
+ """
210
+ from flim.core.registry import tools
211
+
212
+ tool = tools.for_config(self)
213
+ return f"{tool.__module__}.{tool.__qualname__}" if tool else None
214
+
215
+ @classmethod
216
+ def from_legacy_params(cls, params: dict) -> Self:
217
+ """The config for a plugin's parameter dict; unknown keys are logged."""
218
+ # ignored keys first: "input" holds the input tables at run time
219
+ params = copy.deepcopy(
220
+ {k: v for k, v in params.items() if k not in LEGACY_IGNORED}
221
+ )
222
+ values = cls._from_legacy(params)
223
+ for name in cls.model_fields:
224
+ key = cls.legacy_keys.get(name, name)
225
+ if name in values or name in _NOT_LEGACY or key is None:
226
+ continue
227
+ if key in params:
228
+ values[name] = params.pop(key)
229
+ if values.get("features") == LEGACY_ALL_FEATURES:
230
+ values["features"] = "all"
231
+ unknown = set(params) - cls.legacy_dropped
232
+ if unknown:
233
+ logging.warning(
234
+ f"{cls.__name__}: ignoring unknown parameters {sorted(unknown)}"
235
+ )
236
+ return cls(**values)
237
+
238
+ def to_legacy_params(self) -> dict:
239
+ """The plugin parameter dict for this config."""
240
+ params = {}
241
+ for name in type(self).model_fields:
242
+ key = self.legacy_keys.get(name, name)
243
+ if name in _NOT_LEGACY or key is None:
244
+ continue
245
+ params[key] = copy.deepcopy(getattr(self, name))
246
+ if params.get("features") == "all":
247
+ params["features"] = LEGACY_ALL_FEATURES
248
+ params.update(self._to_legacy())
249
+ return params
250
+
251
+ @classmethod
252
+ def _from_legacy(cls, params: dict) -> dict:
253
+ """Converts (and pops) the legacy parameters that need more than a rename."""
254
+ return {}
255
+
256
+ def _to_legacy(self) -> dict:
257
+ """Legacy parameters that need more than a rename."""
258
+ return {}
259
+
260
+
261
+ @dataclass(frozen=True)
262
+ class Issue:
263
+ """A problem found when validating a config, e.g. against its input data.
264
+
265
+ ``field`` is a path into the config, e.g. ``"x_vars[1]"`` or
266
+ ``"range_filters[0].column"``. A ``"guessed"`` issue reports a value that was
267
+ replaced (``old``) by the closest match (``suggestion``) after an upstream change.
268
+ """
269
+
270
+ field: Optional[str]
271
+ message: str
272
+ severity: Literal["error", "warning", "guessed"] = "error"
273
+ step: Optional[str] = None # workflow step, None for a single tool
274
+ old: Any = None
275
+ suggestion: Any = None
276
+
277
+
278
+ def column_references(config: BaseModel, path: str = ""):
279
+ """Yields ``(path, column, role)`` for every column a config names, also in nested
280
+ models (e.g. range filters) and in lists and dict keys of column fields."""
281
+ for name, field in type(config).model_fields.items():
282
+ value = getattr(config, name)
283
+ field_path = f"{path}{name}"
284
+ role = column_role(field)
285
+ if role is not None:
286
+ if name == "features" and value == "all":
287
+ continue
288
+ if isinstance(value, str):
289
+ yield field_path, value, role
290
+ elif isinstance(value, (list, tuple)):
291
+ for i, v in enumerate(value):
292
+ yield f"{field_path}[{i}]", v, role
293
+ elif isinstance(value, dict):
294
+ for k in value:
295
+ yield f"{field_path}[{k!r}]", k, role
296
+ elif isinstance(value, BaseModel):
297
+ yield from column_references(value, f"{field_path}.")
298
+ elif isinstance(value, (list, tuple)):
299
+ for i, v in enumerate(value):
300
+ if isinstance(v, BaseModel):
301
+ yield from column_references(v, f"{field_path}[{i}].")
302
+ elif isinstance(value, dict):
303
+ for k, v in value.items():
304
+ if isinstance(v, BaseModel):
305
+ yield from column_references(v, f"{field_path}[{k!r}].")
flim/core/execution.py ADDED
@@ -0,0 +1,110 @@
1
+ """Execution options: *where and how* tasks run, kept out of the tool configs."""
2
+
3
+ import importlib.util
4
+ import logging
5
+ from pathlib import Path
6
+ from typing import Any, Literal, Optional
7
+
8
+ from pydantic import Field, model_validator
9
+
10
+ from flim.core.config import BaseConfig
11
+
12
+ # executor names of the command line (-e) and of saved settings before ExecutionOptions
13
+ LEGACY_EXECUTORS = {
14
+ "LocalExecutor": "local",
15
+ "LocalDaskExecutor": "dask-local",
16
+ "DaskExecutor": "dask",
17
+ }
18
+ DASK = ("dask-local", "dask")
19
+
20
+
21
+ class ExecutionOptions(BaseConfig):
22
+ executor: Literal["local", "dask-local", "dask"] = "local"
23
+ # parallel tasks; None = flim.core.runner.default_max_workers()
24
+ num_workers: Optional[int] = Field(default=None, ge=1)
25
+ scheduler: Literal["threads", "processes"] = "threads" # dask-local only
26
+ address: Optional[str] = None # dask only: scheduler of an existing cluster
27
+ dask_kwargs: dict[str, Any] = {} # further DaskTaskRunner / cluster arguments
28
+ autosave: bool = False
29
+ results_dir: Optional[Path] = None
30
+ task_tags: list[str] = []
31
+ device_policy: Literal["prefer", "cpu-only"] = "prefer"
32
+
33
+ @model_validator(mode="after")
34
+ def _check_executor_args(self):
35
+ if self.address is not None and self.executor != "dask":
36
+ raise ValueError("address is only used by the dask executor")
37
+ if self.dask_kwargs and self.executor not in DASK:
38
+ raise ValueError("dask_kwargs are only used by the dask executors")
39
+ if self.autosave and self.results_dir is None:
40
+ raise ValueError("autosave needs a results_dir")
41
+ return self
42
+
43
+ @classmethod
44
+ def from_legacy(cls, executor: str = "LocalExecutor", execargs: str = "", **kwargs):
45
+ """Options from the command line's ``-e`` executor name and ``"k=v,k=v"`` args.
46
+
47
+ As before, an unknown executor runs locally and the local executor uses only an
48
+ integer ``num_workers``; for Dask, unknown args go to the Dask task runner.
49
+ """
50
+ args = {}
51
+ for item in (execargs or "").split(","):
52
+ parts = item.split("=")
53
+ if len(parts) != 2:
54
+ continue
55
+ key, value = parts[0].strip(), parts[1].strip()
56
+ try:
57
+ value = int(value)
58
+ except ValueError:
59
+ pass
60
+ args[key] = value
61
+ name = LEGACY_EXECUTORS.get(executor, "local")
62
+ if executor not in LEGACY_EXECUTORS:
63
+ logging.warning(f"Unknown executor {executor}, using LocalExecutor")
64
+ num_workers = args.pop("num_workers", None)
65
+ if not isinstance(num_workers, int) or num_workers < 1:
66
+ num_workers = None
67
+ options = {"executor": name, "num_workers": num_workers, **kwargs}
68
+ if name == "dask-local" and "scheduler" in args:
69
+ scheduler = args.pop("scheduler")
70
+ options["scheduler"] = (
71
+ "processes" if scheduler == "processes" else "threads"
72
+ )
73
+ if name == "dask" and "address" in args:
74
+ options["address"] = str(args.pop("address"))
75
+ if name in DASK:
76
+ options["dask_kwargs"] = args
77
+ return cls(**options)
78
+
79
+ def check_available(self) -> None:
80
+ """Raises RuntimeError if the executor needs packages that are not installed."""
81
+ if self.executor in DASK and importlib.util.find_spec("prefect_dask") is None:
82
+ raise RuntimeError(
83
+ f"The {self.executor} executor needs the optional Dask support:"
84
+ " uv sync --extra dask (or pip install flimanalyzer[dask])"
85
+ )
86
+
87
+ def task_runner(self):
88
+ """The Prefect task runner.
89
+
90
+ ``local``: a ``ThreadPoolTaskRunner`` with ``num_workers`` threads.
91
+ ``dask-local``: a ``DaskTaskRunner`` starting a local Dask cluster.
92
+ ``dask``: a ``DaskTaskRunner`` connecting to the cluster at ``address`` (or
93
+ starting a local one if no address is given).
94
+ """
95
+ if self.executor == "local":
96
+ from flim.core.runner import default_task_runner
97
+
98
+ return default_task_runner(self.num_workers)
99
+ self.check_available()
100
+ from prefect_dask import DaskTaskRunner
101
+
102
+ if self.executor == "dask":
103
+ args = dict(self.dask_kwargs)
104
+ if self.address is not None:
105
+ args["address"] = self.address
106
+ return DaskTaskRunner(**args)
107
+ cluster_kwargs = {"processes": self.scheduler == "processes"}
108
+ if self.num_workers is not None:
109
+ cluster_kwargs["n_workers"] = self.num_workers
110
+ return DaskTaskRunner(cluster_kwargs={**cluster_kwargs, **self.dask_kwargs})