flimanalyzer 0.6.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- flim/__init__.py +2 -0
- flim/_version.py +5 -0
- flim/analysis/__init__.py +1 -0
- flim/analysis/ml/__init__.py +0 -0
- flim/analysis/ml/autoencoder.py +188 -0
- flim/analyzerapp.py +131 -0
- flim/batch.py +235 -0
- flim/configs/__init__.py +77 -0
- flim/configs/analysis.py +554 -0
- flim/configs/data.py +278 -0
- flim/configs/plot.py +543 -0
- flim/configs/workflow.py +204 -0
- flim/core/__init__.py +0 -0
- flim/core/calculator.py +0 -0
- flim/core/config.py +305 -0
- flim/core/execution.py +110 -0
- flim/core/fields.py +226 -0
- flim/core/filter.py +179 -0
- flim/core/graph.py +111 -0
- flim/core/headers.py +72 -0
- flim/core/importer.py +269 -0
- flim/core/parser.py +305 -0
- flim/core/paths.py +66 -0
- flim/core/prefect_server.py +152 -0
- flim/core/preprocessor.py +287 -0
- flim/core/registry.py +108 -0
- flim/core/runner.py +703 -0
- flim/core/schema.py +114 -0
- flim/core/settings.py +419 -0
- flim/core/tool.py +284 -0
- flim/core/tools.py +76 -0
- flim/core/types.py +113 -0
- flim/core/validation.py +272 -0
- flim/core/workflow.py +148 -0
- flim/gui/__init__.py +12 -0
- flim/gui/app.py +953 -0
- flim/gui/configdialog.py +495 -0
- flim/gui/datapanel.py +841 -0
- flim/gui/delimpanel.py +56 -0
- flim/gui/dialogs/__init__.py +473 -0
- flim/gui/dialogs/autoencoder.py +89 -0
- flim/gui/dialogs/categoryorder.py +130 -0
- flim/gui/dialogs/filter.py +86 -0
- flim/gui/dialogs/unpivot.py +72 -0
- flim/gui/dicttablepanel.py +178 -0
- flim/gui/editors.py +1578 -0
- flim/gui/events.py +162 -0
- flim/gui/icons.py +16 -0
- flim/gui/importdlg.py +391 -0
- flim/gui/listcontrol.py +472 -0
- flim/gui/macos.py +77 -0
- flim/gui/plotframe.py +73 -0
- flim/gui/progressdlg.py +290 -0
- flim/resources/aeaugment.png +0 -0
- flim/resources/aefeatureworkflow.png +0 -0
- flim/resources/aerun.png +0 -0
- flim/resources/aetrain.png +0 -0
- flim/resources/aetuneworkflow.png +0 -0
- flim/resources/barplot.png +0 -0
- flim/resources/boxplot.png +0 -0
- flim/resources/categorize.png +0 -0
- flim/resources/concatenate.png +0 -0
- flim/resources/fileopen.png +0 -0
- flim/resources/filter.png +0 -0
- flim/resources/flimanalyzer.ico +0 -0
- flim/resources/flimanalyzer.png +0 -0
- flim/resources/flirrplot.png +0 -0
- flim/resources/flirrworkflow.png +0 -0
- flim/resources/gmm.png +0 -0
- flim/resources/heatmap.png +0 -0
- flim/resources/histogram.png +0 -0
- flim/resources/import.png +0 -0
- flim/resources/kde.png +0 -0
- flim/resources/kmeans.png +0 -0
- flim/resources/ks.png +0 -0
- flim/resources/lineplot.png +0 -0
- flim/resources/merge.png +0 -0
- flim/resources/order.png +0 -0
- flim/resources/pairplot.png +0 -0
- flim/resources/pca.png +0 -0
- flim/resources/pivot.png +0 -0
- flim/resources/randomforest.png +0 -0
- flim/resources/relchange.png +0 -0
- flim/resources/scatter.png +0 -0
- flim/resources/seriesanalysis.png +0 -0
- flim/resources/sort.png +0 -0
- flim/resources/split.png +0 -0
- flim/resources/summary.png +0 -0
- flim/resources/swarmplot.png +0 -0
- flim/resources/unpivot.png +0 -0
- flim/resources/violinplot.png +0 -0
- flim/resources/workflow.png +0 -0
- flim/results.py +70 -0
- flim/tools/__init__.py +7 -0
- flim/tools/autoencoder.py +662 -0
- flim/tools/common.py +62 -0
- flim/tools/data.py +301 -0
- flim/tools/ml.py +311 -0
- flim/tools/plot.py +1439 -0
- flim/tools/statistics.py +348 -0
- flim/utils.py +43 -0
- flim/workflow/__init__.py +3 -0
- flim/workflow/aefeature.py +145 -0
- flim/workflow/aetune.py +163 -0
- flim/workflow/basicflow.py +99 -0
- flimanalyzer-0.6.0.dist-info/METADATA +262 -0
- flimanalyzer-0.6.0.dist-info/RECORD +110 -0
- flimanalyzer-0.6.0.dist-info/WHEEL +4 -0
- flimanalyzer-0.6.0.dist-info/entry_points.txt +5 -0
- flimanalyzer-0.6.0.dist-info/licenses/LICENSE +19 -0
flim/configs/workflow.py
ADDED
|
@@ -0,0 +1,204 @@
|
|
|
1
|
+
"""Configs of the workflows: their own fields plus the configs of their steps.
|
|
2
|
+
|
|
3
|
+
A workflow config owns the fields its steps share (grouping, features, ...); each step
|
|
4
|
+
config holds only the settings of that step. ``bound_fields`` names the fields the
|
|
5
|
+
workflow sets in each step: they are copied from the workflow config when it is
|
|
6
|
+
validated, hidden in dialogs and not saved.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
import copy
|
|
10
|
+
from typing import Annotated, ClassVar, Optional
|
|
11
|
+
|
|
12
|
+
from pydantic import Field, model_validator
|
|
13
|
+
|
|
14
|
+
from flim.configs.analysis import (
|
|
15
|
+
AE_FEATURES,
|
|
16
|
+
AEAugmentConfig,
|
|
17
|
+
AERunConfig,
|
|
18
|
+
AETrainConfig,
|
|
19
|
+
AutoencoderResources,
|
|
20
|
+
)
|
|
21
|
+
from flim.core.config import CategoryColumn, FeatureColumn, TaskConfig, ValueOf
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class WorkflowConfig(TaskConfig):
|
|
25
|
+
legacy_keys = {"strict": None}
|
|
26
|
+
legacy_dropped = frozenset({"executor"}) # an execution option
|
|
27
|
+
|
|
28
|
+
# step name -> fields the workflow sets in that step's config. Fields the workflow
|
|
29
|
+
# config also has are copied from it; the workflow's code sets the others.
|
|
30
|
+
bound_fields: ClassVar[dict[str, frozenset[str]]] = {}
|
|
31
|
+
|
|
32
|
+
# False: step settings naming columns that changed upstream are replaced by the
|
|
33
|
+
# closest match (and reported); True: they are errors to fix before running.
|
|
34
|
+
# None: the mode's default, False in the GUI and True in batch runs.
|
|
35
|
+
strict: Optional[bool] = Field(
|
|
36
|
+
default=None,
|
|
37
|
+
title="Strict",
|
|
38
|
+
description=(
|
|
39
|
+
"Columns that changed upstream are errors to fix; otherwise they are"
|
|
40
|
+
" replaced by the closest match. Automatic: strict in batch runs only."
|
|
41
|
+
),
|
|
42
|
+
)
|
|
43
|
+
|
|
44
|
+
@model_validator(mode="after")
|
|
45
|
+
def _share_fields(self):
|
|
46
|
+
for step, names in self.bound_fields.items():
|
|
47
|
+
config = getattr(self, step)
|
|
48
|
+
for name in names & set(type(self).model_fields):
|
|
49
|
+
value = getattr(self, name)
|
|
50
|
+
if getattr(config, name) != value:
|
|
51
|
+
setattr(config, name, copy.deepcopy(value))
|
|
52
|
+
return self
|
|
53
|
+
|
|
54
|
+
def _after_dump(self, data):
|
|
55
|
+
for step, names in self.bound_fields.items():
|
|
56
|
+
if isinstance(data.get(step), dict):
|
|
57
|
+
for name in names:
|
|
58
|
+
data[step].pop(name, None)
|
|
59
|
+
return data
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
class BasicFLIRRConfig(WorkflowConfig):
|
|
63
|
+
grouping: list[CategoryColumn] = ["Treatment", "FOV", "Cell"]
|
|
64
|
+
# values the workflow used to take from the data, now settings with that default
|
|
65
|
+
reference_value: Annotated[Optional[str], ValueOf("grouping")] = Field(
|
|
66
|
+
default=None,
|
|
67
|
+
title="Reference value",
|
|
68
|
+
description=(
|
|
69
|
+
"Relative changes: the features are divided by their mean in the rows with"
|
|
70
|
+
" this value of the first grouping column. Empty: its first value (in the"
|
|
71
|
+
" order of its categories)."
|
|
72
|
+
),
|
|
73
|
+
)
|
|
74
|
+
fad_a1: Optional[FeatureColumn] = Field(
|
|
75
|
+
default=None,
|
|
76
|
+
title="Scatter x (FAD a1 %)",
|
|
77
|
+
description="Empty: found by name (FAD, a1, %).",
|
|
78
|
+
)
|
|
79
|
+
nadph_a2: Optional[FeatureColumn] = Field(
|
|
80
|
+
default=None,
|
|
81
|
+
title="Scatter y (NAD(P)H a2 %)",
|
|
82
|
+
description="Empty: found by name (NAD, a2, %).",
|
|
83
|
+
)
|
|
84
|
+
features: list[FeatureColumn] = [
|
|
85
|
+
"FAD a1",
|
|
86
|
+
"FAD a1%",
|
|
87
|
+
"FAD a2",
|
|
88
|
+
"FAD a2%",
|
|
89
|
+
"FAD t1",
|
|
90
|
+
"FAD t2",
|
|
91
|
+
"FAD photons",
|
|
92
|
+
"NAD(P)H a1",
|
|
93
|
+
"NAD(P)H a1%",
|
|
94
|
+
"NAD(P)H a2",
|
|
95
|
+
"NAD(P)H a2%",
|
|
96
|
+
"NAD(P)H t1",
|
|
97
|
+
"NAD(P)H t2",
|
|
98
|
+
"NAD(P)H photons",
|
|
99
|
+
]
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
# legacy workflow parameters that are settings of the training step
|
|
103
|
+
_TRAINING_KEYS = (
|
|
104
|
+
"epoches",
|
|
105
|
+
"learning_rate",
|
|
106
|
+
"weight_decay",
|
|
107
|
+
"batch_size",
|
|
108
|
+
"model",
|
|
109
|
+
"rescale",
|
|
110
|
+
"checkpoint_interval",
|
|
111
|
+
)
|
|
112
|
+
_SHARED = frozenset({"grouping", "features", "resources"})
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def _workflow_training(**changes) -> AETrainConfig:
|
|
116
|
+
"""Training step defaults of the autoencoder workflows."""
|
|
117
|
+
return AETrainConfig(
|
|
118
|
+
epochs=6,
|
|
119
|
+
learning_rates=[1e-5, 2e-5],
|
|
120
|
+
weight_decays=[1e-7, 1e-8],
|
|
121
|
+
batch_sizes=[128, 64],
|
|
122
|
+
rescale=True,
|
|
123
|
+
checkpoint_interval=2,
|
|
124
|
+
**changes,
|
|
125
|
+
)
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
class _AEWorkflowConfig(WorkflowConfig):
|
|
129
|
+
legacy_keys = {**WorkflowConfig.legacy_keys, "training": None}
|
|
130
|
+
bound_fields = {"training": _SHARED | {"timeseries"}}
|
|
131
|
+
|
|
132
|
+
grouping: list[CategoryColumn] = ["Treatment", "FOV", "Cell"]
|
|
133
|
+
features: list[FeatureColumn] = Field(default_factory=lambda: list(AE_FEATURES))
|
|
134
|
+
timeseries: CategoryColumn = Field(default="Treatment", title="Time series")
|
|
135
|
+
resources: AutoencoderResources = Field(default_factory=AutoencoderResources)
|
|
136
|
+
training: AETrainConfig
|
|
137
|
+
|
|
138
|
+
@classmethod
|
|
139
|
+
def _from_legacy(cls, params):
|
|
140
|
+
values = {}
|
|
141
|
+
if "device" in params:
|
|
142
|
+
values["resources"] = {"preferred_device": params.pop("device")}
|
|
143
|
+
legacy = {k: params.pop(k) for k in _TRAINING_KEYS if k in params}
|
|
144
|
+
default = cls.model_fields["training"].default_factory()
|
|
145
|
+
values["training"] = AETrainConfig.from_legacy_params(
|
|
146
|
+
{**default.to_legacy_params(), **legacy}
|
|
147
|
+
)
|
|
148
|
+
return values
|
|
149
|
+
|
|
150
|
+
def _to_legacy(self):
|
|
151
|
+
params = self.training.to_legacy_params()
|
|
152
|
+
return {
|
|
153
|
+
**{k: params[k] for k in _TRAINING_KEYS},
|
|
154
|
+
"device": self.resources.preferred_device,
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
class AETuneConfig(_AEWorkflowConfig):
|
|
159
|
+
"""FLIM Data Augmentation Tuning: trains autoencoders, then augments the data."""
|
|
160
|
+
|
|
161
|
+
legacy_keys = {**_AEWorkflowConfig.legacy_keys, "augment": None}
|
|
162
|
+
bound_fields = {**_AEWorkflowConfig.bound_fields, "augment": _SHARED}
|
|
163
|
+
|
|
164
|
+
training: AETrainConfig = Field(
|
|
165
|
+
default_factory=lambda: _workflow_training(architecture="AE Simulator 1-6")
|
|
166
|
+
)
|
|
167
|
+
# the workflow simulates without noise
|
|
168
|
+
augment: AEAugmentConfig = Field(
|
|
169
|
+
default_factory=lambda: AEAugmentConfig(sets=4, add_noise=False)
|
|
170
|
+
)
|
|
171
|
+
|
|
172
|
+
@classmethod
|
|
173
|
+
def _from_legacy(cls, params):
|
|
174
|
+
values = super()._from_legacy(params)
|
|
175
|
+
augment = cls.model_fields["augment"].default_factory()
|
|
176
|
+
if "sets" in params:
|
|
177
|
+
augment.sets = params.pop("sets")
|
|
178
|
+
return {**values, "augment": augment}
|
|
179
|
+
|
|
180
|
+
def _to_legacy(self):
|
|
181
|
+
return {**super()._to_legacy(), "sets": self.augment.sets}
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
class AEFeatureConfig(_AEWorkflowConfig):
|
|
185
|
+
"""FLIM Feature Analysis: trains autoencoders, then runs them on the data."""
|
|
186
|
+
|
|
187
|
+
legacy_keys = {
|
|
188
|
+
**_AEWorkflowConfig.legacy_keys,
|
|
189
|
+
"run": None,
|
|
190
|
+
"control_group": "ctrl_group",
|
|
191
|
+
}
|
|
192
|
+
# the run step uses the models the training step saves
|
|
193
|
+
bound_fields = {
|
|
194
|
+
**_AEWorkflowConfig.bound_fields,
|
|
195
|
+
"run": _SHARED | {"model_files"},
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
training: AETrainConfig = Field(default_factory=_workflow_training)
|
|
199
|
+
run: AERunConfig = Field(default_factory=AERunConfig)
|
|
200
|
+
control_group: Annotated[str, ValueOf("timeseries")] = Field(
|
|
201
|
+
default="Ctrl",
|
|
202
|
+
title="Control",
|
|
203
|
+
description="The value of the time series column used as control.",
|
|
204
|
+
)
|
flim/core/__init__.py
ADDED
|
File without changes
|
flim/core/calculator.py
ADDED
|
File without changes
|
flim/core/config.py
ADDED
|
@@ -0,0 +1,305 @@
|
|
|
1
|
+
"""Configuration models of tools: *how* a task processes its input.
|
|
2
|
+
|
|
3
|
+
Configs are Pydantic models: validated when created and when a field changes, saved as
|
|
4
|
+
JSON, and the source from which the GUI builds configuration dialogs.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
import copy
|
|
8
|
+
import logging
|
|
9
|
+
import typing
|
|
10
|
+
from dataclasses import dataclass
|
|
11
|
+
from typing import Annotated, Any, ClassVar, Literal, Optional, Self, Union
|
|
12
|
+
|
|
13
|
+
from pydantic import BaseModel, ConfigDict, Field, model_serializer, model_validator
|
|
14
|
+
from pydantic.fields import FieldInfo
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class BaseConfig(BaseModel):
|
|
18
|
+
"""Base of all configuration models."""
|
|
19
|
+
|
|
20
|
+
model_config = ConfigDict(
|
|
21
|
+
extra="forbid", validate_assignment=True, arbitrary_types_allowed=True
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
classname: Optional[str] = (
|
|
25
|
+
None # tool class, "module.Class", for loading/reproducing
|
|
26
|
+
)
|
|
27
|
+
version: Optional[str] = None # FLIMAnalyzer version that wrote the config
|
|
28
|
+
|
|
29
|
+
@classmethod
|
|
30
|
+
def allowed_values(cls, values: dict) -> dict[str, tuple]:
|
|
31
|
+
"""Fields that allow only some of their values, given the other ``values``:
|
|
32
|
+
name -> the allowed values (the first: the default).
|
|
33
|
+
|
|
34
|
+
E.g. a lower half of a grid only exists if the grid is symmetric; legends of
|
|
35
|
+
a single figure are not inside a plot. Dialogs offer only the allowed values,
|
|
36
|
+
and hide a field with one. Configs that define this apply it with
|
|
37
|
+
``_apply_allowed_values`` at the end of their validator.
|
|
38
|
+
"""
|
|
39
|
+
return {}
|
|
40
|
+
|
|
41
|
+
@classmethod
|
|
42
|
+
def fixed_fields(cls, values: dict) -> dict:
|
|
43
|
+
"""Fields with one allowed value: name -> the value."""
|
|
44
|
+
return {n: v[0] for n, v in cls.allowed_values(values).items() if len(v) == 1}
|
|
45
|
+
|
|
46
|
+
def _apply_allowed_values(self) -> None:
|
|
47
|
+
# through __dict__: assignments would run the validators again
|
|
48
|
+
for name, allowed in self.allowed_values(self.__dict__).items():
|
|
49
|
+
if self.__dict__.get(name) not in allowed:
|
|
50
|
+
self.__dict__[name] = allowed[0]
|
|
51
|
+
|
|
52
|
+
@model_serializer(mode="wrap")
|
|
53
|
+
def _serialize(self, handler):
|
|
54
|
+
data = handler(self)
|
|
55
|
+
# the version is set once per settings file; classname names the tool of a
|
|
56
|
+
# task config (see TaskConfig) and means nothing in nested models such as
|
|
57
|
+
# Resources
|
|
58
|
+
if data.get("version") is None:
|
|
59
|
+
data.pop("version", None)
|
|
60
|
+
if data.get("classname") is None:
|
|
61
|
+
if isinstance(self, TaskConfig):
|
|
62
|
+
# created before its tool was registered
|
|
63
|
+
data["classname"] = self._registered_classname()
|
|
64
|
+
else:
|
|
65
|
+
data.pop("classname", None)
|
|
66
|
+
return self._after_dump(data)
|
|
67
|
+
|
|
68
|
+
def _after_dump(self, data: dict) -> dict:
|
|
69
|
+
"""Adjusts the dumped fields of this model (subclasses)."""
|
|
70
|
+
return data
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
# --- annotated field types: meaning for validation and for the GUI's widgets ---
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
@dataclass(frozen=True)
|
|
77
|
+
class ColumnRole:
|
|
78
|
+
"""Marks a field as naming table columns of a kind: "category" or "feature"."""
|
|
79
|
+
|
|
80
|
+
role: str
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
@dataclass(frozen=True)
|
|
84
|
+
class ValueOf:
|
|
85
|
+
"""Marks a field as holding a value of the category column that the field
|
|
86
|
+
``column_field`` of the same config names (of a list of columns: the first)."""
|
|
87
|
+
|
|
88
|
+
column_field: str
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
CategoryColumn = Annotated[str, ColumnRole("category")]
|
|
92
|
+
FeatureColumn = Annotated[str, ColumnRole("feature")]
|
|
93
|
+
AllFeatures = Literal["all"]
|
|
94
|
+
Device = Literal["auto", "cpu", "cuda", "mps"]
|
|
95
|
+
Probability = Annotated[float, Field(gt=0, lt=1)]
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def column_role(field: FieldInfo) -> Optional[str]:
|
|
99
|
+
"""The ColumnRole of a field, also inside list[...] or unions; None if it has none."""
|
|
100
|
+
for meta in field.metadata:
|
|
101
|
+
if isinstance(meta, ColumnRole):
|
|
102
|
+
return meta.role
|
|
103
|
+
return _column_role_of(field.annotation)
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def _column_role_of(annotation) -> Optional[str]:
|
|
107
|
+
if typing.get_origin(annotation) is Annotated:
|
|
108
|
+
for meta in typing.get_args(annotation)[1:]:
|
|
109
|
+
if isinstance(meta, ColumnRole):
|
|
110
|
+
return meta.role
|
|
111
|
+
for arg in typing.get_args(annotation):
|
|
112
|
+
role = _column_role_of(arg)
|
|
113
|
+
if role:
|
|
114
|
+
return role
|
|
115
|
+
return None
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def fixed_value(model: type[BaseModel], name: str) -> tuple[bool, Any]:
|
|
119
|
+
"""(True, value) if the field allows only one value, e.g. ``Literal["cpu"]``.
|
|
120
|
+
|
|
121
|
+
Such fields are not shown in configuration dialogs.
|
|
122
|
+
"""
|
|
123
|
+
annotation = model.model_fields[name].annotation
|
|
124
|
+
if typing.get_origin(annotation) is Literal:
|
|
125
|
+
values = typing.get_args(annotation)
|
|
126
|
+
if len(values) == 1:
|
|
127
|
+
return True, values[0]
|
|
128
|
+
return False, None
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
# --- resources and the task config base ---
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
class Resources(BaseConfig):
|
|
135
|
+
"""What a task requests; the executor decides what it actually gets.
|
|
136
|
+
|
|
137
|
+
Defaults are conservative (CPU, one thread); tools that can use more widen the
|
|
138
|
+
allowed values in a subclass.
|
|
139
|
+
"""
|
|
140
|
+
|
|
141
|
+
# shown collapsed in dialogs (rarely changed)
|
|
142
|
+
advanced: ClassVar[bool] = True
|
|
143
|
+
|
|
144
|
+
preferred_device: Literal["cpu"] = Field(
|
|
145
|
+
default="cpu",
|
|
146
|
+
title="Device",
|
|
147
|
+
description="The device to compute on, if available where the task runs.",
|
|
148
|
+
)
|
|
149
|
+
# threads/processes inside one task; -1 = all cores allocated to the task
|
|
150
|
+
preferred_threads: Literal[1] = Field(
|
|
151
|
+
default=1,
|
|
152
|
+
title="Threads",
|
|
153
|
+
description="Threads within the task; -1: all cores allocated to the task.",
|
|
154
|
+
)
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
# legacy parameters of every plugin that are not part of a config: the input (1),
|
|
158
|
+
# execution (3) and the dialogs' "save configuration" option
|
|
159
|
+
LEGACY_IGNORED = frozenset(
|
|
160
|
+
{"input", "autosave", "working_dir", "saveconfig", "config_file", "_config"}
|
|
161
|
+
)
|
|
162
|
+
LEGACY_ALL_FEATURES = "all_features"
|
|
163
|
+
_NOT_LEGACY = ("classname", "version", "resources")
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
class TaskConfig(BaseConfig):
|
|
167
|
+
"""Base of the configs of all tools and workflows.
|
|
168
|
+
|
|
169
|
+
``from_legacy_params`` and ``to_legacy_params`` convert from and to the parameter
|
|
170
|
+
dicts of the 0.5 plugins (as stored in 0.5 settings files). Fields are matched by
|
|
171
|
+
name; ``legacy_keys`` names the legacy key where it differs, or None for fields the
|
|
172
|
+
hooks ``_from_legacy`` and ``_to_legacy`` convert (or that have no legacy key).
|
|
173
|
+
"""
|
|
174
|
+
|
|
175
|
+
legacy_keys: ClassVar[dict[str, Optional[str]]] = {}
|
|
176
|
+
# legacy keys that are dropped: unused, or meaningless for tasks
|
|
177
|
+
legacy_dropped: ClassVar[frozenset[str]] = frozenset()
|
|
178
|
+
# inherited fields the tool does not use; not shown in dialogs
|
|
179
|
+
hidden_fields: ClassVar[frozenset[str]] = frozenset()
|
|
180
|
+
|
|
181
|
+
grouping: list[CategoryColumn] = Field(
|
|
182
|
+
default=[],
|
|
183
|
+
title="Grouping",
|
|
184
|
+
description="Category columns to group the data by, in this order.",
|
|
185
|
+
)
|
|
186
|
+
features: Union[list[FeatureColumn], AllFeatures] = Field(
|
|
187
|
+
default=[], title="Features", description="The feature columns to use."
|
|
188
|
+
)
|
|
189
|
+
prefix: Optional[str] = Field(
|
|
190
|
+
default=None,
|
|
191
|
+
title="Result prefix",
|
|
192
|
+
description="Text put in front of the names of the results.",
|
|
193
|
+
)
|
|
194
|
+
resources: Resources = Field(default_factory=Resources, title="Resources")
|
|
195
|
+
|
|
196
|
+
@model_validator(mode="after")
|
|
197
|
+
def _set_classname(self):
|
|
198
|
+
"""Names the registered tool of this config, unless a classname is given."""
|
|
199
|
+
if self.classname is None:
|
|
200
|
+
# not setattr: that would run the validators again
|
|
201
|
+
self.__dict__["classname"] = self._registered_classname()
|
|
202
|
+
return self
|
|
203
|
+
|
|
204
|
+
def _registered_classname(self) -> Optional[str]:
|
|
205
|
+
"""``module.Class`` of the registered tool of this config, None if none is.
|
|
206
|
+
|
|
207
|
+
Tools are registered when their module is imported, so configs are best
|
|
208
|
+
created after that (the application imports the tools at startup).
|
|
209
|
+
"""
|
|
210
|
+
from flim.core.registry import tools
|
|
211
|
+
|
|
212
|
+
tool = tools.for_config(self)
|
|
213
|
+
return f"{tool.__module__}.{tool.__qualname__}" if tool else None
|
|
214
|
+
|
|
215
|
+
@classmethod
|
|
216
|
+
def from_legacy_params(cls, params: dict) -> Self:
|
|
217
|
+
"""The config for a plugin's parameter dict; unknown keys are logged."""
|
|
218
|
+
# ignored keys first: "input" holds the input tables at run time
|
|
219
|
+
params = copy.deepcopy(
|
|
220
|
+
{k: v for k, v in params.items() if k not in LEGACY_IGNORED}
|
|
221
|
+
)
|
|
222
|
+
values = cls._from_legacy(params)
|
|
223
|
+
for name in cls.model_fields:
|
|
224
|
+
key = cls.legacy_keys.get(name, name)
|
|
225
|
+
if name in values or name in _NOT_LEGACY or key is None:
|
|
226
|
+
continue
|
|
227
|
+
if key in params:
|
|
228
|
+
values[name] = params.pop(key)
|
|
229
|
+
if values.get("features") == LEGACY_ALL_FEATURES:
|
|
230
|
+
values["features"] = "all"
|
|
231
|
+
unknown = set(params) - cls.legacy_dropped
|
|
232
|
+
if unknown:
|
|
233
|
+
logging.warning(
|
|
234
|
+
f"{cls.__name__}: ignoring unknown parameters {sorted(unknown)}"
|
|
235
|
+
)
|
|
236
|
+
return cls(**values)
|
|
237
|
+
|
|
238
|
+
def to_legacy_params(self) -> dict:
|
|
239
|
+
"""The plugin parameter dict for this config."""
|
|
240
|
+
params = {}
|
|
241
|
+
for name in type(self).model_fields:
|
|
242
|
+
key = self.legacy_keys.get(name, name)
|
|
243
|
+
if name in _NOT_LEGACY or key is None:
|
|
244
|
+
continue
|
|
245
|
+
params[key] = copy.deepcopy(getattr(self, name))
|
|
246
|
+
if params.get("features") == "all":
|
|
247
|
+
params["features"] = LEGACY_ALL_FEATURES
|
|
248
|
+
params.update(self._to_legacy())
|
|
249
|
+
return params
|
|
250
|
+
|
|
251
|
+
@classmethod
|
|
252
|
+
def _from_legacy(cls, params: dict) -> dict:
|
|
253
|
+
"""Converts (and pops) the legacy parameters that need more than a rename."""
|
|
254
|
+
return {}
|
|
255
|
+
|
|
256
|
+
def _to_legacy(self) -> dict:
|
|
257
|
+
"""Legacy parameters that need more than a rename."""
|
|
258
|
+
return {}
|
|
259
|
+
|
|
260
|
+
|
|
261
|
+
@dataclass(frozen=True)
|
|
262
|
+
class Issue:
|
|
263
|
+
"""A problem found when validating a config, e.g. against its input data.
|
|
264
|
+
|
|
265
|
+
``field`` is a path into the config, e.g. ``"x_vars[1]"`` or
|
|
266
|
+
``"range_filters[0].column"``. A ``"guessed"`` issue reports a value that was
|
|
267
|
+
replaced (``old``) by the closest match (``suggestion``) after an upstream change.
|
|
268
|
+
"""
|
|
269
|
+
|
|
270
|
+
field: Optional[str]
|
|
271
|
+
message: str
|
|
272
|
+
severity: Literal["error", "warning", "guessed"] = "error"
|
|
273
|
+
step: Optional[str] = None # workflow step, None for a single tool
|
|
274
|
+
old: Any = None
|
|
275
|
+
suggestion: Any = None
|
|
276
|
+
|
|
277
|
+
|
|
278
|
+
def column_references(config: BaseModel, path: str = ""):
|
|
279
|
+
"""Yields ``(path, column, role)`` for every column a config names, also in nested
|
|
280
|
+
models (e.g. range filters) and in lists and dict keys of column fields."""
|
|
281
|
+
for name, field in type(config).model_fields.items():
|
|
282
|
+
value = getattr(config, name)
|
|
283
|
+
field_path = f"{path}{name}"
|
|
284
|
+
role = column_role(field)
|
|
285
|
+
if role is not None:
|
|
286
|
+
if name == "features" and value == "all":
|
|
287
|
+
continue
|
|
288
|
+
if isinstance(value, str):
|
|
289
|
+
yield field_path, value, role
|
|
290
|
+
elif isinstance(value, (list, tuple)):
|
|
291
|
+
for i, v in enumerate(value):
|
|
292
|
+
yield f"{field_path}[{i}]", v, role
|
|
293
|
+
elif isinstance(value, dict):
|
|
294
|
+
for k in value:
|
|
295
|
+
yield f"{field_path}[{k!r}]", k, role
|
|
296
|
+
elif isinstance(value, BaseModel):
|
|
297
|
+
yield from column_references(value, f"{field_path}.")
|
|
298
|
+
elif isinstance(value, (list, tuple)):
|
|
299
|
+
for i, v in enumerate(value):
|
|
300
|
+
if isinstance(v, BaseModel):
|
|
301
|
+
yield from column_references(v, f"{field_path}[{i}].")
|
|
302
|
+
elif isinstance(value, dict):
|
|
303
|
+
for k, v in value.items():
|
|
304
|
+
if isinstance(v, BaseModel):
|
|
305
|
+
yield from column_references(v, f"{field_path}[{k!r}].")
|
flim/core/execution.py
ADDED
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
"""Execution options: *where and how* tasks run, kept out of the tool configs."""
|
|
2
|
+
|
|
3
|
+
import importlib.util
|
|
4
|
+
import logging
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from typing import Any, Literal, Optional
|
|
7
|
+
|
|
8
|
+
from pydantic import Field, model_validator
|
|
9
|
+
|
|
10
|
+
from flim.core.config import BaseConfig
|
|
11
|
+
|
|
12
|
+
# executor names of the command line (-e) and of saved settings before ExecutionOptions
|
|
13
|
+
LEGACY_EXECUTORS = {
|
|
14
|
+
"LocalExecutor": "local",
|
|
15
|
+
"LocalDaskExecutor": "dask-local",
|
|
16
|
+
"DaskExecutor": "dask",
|
|
17
|
+
}
|
|
18
|
+
DASK = ("dask-local", "dask")
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class ExecutionOptions(BaseConfig):
|
|
22
|
+
executor: Literal["local", "dask-local", "dask"] = "local"
|
|
23
|
+
# parallel tasks; None = flim.core.runner.default_max_workers()
|
|
24
|
+
num_workers: Optional[int] = Field(default=None, ge=1)
|
|
25
|
+
scheduler: Literal["threads", "processes"] = "threads" # dask-local only
|
|
26
|
+
address: Optional[str] = None # dask only: scheduler of an existing cluster
|
|
27
|
+
dask_kwargs: dict[str, Any] = {} # further DaskTaskRunner / cluster arguments
|
|
28
|
+
autosave: bool = False
|
|
29
|
+
results_dir: Optional[Path] = None
|
|
30
|
+
task_tags: list[str] = []
|
|
31
|
+
device_policy: Literal["prefer", "cpu-only"] = "prefer"
|
|
32
|
+
|
|
33
|
+
@model_validator(mode="after")
|
|
34
|
+
def _check_executor_args(self):
|
|
35
|
+
if self.address is not None and self.executor != "dask":
|
|
36
|
+
raise ValueError("address is only used by the dask executor")
|
|
37
|
+
if self.dask_kwargs and self.executor not in DASK:
|
|
38
|
+
raise ValueError("dask_kwargs are only used by the dask executors")
|
|
39
|
+
if self.autosave and self.results_dir is None:
|
|
40
|
+
raise ValueError("autosave needs a results_dir")
|
|
41
|
+
return self
|
|
42
|
+
|
|
43
|
+
@classmethod
|
|
44
|
+
def from_legacy(cls, executor: str = "LocalExecutor", execargs: str = "", **kwargs):
|
|
45
|
+
"""Options from the command line's ``-e`` executor name and ``"k=v,k=v"`` args.
|
|
46
|
+
|
|
47
|
+
As before, an unknown executor runs locally and the local executor uses only an
|
|
48
|
+
integer ``num_workers``; for Dask, unknown args go to the Dask task runner.
|
|
49
|
+
"""
|
|
50
|
+
args = {}
|
|
51
|
+
for item in (execargs or "").split(","):
|
|
52
|
+
parts = item.split("=")
|
|
53
|
+
if len(parts) != 2:
|
|
54
|
+
continue
|
|
55
|
+
key, value = parts[0].strip(), parts[1].strip()
|
|
56
|
+
try:
|
|
57
|
+
value = int(value)
|
|
58
|
+
except ValueError:
|
|
59
|
+
pass
|
|
60
|
+
args[key] = value
|
|
61
|
+
name = LEGACY_EXECUTORS.get(executor, "local")
|
|
62
|
+
if executor not in LEGACY_EXECUTORS:
|
|
63
|
+
logging.warning(f"Unknown executor {executor}, using LocalExecutor")
|
|
64
|
+
num_workers = args.pop("num_workers", None)
|
|
65
|
+
if not isinstance(num_workers, int) or num_workers < 1:
|
|
66
|
+
num_workers = None
|
|
67
|
+
options = {"executor": name, "num_workers": num_workers, **kwargs}
|
|
68
|
+
if name == "dask-local" and "scheduler" in args:
|
|
69
|
+
scheduler = args.pop("scheduler")
|
|
70
|
+
options["scheduler"] = (
|
|
71
|
+
"processes" if scheduler == "processes" else "threads"
|
|
72
|
+
)
|
|
73
|
+
if name == "dask" and "address" in args:
|
|
74
|
+
options["address"] = str(args.pop("address"))
|
|
75
|
+
if name in DASK:
|
|
76
|
+
options["dask_kwargs"] = args
|
|
77
|
+
return cls(**options)
|
|
78
|
+
|
|
79
|
+
def check_available(self) -> None:
|
|
80
|
+
"""Raises RuntimeError if the executor needs packages that are not installed."""
|
|
81
|
+
if self.executor in DASK and importlib.util.find_spec("prefect_dask") is None:
|
|
82
|
+
raise RuntimeError(
|
|
83
|
+
f"The {self.executor} executor needs the optional Dask support:"
|
|
84
|
+
" uv sync --extra dask (or pip install flimanalyzer[dask])"
|
|
85
|
+
)
|
|
86
|
+
|
|
87
|
+
def task_runner(self):
|
|
88
|
+
"""The Prefect task runner.
|
|
89
|
+
|
|
90
|
+
``local``: a ``ThreadPoolTaskRunner`` with ``num_workers`` threads.
|
|
91
|
+
``dask-local``: a ``DaskTaskRunner`` starting a local Dask cluster.
|
|
92
|
+
``dask``: a ``DaskTaskRunner`` connecting to the cluster at ``address`` (or
|
|
93
|
+
starting a local one if no address is given).
|
|
94
|
+
"""
|
|
95
|
+
if self.executor == "local":
|
|
96
|
+
from flim.core.runner import default_task_runner
|
|
97
|
+
|
|
98
|
+
return default_task_runner(self.num_workers)
|
|
99
|
+
self.check_available()
|
|
100
|
+
from prefect_dask import DaskTaskRunner
|
|
101
|
+
|
|
102
|
+
if self.executor == "dask":
|
|
103
|
+
args = dict(self.dask_kwargs)
|
|
104
|
+
if self.address is not None:
|
|
105
|
+
args["address"] = self.address
|
|
106
|
+
return DaskTaskRunner(**args)
|
|
107
|
+
cluster_kwargs = {"processes": self.scheduler == "processes"}
|
|
108
|
+
if self.num_workers is not None:
|
|
109
|
+
cluster_kwargs["n_workers"] = self.num_workers
|
|
110
|
+
return DaskTaskRunner(cluster_kwargs={**cluster_kwargs, **self.dask_kwargs})
|