aisc-plugin-interface 0.2.4__tar.gz → 0.2.5__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {aisc_plugin_interface-0.2.4 → aisc_plugin_interface-0.2.5}/PKG-INFO +3 -2
- {aisc_plugin_interface-0.2.4 → aisc_plugin_interface-0.2.5}/pyproject.toml +5 -6
- aisc_plugin_interface-0.2.5/src/aisc_plugin_interface/__init__.py +24 -0
- aisc_plugin_interface-0.2.5/src/aisc_plugin_interface/base_evaluation_plugin.py +444 -0
- aisc_plugin_interface-0.2.5/src/aisc_plugin_interface/cli.py +143 -0
- aisc_plugin_interface-0.2.5/src/aisc_plugin_interface/decorators/__init__.py +0 -0
- aisc_plugin_interface-0.2.5/src/aisc_plugin_interface/decorators/evaluation_input.py +24 -0
- aisc_plugin_interface-0.2.5/src/aisc_plugin_interface/decorators/metric.py +12 -0
- aisc_plugin_interface-0.2.5/src/aisc_plugin_interface/input_providers/__init__.py +0 -0
- aisc_plugin_interface-0.2.5/src/aisc_plugin_interface/input_providers/base_input_provider.py +31 -0
- aisc_plugin_interface-0.2.5/src/aisc_plugin_interface/input_providers/csv_input_provider.py +20 -0
- aisc_plugin_interface-0.2.5/src/aisc_plugin_interface/models/__init__.py +0 -0
- aisc_plugin_interface-0.2.5/src/aisc_plugin_interface/models/evaluation_input.py +14 -0
- aisc_plugin_interface-0.2.5/src/aisc_plugin_interface/models/measure.py +30 -0
- aisc_plugin_interface-0.2.5/src/aisc_plugin_interface/models/task.py +8 -0
- aisc_plugin_interface-0.2.5/src/aisc_plugin_interface/utils.py +43 -0
- {aisc_plugin_interface-0.2.4 → aisc_plugin_interface-0.2.5}/.gitignore +0 -0
- {aisc_plugin_interface-0.2.4 → aisc_plugin_interface-0.2.5}/README.md +0 -0
- {aisc_plugin_interface-0.2.4 → aisc_plugin_interface-0.2.5}/src/aisc_plugin_interface/templates/init.py.tpl +0 -0
- {aisc_plugin_interface-0.2.4 → aisc_plugin_interface-0.2.5}/src/aisc_plugin_interface/templates/plugin.py.tpl +0 -0
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: aisc-plugin-interface
|
|
3
|
-
Version: 0.2.
|
|
4
|
-
Summary:
|
|
3
|
+
Version: 0.2.5
|
|
4
|
+
Summary: Plugin interface for the AISC evaluation framework
|
|
5
|
+
Author-email: Sean Blevins <sean.blevins@uni.lu>
|
|
5
6
|
Requires-Python: >=3.12
|
|
6
7
|
Requires-Dist: pydantic>=2.12.5
|
|
7
8
|
Requires-Dist: rich>=14.3.3
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "aisc-plugin-interface"
|
|
3
|
-
version = "0.2.
|
|
4
|
-
description = "
|
|
3
|
+
version = "0.2.5"
|
|
4
|
+
description = "Plugin interface for the AISC evaluation framework"
|
|
5
|
+
authors = [{ name = "Sean Blevins", email = "sean.blevins@uni.lu" }]
|
|
5
6
|
readme = "README.md"
|
|
6
7
|
requires-python = ">=3.12"
|
|
7
8
|
dependencies = [
|
|
@@ -16,10 +17,8 @@ build-backend = "hatchling.build"
|
|
|
16
17
|
[tool.hatch.build.targets.wheel]
|
|
17
18
|
packages = ["src/aisc_plugin_interface"]
|
|
18
19
|
|
|
19
|
-
[tool.hatch.build]
|
|
20
|
-
include = [
|
|
21
|
-
"src/aisc_plugin_interface/templates/*.tpl",
|
|
22
|
-
]
|
|
20
|
+
[tool.hatch.build.targets.sdist]
|
|
21
|
+
include = ["src/aisc_plugin_interface"]
|
|
23
22
|
|
|
24
23
|
[project.scripts]
|
|
25
24
|
aisc-plugin-interface = "aisc_plugin_interface.cli:main"
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
from aisc_plugin_interface.base_evaluation_plugin import (
|
|
2
|
+
BaseEvaluationPlugin,
|
|
3
|
+
PluginFeatureFlags,
|
|
4
|
+
)
|
|
5
|
+
from aisc_plugin_interface.input_providers.base_input_provider import BaseInputProvider
|
|
6
|
+
from aisc_plugin_interface.decorators.metric import metric
|
|
7
|
+
from aisc_plugin_interface.decorators.evaluation_input import evaluation_input
|
|
8
|
+
from aisc_plugin_interface.models.measure import Measure, MetricVisualization, ChartType
|
|
9
|
+
from aisc_plugin_interface.models.evaluation_input import InputDefinition, InputType
|
|
10
|
+
from aisc_plugin_interface.models.task import TaskProgress
|
|
11
|
+
|
|
12
|
+
__all__ = [
|
|
13
|
+
"BaseEvaluationPlugin",
|
|
14
|
+
"PluginFeatureFlags",
|
|
15
|
+
"BaseInputProvider",
|
|
16
|
+
"metric",
|
|
17
|
+
"evaluation_input",
|
|
18
|
+
"Measure",
|
|
19
|
+
"MetricVisualization",
|
|
20
|
+
"ChartType",
|
|
21
|
+
"InputDefinition",
|
|
22
|
+
"InputType",
|
|
23
|
+
"TaskProgress",
|
|
24
|
+
]
|
|
@@ -0,0 +1,444 @@
|
|
|
1
|
+
import copy
|
|
2
|
+
import inspect
|
|
3
|
+
import logging
|
|
4
|
+
from abc import ABC, abstractmethod
|
|
5
|
+
from collections.abc import Iterable, Sized
|
|
6
|
+
from typing import (
|
|
7
|
+
Any,
|
|
8
|
+
get_args,
|
|
9
|
+
get_origin,
|
|
10
|
+
Callable,
|
|
11
|
+
Tuple,
|
|
12
|
+
TypeAlias,
|
|
13
|
+
TypeVar,
|
|
14
|
+
Generic,
|
|
15
|
+
final,
|
|
16
|
+
Type,
|
|
17
|
+
Self,
|
|
18
|
+
)
|
|
19
|
+
|
|
20
|
+
from pydantic import BaseModel, Field
|
|
21
|
+
|
|
22
|
+
from aisc_plugin_interface.decorators.evaluation_input import InputDefinition
|
|
23
|
+
from aisc_plugin_interface.utils import classproperty
|
|
24
|
+
from aisc_plugin_interface.models.task import TaskProgress
|
|
25
|
+
from aisc_plugin_interface.input_providers.base_input_provider import BaseInputProvider
|
|
26
|
+
from aisc_plugin_interface.models.measure import Measure, MetricVisualization, ChartType
|
|
27
|
+
|
|
28
|
+
ProgressCallback: TypeAlias = Callable[[TaskProgress], None]
|
|
29
|
+
ArtifactCallback: TypeAlias = Callable[[str, bytes], None]
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class PluginFeatureFlags(BaseModel):
|
|
33
|
+
can_parse_config_from_dataset: bool = Field(
|
|
34
|
+
False, description="Show the dataset dropdown"
|
|
35
|
+
)
|
|
36
|
+
show_dimensions_visualisation: bool = Field(
|
|
37
|
+
False, description="Show the dimensions visualisation on the results page"
|
|
38
|
+
)
|
|
39
|
+
extra: dict = Field({}, description="Additional feature flags")
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
class BaseEvaluationPlugin[T: BaseModel](ABC):
|
|
43
|
+
"""
|
|
44
|
+
Abstract Base Class for evaluation plugins.
|
|
45
|
+
Plugins should inherit from this class and provide a Pydantic model for their configuration.
|
|
46
|
+
|
|
47
|
+
Example:
|
|
48
|
+
class MyPlugin(BaseEvaluationPlugin[MyConfigModel]):
|
|
49
|
+
...
|
|
50
|
+
"""
|
|
51
|
+
|
|
52
|
+
# UI Schema for RJSF (react-jsonschema-form) to customize form appearance
|
|
53
|
+
form_ui_schema: dict = {}
|
|
54
|
+
|
|
55
|
+
# The plugin name that will be displayed in the UI
|
|
56
|
+
plugin_name: str | None = None
|
|
57
|
+
|
|
58
|
+
# Controls the icon displayed in the plugin list.
|
|
59
|
+
# Use a Material Design icon name https://fonts.google.com/icons
|
|
60
|
+
ui_icon: str | None = None
|
|
61
|
+
|
|
62
|
+
_input_definitions: list[InputDefinition] = []
|
|
63
|
+
_input_provider_types: dict[str, Type[BaseInputProvider]] = {}
|
|
64
|
+
|
|
65
|
+
def __init__(self):
|
|
66
|
+
self._input_provider_instances: dict[str, BaseInputProvider] = {}
|
|
67
|
+
self._progress_callback: ProgressCallback | None = None
|
|
68
|
+
self._artifact_callback: ArtifactCallback | None = None
|
|
69
|
+
self._logger = None
|
|
70
|
+
|
|
71
|
+
@classmethod
|
|
72
|
+
def is_direct_subclass(cls: type[Self]) -> bool:
|
|
73
|
+
return BaseEvaluationPlugin in cls.__bases__
|
|
74
|
+
|
|
75
|
+
@classproperty
|
|
76
|
+
def display_name(cls: type[Self]) -> str:
|
|
77
|
+
"""
|
|
78
|
+
Returns a display name for the plugin.
|
|
79
|
+
|
|
80
|
+
If the class defines a `plugin_name` attribute, its value is returned.
|
|
81
|
+
If not, the class's name (`cls.__name__`) is used as a fallback.
|
|
82
|
+
"""
|
|
83
|
+
# NOTE: use cls.plugin_name only if cls is direct subclass of BaseEvaluationPlugin
|
|
84
|
+
plugin_name = cls.plugin_name
|
|
85
|
+
default = cls.__name__
|
|
86
|
+
|
|
87
|
+
if cls.is_direct_subclass():
|
|
88
|
+
# cls is a direct subclass of BaseInputProvider
|
|
89
|
+
return plugin_name or default
|
|
90
|
+
else:
|
|
91
|
+
if plugin_name is None:
|
|
92
|
+
return default
|
|
93
|
+
# avoid using the plugin name that was set for the parent plugin
|
|
94
|
+
for base in cls.__bases__:
|
|
95
|
+
if (
|
|
96
|
+
issubclass(base, BaseEvaluationPlugin)
|
|
97
|
+
and plugin_name == base.plugin_name
|
|
98
|
+
):
|
|
99
|
+
return default
|
|
100
|
+
return plugin_name
|
|
101
|
+
|
|
102
|
+
@property
|
|
103
|
+
def logger(self):
|
|
104
|
+
"""
|
|
105
|
+
Returns the cached logger for this plugin class.
|
|
106
|
+
|
|
107
|
+
The logger is shared across all instances of the class (per-class caching)
|
|
108
|
+
and named as "<module> - __name__".
|
|
109
|
+
"""
|
|
110
|
+
if self._logger is None:
|
|
111
|
+
cls = self.__class__
|
|
112
|
+
self._logger = logging.getLogger(f"{cls.__module__} - {cls.__name__}")
|
|
113
|
+
return self._logger
|
|
114
|
+
|
|
115
|
+
@property
|
|
116
|
+
def feature_flags(self) -> PluginFeatureFlags:
|
|
117
|
+
"""
|
|
118
|
+
Controls UI behavior on the frontend.
|
|
119
|
+
Override this property in your subclass to change defaults.
|
|
120
|
+
"""
|
|
121
|
+
return PluginFeatureFlags()
|
|
122
|
+
|
|
123
|
+
@property
|
|
124
|
+
def input_definitions(self) -> list[InputDefinition]:
|
|
125
|
+
"""
|
|
126
|
+
Controls the evaluation input form at evaluation creation
|
|
127
|
+
"""
|
|
128
|
+
return self._input_definitions
|
|
129
|
+
|
|
130
|
+
@property
|
|
131
|
+
def display_icon(self) -> str:
|
|
132
|
+
"""
|
|
133
|
+
Controls the icon displayed in the plugin list.
|
|
134
|
+
Use a Material Design icon name
|
|
135
|
+
https://fonts.google.com/icons
|
|
136
|
+
|
|
137
|
+
For a new plugin, either redefine this property, or set the class attribute `ui_icon`
|
|
138
|
+
"""
|
|
139
|
+
return self.ui_icon or "extension"
|
|
140
|
+
|
|
141
|
+
@property
|
|
142
|
+
def config_type(self) -> type[T]:
|
|
143
|
+
"""
|
|
144
|
+
Retrieves the Pydantic model type used for plugin configuration.
|
|
145
|
+
"""
|
|
146
|
+
for base in getattr(self.__class__, "__orig_bases__", []):
|
|
147
|
+
if get_origin(base) is BaseEvaluationPlugin:
|
|
148
|
+
return get_args(base)[0]
|
|
149
|
+
raise TypeError("Could not determine Config type T")
|
|
150
|
+
|
|
151
|
+
def get_metrics(self) -> list[str]:
|
|
152
|
+
"""
|
|
153
|
+
Returns a list of all metric names defined in this plugin via the @metric decorator.
|
|
154
|
+
"""
|
|
155
|
+
metrics = []
|
|
156
|
+
for name, method in inspect.getmembers(self, predicate=inspect.ismethod):
|
|
157
|
+
if hasattr(method, "metric_name"):
|
|
158
|
+
metrics.append(method.metric_name)
|
|
159
|
+
return metrics
|
|
160
|
+
|
|
161
|
+
def get_metric_visualizations(self, config_data: dict) -> list[MetricVisualization]:
|
|
162
|
+
"""
|
|
163
|
+
Returns a list of MetricVisualization objects to render a list of
|
|
164
|
+
visualizations on the front end and the metrics to display for each
|
|
165
|
+
|
|
166
|
+
The config_data could be used to define specific visualizations from the config
|
|
167
|
+
|
|
168
|
+
By default, returns a single visualization (TABLE) with all metrics
|
|
169
|
+
"""
|
|
170
|
+
return [
|
|
171
|
+
MetricVisualization(chart_type=ChartType.TABLE, metrics=self.get_metrics())
|
|
172
|
+
]
|
|
173
|
+
|
|
174
|
+
def export_metrics(self, *args, **kwargs) -> list[Measure]:
|
|
175
|
+
"""
|
|
176
|
+
Executes all methods decorated with @metric and aggregates their results.
|
|
177
|
+
"""
|
|
178
|
+
results: list[Measure] = []
|
|
179
|
+
for name, method in inspect.getmembers(self, predicate=inspect.ismethod):
|
|
180
|
+
if hasattr(method, "metric_name"):
|
|
181
|
+
metric_measures: list[Measure] = method(*args, **kwargs)
|
|
182
|
+
results.extend(metric_measures)
|
|
183
|
+
return results
|
|
184
|
+
|
|
185
|
+
@abstractmethod
|
|
186
|
+
def evaluate(self, config_data: dict) -> Any:
|
|
187
|
+
"""
|
|
188
|
+
The main execution logic of the plugin.
|
|
189
|
+
Developers should process the dataset/model here and return an intermediate result
|
|
190
|
+
that will be passed to the metric methods.
|
|
191
|
+
"""
|
|
192
|
+
raise NotImplementedError
|
|
193
|
+
|
|
194
|
+
@final
|
|
195
|
+
def _set_progress_callback(
|
|
196
|
+
self, progress_callback: ProgressCallback | None
|
|
197
|
+
) -> None:
|
|
198
|
+
"""
|
|
199
|
+
Internal: called by the evaluation runtime (eval module) to feedback progress reporting.
|
|
200
|
+
Plugin implementations should not call or override this.
|
|
201
|
+
"""
|
|
202
|
+
if progress_callback is not None and not callable(progress_callback):
|
|
203
|
+
raise TypeError("Progress sink must be callable or None")
|
|
204
|
+
self._progress_callback = progress_callback
|
|
205
|
+
|
|
206
|
+
@final
|
|
207
|
+
def report_progress(self, task_progress: TaskProgress) -> None:
|
|
208
|
+
"""
|
|
209
|
+
Public, stable API for plugin authors.
|
|
210
|
+
No-op if no sink is configured by the evaluation runtime.
|
|
211
|
+
"""
|
|
212
|
+
if self._progress_callback is None:
|
|
213
|
+
return
|
|
214
|
+
self._progress_callback(task_progress)
|
|
215
|
+
|
|
216
|
+
@final
|
|
217
|
+
def progress_bar(
|
|
218
|
+
self,
|
|
219
|
+
iterable,
|
|
220
|
+
*,
|
|
221
|
+
total: int | None = None,
|
|
222
|
+
desc: str | None = None,
|
|
223
|
+
start: int = 0,
|
|
224
|
+
with_index: bool = False,
|
|
225
|
+
extra: dict | None = None,
|
|
226
|
+
) -> "ProgressBar":
|
|
227
|
+
"""
|
|
228
|
+
Wrap an iterable to yield items (optionally with index)
|
|
229
|
+
while emitting progress for each item that is processed.
|
|
230
|
+
|
|
231
|
+
Returns a ProgressBar yielding items (or index, item) with automatic progress reporting.
|
|
232
|
+
|
|
233
|
+
Args:
|
|
234
|
+
iterable: Items to iterate.
|
|
235
|
+
total: Optional total length.
|
|
236
|
+
desc: Label.
|
|
237
|
+
start: Enumeration start.
|
|
238
|
+
with_index: Yield index with items.
|
|
239
|
+
extra: Extra progress fields.
|
|
240
|
+
"""
|
|
241
|
+
return ProgressBar(
|
|
242
|
+
iterable=iterable,
|
|
243
|
+
plugin=self,
|
|
244
|
+
total=total,
|
|
245
|
+
desc=desc,
|
|
246
|
+
start=start,
|
|
247
|
+
with_index=with_index,
|
|
248
|
+
extra=extra,
|
|
249
|
+
)
|
|
250
|
+
|
|
251
|
+
@final
|
|
252
|
+
def _set_artifact_callback(
|
|
253
|
+
self, artifact_callback: ArtifactCallback | None
|
|
254
|
+
) -> None:
|
|
255
|
+
"""Internal: called by the evaluation runtime to hook into artifact uploading."""
|
|
256
|
+
self._artifact_callback = artifact_callback
|
|
257
|
+
|
|
258
|
+
@final
|
|
259
|
+
def upload_artifact(self, name: str, content: bytes) -> None:
|
|
260
|
+
"""
|
|
261
|
+
Public API for plugin authors to upload arbitrary files.
|
|
262
|
+
"""
|
|
263
|
+
if self._artifact_callback:
|
|
264
|
+
self._artifact_callback(name, content)
|
|
265
|
+
else:
|
|
266
|
+
self.logger.warning(
|
|
267
|
+
f"Artifact callback not configured. Dropping artifact: {name}"
|
|
268
|
+
)
|
|
269
|
+
|
|
270
|
+
def set_input_content(self, name: str, file_content: bytes | None) -> None:
|
|
271
|
+
"""
|
|
272
|
+
Called by the runtime. Instantiates the provider mapped via @input.
|
|
273
|
+
"""
|
|
274
|
+
provider_cls = self._input_provider_types.get(name)
|
|
275
|
+
if provider_cls and file_content is not None:
|
|
276
|
+
self._input_provider_instances[name] = provider_cls(file_content)
|
|
277
|
+
|
|
278
|
+
def get_input_data(self, name: str) -> Any | None:
|
|
279
|
+
"""
|
|
280
|
+
Get data from InputProvider using name set in evaluation_input decorator
|
|
281
|
+
"""
|
|
282
|
+
provider = self._input_provider_instances.get(name)
|
|
283
|
+
if provider is None:
|
|
284
|
+
return None
|
|
285
|
+
return provider.get_data()
|
|
286
|
+
|
|
287
|
+
def get_config_form_schema(self) -> dict:
|
|
288
|
+
"""
|
|
289
|
+
Generates a JSON Schema from the Pydantic config model for the frontend UI.
|
|
290
|
+
"""
|
|
291
|
+
return self.config_type.model_json_schema(mode="validation")
|
|
292
|
+
|
|
293
|
+
def validate_config_form_data(self, config_form_data: dict) -> T:
|
|
294
|
+
"""
|
|
295
|
+
Validates incoming form data from frontend UI/backend DB against the Pydantic config model.
|
|
296
|
+
"""
|
|
297
|
+
return self.config_type.model_validate(config_form_data)
|
|
298
|
+
|
|
299
|
+
def get_config_form_ui_schema(self) -> dict:
|
|
300
|
+
"""
|
|
301
|
+
Returns a deep copy of the UI schema for form customization.
|
|
302
|
+
"""
|
|
303
|
+
return copy.deepcopy(self.form_ui_schema)
|
|
304
|
+
|
|
305
|
+
def form_schema_to_internal(self, form_schema: T) -> dict:
|
|
306
|
+
"""
|
|
307
|
+
Optional: Converts the validated Pydantic model used for the UI form
|
|
308
|
+
into a dict for internal use
|
|
309
|
+
This can be overridden to add/change the structure of the input config data
|
|
310
|
+
for use in the evaluate method
|
|
311
|
+
"""
|
|
312
|
+
return form_schema.model_dump()
|
|
313
|
+
|
|
314
|
+
def get_full_schema(self) -> Tuple[dict, dict]:
|
|
315
|
+
"""Helper to get the fresh, static baseline."""
|
|
316
|
+
return self.get_config_form_schema(), self.get_config_form_ui_schema()
|
|
317
|
+
|
|
318
|
+
# form_data passed here may be incomplete, so we don't validate and use MyConfigModel
|
|
319
|
+
# It is the developer's responsibility to check for and use data accordingly here
|
|
320
|
+
def on_config_change(self, form_data: T | None) -> Tuple[T | None, dict, dict]:
|
|
321
|
+
"""
|
|
322
|
+
Hook called whenever the user changes a form value.
|
|
323
|
+
Allows the plugin to dynamically update the schema (e.g. drop downs),
|
|
324
|
+
the data (e.g. auto-fill), or the UI (e.g. hide fields).
|
|
325
|
+
"""
|
|
326
|
+
# Default: Do nothing, just return what came in
|
|
327
|
+
schema, ui_schema = self.get_full_schema()
|
|
328
|
+
return form_data, schema, ui_schema
|
|
329
|
+
|
|
330
|
+
def parse_config_from_dataset(self, file_content: bytes) -> dict | None:
|
|
331
|
+
"""
|
|
332
|
+
Optional: Try to parse a valid config from the dataset.
|
|
333
|
+
Use a InputProvider to read the file contents
|
|
334
|
+
"""
|
|
335
|
+
return None
|
|
336
|
+
|
|
337
|
+
|
|
338
|
+
Item = TypeVar("Item")
|
|
339
|
+
|
|
340
|
+
|
|
341
|
+
class ProgressBar(Generic[Item]):
|
|
342
|
+
"""
|
|
343
|
+
Iterator that wraps an iterable, yielding each item (and optionally its index)
|
|
344
|
+
and reporting progress via a plugin.
|
|
345
|
+
|
|
346
|
+
Each emit reports the progress for the current item that is processed.
|
|
347
|
+
|
|
348
|
+
Args:
|
|
349
|
+
iterable: Items to iterate over.
|
|
350
|
+
plugin: Receives progress reports (must implement `report_progress`).
|
|
351
|
+
total: Length of iteration (inferred if not given).
|
|
352
|
+
desc: Progress label.
|
|
353
|
+
start: Enumeration start index (min 1 if negative).
|
|
354
|
+
with_index: Yield (index, item) pairs if True.
|
|
355
|
+
extra: Extra fields for progress payloads.
|
|
356
|
+
"""
|
|
357
|
+
|
|
358
|
+
def __init__(
|
|
359
|
+
self,
|
|
360
|
+
iterable: Iterable[Item],
|
|
361
|
+
*,
|
|
362
|
+
plugin: BaseEvaluationPlugin,
|
|
363
|
+
total: int | None = None,
|
|
364
|
+
desc: str | None = None,
|
|
365
|
+
start: int = 0,
|
|
366
|
+
with_index: bool = False,
|
|
367
|
+
extra: dict | None = None,
|
|
368
|
+
):
|
|
369
|
+
self._start = start
|
|
370
|
+
self._with_index = with_index
|
|
371
|
+
|
|
372
|
+
# infer total if not provided
|
|
373
|
+
if total is None:
|
|
374
|
+
if isinstance(iterable, Sized):
|
|
375
|
+
total = len(iterable)
|
|
376
|
+
else:
|
|
377
|
+
iterable = list(iterable)
|
|
378
|
+
total = len(iterable)
|
|
379
|
+
self._total = total
|
|
380
|
+
|
|
381
|
+
self._desc = desc or ""
|
|
382
|
+
self._extra = {} if extra is None else extra.copy()
|
|
383
|
+
|
|
384
|
+
self._report_progress = getattr(plugin, "report_progress", None)
|
|
385
|
+
|
|
386
|
+
# prepare enumerate on iterable with correct start index
|
|
387
|
+
self._enumerated_iter = enumerate(iterable, start=self._start)
|
|
388
|
+
|
|
389
|
+
# start emitting at 0
|
|
390
|
+
self.emit(0)
|
|
391
|
+
|
|
392
|
+
@property
|
|
393
|
+
def desc(self) -> str:
|
|
394
|
+
return self._desc
|
|
395
|
+
|
|
396
|
+
def set_description(self, description: str):
|
|
397
|
+
self._desc = description
|
|
398
|
+
|
|
399
|
+
@property
|
|
400
|
+
def extra(self) -> dict:
|
|
401
|
+
return self._extra
|
|
402
|
+
|
|
403
|
+
def set_extra(self, fields: dict) -> None:
|
|
404
|
+
self._extra.update(fields)
|
|
405
|
+
|
|
406
|
+
def emit(self, i: int):
|
|
407
|
+
"""
|
|
408
|
+
Report progress for index `i` using the plugin.
|
|
409
|
+
"""
|
|
410
|
+
if self._report_progress and self._total:
|
|
411
|
+
progress = min(float(min(i, self._total)) / self._total, 1.0)
|
|
412
|
+
payload = {
|
|
413
|
+
"iteration": int(i),
|
|
414
|
+
"total": self._total,
|
|
415
|
+
}
|
|
416
|
+
if self._desc:
|
|
417
|
+
payload["desc"] = self._desc
|
|
418
|
+
if self._extra:
|
|
419
|
+
payload.update(self._extra)
|
|
420
|
+
self._report_progress(
|
|
421
|
+
TaskProgress(
|
|
422
|
+
progress=progress,
|
|
423
|
+
extra=payload,
|
|
424
|
+
)
|
|
425
|
+
)
|
|
426
|
+
|
|
427
|
+
def __len__(self) -> int:
|
|
428
|
+
return self._total
|
|
429
|
+
|
|
430
|
+
def __iter__(self) -> Self:
|
|
431
|
+
return self
|
|
432
|
+
|
|
433
|
+
def __next__(self) -> Item | tuple[int, Item]:
|
|
434
|
+
"""
|
|
435
|
+
Yield the next item (and index if requested),
|
|
436
|
+
emitting progress for the current item being processed.
|
|
437
|
+
|
|
438
|
+
Raises:
|
|
439
|
+
StopIteration when exhausted.
|
|
440
|
+
"""
|
|
441
|
+
idx, item = next(self._enumerated_iter)
|
|
442
|
+
|
|
443
|
+
self.emit(idx - self._start + 1)
|
|
444
|
+
return (idx, item) if self._with_index else item
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
import argparse
|
|
2
|
+
import tomllib
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
from importlib.resources import files
|
|
5
|
+
|
|
6
|
+
from rich.console import Console
|
|
7
|
+
from rich.prompt import Prompt
|
|
8
|
+
|
|
9
|
+
console = Console()
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def _prompt(text, default):
|
|
13
|
+
return Prompt.ask(f"[bold cyan]{text}[/]", console=console, default=default)
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def _get_package_name(pyproject: Path):
|
|
17
|
+
data = tomllib.loads(pyproject.read_text())
|
|
18
|
+
return data["project"]["name"].replace("-", "_")
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _render(template_name: str, context: dict) -> str:
|
|
22
|
+
template_file = files("aisc_plugin_interface").joinpath("templates", template_name)
|
|
23
|
+
template = template_file.read_text(encoding="utf-8")
|
|
24
|
+
for key, value in context.items():
|
|
25
|
+
template = template.replace(f"{{{{ {key} }}}}", value)
|
|
26
|
+
return template
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _init_or_update_init_file(src_pkg, context) -> bool:
|
|
30
|
+
init_file = src_pkg / "__init__.py"
|
|
31
|
+
init_content = _render("init.py.tpl", context)
|
|
32
|
+
|
|
33
|
+
if init_file.exists():
|
|
34
|
+
existing = init_file.read_text(encoding="utf-8")
|
|
35
|
+
|
|
36
|
+
# check if import line already exists
|
|
37
|
+
import_line = f"from .{context['import_path']} import {context['plugin_name']}"
|
|
38
|
+
if import_line not in existing:
|
|
39
|
+
# append the new import
|
|
40
|
+
existing = f"{import_line}\n" + existing.rstrip()
|
|
41
|
+
else:
|
|
42
|
+
console.print(
|
|
43
|
+
f"[red]❌ Import already in {str(init_file)}: '{import_line}'[/]"
|
|
44
|
+
)
|
|
45
|
+
return False
|
|
46
|
+
|
|
47
|
+
# update __all__
|
|
48
|
+
if "__all__" in existing:
|
|
49
|
+
# find current __all__ list and append the class
|
|
50
|
+
import re
|
|
51
|
+
|
|
52
|
+
pattern = r"__all__\s*=\s*\[([^\]]*)\]"
|
|
53
|
+
match = re.search(pattern, existing)
|
|
54
|
+
if match:
|
|
55
|
+
classes = match.group(1).split(",")
|
|
56
|
+
classes = [c.strip(" '\"") for c in classes]
|
|
57
|
+
if context["plugin_name"] not in classes:
|
|
58
|
+
classes.append(context["plugin_name"])
|
|
59
|
+
else:
|
|
60
|
+
console.print(
|
|
61
|
+
f"[red]❌ A plugin with the same name '{context['plugin_name']}' "
|
|
62
|
+
"already exists. Abording.[/]"
|
|
63
|
+
)
|
|
64
|
+
return False
|
|
65
|
+
|
|
66
|
+
new_all = ", ".join(f'"{c}"' for c in classes)
|
|
67
|
+
existing = re.sub(pattern, f"__all__ = [{new_all}]", existing)
|
|
68
|
+
else:
|
|
69
|
+
# __all__ does not exist, create it
|
|
70
|
+
existing += f"\n__all__ = ['{context['plugin_name']}']\n"
|
|
71
|
+
|
|
72
|
+
init_file.write_text(existing, encoding="utf-8")
|
|
73
|
+
console.print(f"[green]✅ Updated __init__.py with {context['plugin_name']}[/]")
|
|
74
|
+
else:
|
|
75
|
+
# __init__.py does not exist, create it from template
|
|
76
|
+
init_file.write_text(init_content, encoding="utf-8")
|
|
77
|
+
console.print(f"[green]✅ Created __init__.py with {context['plugin_name']}[/]")
|
|
78
|
+
|
|
79
|
+
return True
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def init_plugin(force=False):
|
|
83
|
+
root = Path.cwd()
|
|
84
|
+
pyproject = root / "pyproject.toml"
|
|
85
|
+
|
|
86
|
+
if not pyproject.exists():
|
|
87
|
+
console.print("[red]❌ No pyproject.toml found. Run inside a project.[/]")
|
|
88
|
+
return
|
|
89
|
+
|
|
90
|
+
package_name = _get_package_name(pyproject)
|
|
91
|
+
src_pkg = root / "src" / package_name
|
|
92
|
+
|
|
93
|
+
plugin_name = _prompt("Plugin class name", "Plugin")
|
|
94
|
+
plugin_path_input = _prompt(
|
|
95
|
+
"Plugin path (path/to/file, .py added automatically if omitted)", "plugin"
|
|
96
|
+
)
|
|
97
|
+
|
|
98
|
+
# remove .py if user included it
|
|
99
|
+
if plugin_path_input.endswith(".py"):
|
|
100
|
+
plugin_path_input = plugin_path_input[:-3]
|
|
101
|
+
|
|
102
|
+
parts = [p for p in plugin_path_input.strip("/").split("/") if p]
|
|
103
|
+
|
|
104
|
+
plugin_dir = src_pkg.joinpath(*parts[:-1]) if len(parts) > 1 else src_pkg
|
|
105
|
+
plugin_dir.mkdir(parents=True, exist_ok=True)
|
|
106
|
+
|
|
107
|
+
plugin_file = plugin_dir / f"{parts[-1]}.py"
|
|
108
|
+
import_path = ".".join(parts)
|
|
109
|
+
|
|
110
|
+
context = {
|
|
111
|
+
"import_path": import_path,
|
|
112
|
+
"package_name": package_name,
|
|
113
|
+
"plugin_name": plugin_name,
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
if not plugin_file.exists() or force:
|
|
117
|
+
is_valid_name_and_dest = _init_or_update_init_file(src_pkg, context)
|
|
118
|
+
if is_valid_name_and_dest:
|
|
119
|
+
plugin_file.write_text(_render("plugin.py.tpl", context))
|
|
120
|
+
console.print(
|
|
121
|
+
f"[green]✅ Plugin class '{plugin_name}' template written to file {plugin_file}[/]"
|
|
122
|
+
)
|
|
123
|
+
|
|
124
|
+
console.print(
|
|
125
|
+
f"[bold magenta]🎉 Plugin '{plugin_name}' has been successfully initialized![/]"
|
|
126
|
+
)
|
|
127
|
+
else:
|
|
128
|
+
console.print(f"[red]❌ File already exists: '{plugin_file}'.[/]")
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def main():
|
|
132
|
+
parser = argparse.ArgumentParser(prog="parent-package")
|
|
133
|
+
sub = parser.add_subparsers(dest="command")
|
|
134
|
+
|
|
135
|
+
init_cmd = sub.add_parser("init-plugin")
|
|
136
|
+
init_cmd.add_argument("--force", action="store_true")
|
|
137
|
+
|
|
138
|
+
args = parser.parse_args()
|
|
139
|
+
|
|
140
|
+
if args.command == "init-plugin":
|
|
141
|
+
init_plugin(force=args.force)
|
|
142
|
+
else:
|
|
143
|
+
parser.print_help()
|
|
File without changes
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
from typing import Type
|
|
2
|
+
from aisc_plugin_interface.input_providers.base_input_provider import BaseInputProvider
|
|
3
|
+
from aisc_plugin_interface.models.evaluation_input import InputDefinition, InputType
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def evaluation_input(name: str, label: str, input_provider_class: Type[BaseInputProvider], input_type: InputType, required: bool = True):
|
|
7
|
+
"""
|
|
8
|
+
Decorator to create input definitions and their provider.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
def decorator(cls):
|
|
12
|
+
if "_input_definitions" not in cls.__dict__:
|
|
13
|
+
cls._input_definitions = []
|
|
14
|
+
if "_input_provider_types" not in cls.__dict__:
|
|
15
|
+
cls._input_provider_types = {}
|
|
16
|
+
|
|
17
|
+
if not any(d.name == name for d in cls._input_definitions):
|
|
18
|
+
cls._input_definitions.append(
|
|
19
|
+
InputDefinition(name=name, label=label, input_type=input_type, required=required)
|
|
20
|
+
)
|
|
21
|
+
cls._input_provider_types[name] = input_provider_class
|
|
22
|
+
return cls
|
|
23
|
+
|
|
24
|
+
return decorator
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
from typing import Callable
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
def metric(name: str):
|
|
5
|
+
"""
|
|
6
|
+
Decorator to mark a method as a metric exporter.
|
|
7
|
+
Methods decorated with this should return a list of Measure objects.
|
|
8
|
+
"""
|
|
9
|
+
def decorator(func: Callable):
|
|
10
|
+
func.metric_name = name
|
|
11
|
+
return func
|
|
12
|
+
return decorator
|
|
File without changes
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
from abc import ABC, abstractmethod
|
|
2
|
+
from typing import Any
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
class BaseInputProvider(ABC):
|
|
6
|
+
"""
|
|
7
|
+
Abstract base class for data loaders used by plugins.
|
|
8
|
+
Input providers handle the transformation of raw bytes (from S3/Storage)
|
|
9
|
+
into a structured format that the plugin can process.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
def __init__(self, file_content: bytes):
|
|
13
|
+
"""
|
|
14
|
+
Initializes the provider and triggers the data reading process.
|
|
15
|
+
:param file_content: Raw bytes of the file to be processed.
|
|
16
|
+
"""
|
|
17
|
+
self._data = self._read_data(file_content)
|
|
18
|
+
|
|
19
|
+
@abstractmethod
|
|
20
|
+
def _read_data(self, file_content: bytes) -> Any:
|
|
21
|
+
"""
|
|
22
|
+
Parses raw bytes into an object (e.g., list, dict, DataFrame, ONNX session etc).
|
|
23
|
+
Must be implemented by subclasses.
|
|
24
|
+
"""
|
|
25
|
+
raise NotImplementedError
|
|
26
|
+
|
|
27
|
+
def get_data(self) -> Any:
|
|
28
|
+
"""
|
|
29
|
+
Returns the parsed data stored in the provider.
|
|
30
|
+
"""
|
|
31
|
+
return self._data
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import csv
|
|
2
|
+
import io
|
|
3
|
+
|
|
4
|
+
from .base_input_provider import BaseInputProvider
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class CsvInputProvider(BaseInputProvider):
|
|
8
|
+
"""
|
|
9
|
+
A concrete implementation of BaseInputProvider for CSV files.
|
|
10
|
+
Parses the file content into a list of dictionaries, where each dict represents a row.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
def _read_data(self, file_content) -> list[dict]:
|
|
14
|
+
"""
|
|
15
|
+
Converts CSV bytes into a list of dictionaries.
|
|
16
|
+
"""
|
|
17
|
+
file_stream = io.BytesIO(file_content)
|
|
18
|
+
wrapper = io.TextIOWrapper(file_stream, encoding='utf-8')
|
|
19
|
+
reader = csv.DictReader(wrapper)
|
|
20
|
+
return list(reader)
|
|
File without changes
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
from datetime import datetime
|
|
2
|
+
import enum
|
|
3
|
+
|
|
4
|
+
from pydantic import BaseModel
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class Measure(BaseModel):
|
|
8
|
+
name: str
|
|
9
|
+
description: str | None = None
|
|
10
|
+
unit: str | None = None
|
|
11
|
+
score: float
|
|
12
|
+
time: datetime = datetime.now()
|
|
13
|
+
error: str | None = None
|
|
14
|
+
dimensions: dict[str, str | int | bool] | None = None
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class ChartType(str, enum.Enum):
|
|
18
|
+
TABLE = "table"
|
|
19
|
+
LINE = "line"
|
|
20
|
+
RADAR = "radar"
|
|
21
|
+
SCATTER = "scatter"
|
|
22
|
+
KDE = "kde"
|
|
23
|
+
BARS = "bars"
|
|
24
|
+
PIE = "pie"
|
|
25
|
+
CSV = "csv"
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class MetricVisualization(BaseModel):
|
|
29
|
+
chart_type: ChartType
|
|
30
|
+
metrics: list[str]
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
from typing import Any
|
|
2
|
+
|
|
3
|
+
from pydantic import BaseModel, Field
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class TaskProgress(BaseModel):
|
|
7
|
+
progress: float = Field(..., ge=0.0, le=1.0, description="Progress between 0 and 1")
|
|
8
|
+
extra: dict[str, Any] = Field(default_factory=dict, description="Plugin-defined extra data")
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
class classproperty:
|
|
2
|
+
"""
|
|
3
|
+
Descriptor for creating a class-level property.
|
|
4
|
+
|
|
5
|
+
Allows the definition of a property on a class, similar to @property for instances,
|
|
6
|
+
so the property can be accessed as `ClassName.attribute` rather than `instance.attribute`.
|
|
7
|
+
|
|
8
|
+
Example:
|
|
9
|
+
class ExampleClass:
|
|
10
|
+
_value = 123
|
|
11
|
+
|
|
12
|
+
@classproperty
|
|
13
|
+
def value(cls):
|
|
14
|
+
return cls._value
|
|
15
|
+
|
|
16
|
+
print(ExampleClass.value) # Output: 123
|
|
17
|
+
|
|
18
|
+
Parameters:
|
|
19
|
+
func (function): A callable that accepts the class as an argument and returns a value.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
def __init__(self, func):
|
|
23
|
+
"""
|
|
24
|
+
Initializes the descriptor with the getter function.
|
|
25
|
+
|
|
26
|
+
Parameters:
|
|
27
|
+
func (function): Function taking the class as an argument.
|
|
28
|
+
"""
|
|
29
|
+
self.func = func
|
|
30
|
+
|
|
31
|
+
def __get__(self, instance, owner):
|
|
32
|
+
"""
|
|
33
|
+
Retrieves the class-level property.
|
|
34
|
+
|
|
35
|
+
Parameters:
|
|
36
|
+
instance (object): The instance accessing the attribute
|
|
37
|
+
(None when accessed via the class).
|
|
38
|
+
owner (type): The class on which the property was accessed.
|
|
39
|
+
|
|
40
|
+
Returns:
|
|
41
|
+
Any: The result of invoking the getter function with the class as an argument.
|
|
42
|
+
"""
|
|
43
|
+
return self.func(owner)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|