sampling-mining-workflows-dsl 0.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sampling_mining_workflows_dsl/ __init__.py +0 -0
- sampling_mining_workflows_dsl/CompleteWorkflow.py +23 -0
- sampling_mining_workflows_dsl/Workflow.py +242 -0
- sampling_mining_workflows_dsl/WorkflowBuilder.py +11 -0
- sampling_mining_workflows_dsl/analysis/ChiSquareAnalysis.py +37 -0
- sampling_mining_workflows_dsl/analysis/CochranTest.py +44 -0
- sampling_mining_workflows_dsl/analysis/CochranWorkflowAnalysis.py +37 -0
- sampling_mining_workflows_dsl/analysis/CoverageTest.py +60 -0
- sampling_mining_workflows_dsl/analysis/DistributionWorkflowAnalysis.py +57 -0
- sampling_mining_workflows_dsl/analysis/HistAnalysis.py +201 -0
- sampling_mining_workflows_dsl/analysis/HistWorkflowAnalysis.py +72 -0
- sampling_mining_workflows_dsl/analysis/KSWorkflowAnalysis.py +217 -0
- sampling_mining_workflows_dsl/analysis/WorkflowAnalysis.py +7 -0
- sampling_mining_workflows_dsl/analysis/YamaneTest.py +16 -0
- sampling_mining_workflows_dsl/analysis/YamaneWorkflowAnalysis.py +27 -0
- sampling_mining_workflows_dsl/analysis/kolmogorov_smirnov.py +31 -0
- sampling_mining_workflows_dsl/constraint/BoolComparator.py +23 -0
- sampling_mining_workflows_dsl/constraint/BoolConstraint.py +50 -0
- sampling_mining_workflows_dsl/constraint/BoolConstraintString.py +66 -0
- sampling_mining_workflows_dsl/constraint/Comparator.py +16 -0
- sampling_mining_workflows_dsl/constraint/Constraint.py +23 -0
- sampling_mining_workflows_dsl/constraint/NaturalComparator.py +17 -0
- sampling_mining_workflows_dsl/element/Element.py +45 -0
- sampling_mining_workflows_dsl/element/Loader.py +17 -0
- sampling_mining_workflows_dsl/element/Repository.py +28 -0
- sampling_mining_workflows_dsl/element/Set.py +210 -0
- sampling_mining_workflows_dsl/element/Writer.py +9 -0
- sampling_mining_workflows_dsl/element/loader/CsvLoader.py +67 -0
- sampling_mining_workflows_dsl/element/loader/JsonLoader.py +84 -0
- sampling_mining_workflows_dsl/element/loader/LoaderFactory.py +14 -0
- sampling_mining_workflows_dsl/element/writer/CsvWriter.py +46 -0
- sampling_mining_workflows_dsl/element/writer/JsonWriter.py +58 -0
- sampling_mining_workflows_dsl/element/writer/WriterFactory.py +7 -0
- sampling_mining_workflows_dsl/exec_visualizer/WorkflowVisualizer.py +189 -0
- sampling_mining_workflows_dsl/github_seart/loader.py +17 -0
- sampling_mining_workflows_dsl/github_seart/metadata.py +106 -0
- sampling_mining_workflows_dsl/metadata/Metadata.py +117 -0
- sampling_mining_workflows_dsl/metadata/MetadataBoolean.py +14 -0
- sampling_mining_workflows_dsl/metadata/MetadataDate.py +75 -0
- sampling_mining_workflows_dsl/metadata/MetadataDict.py +33 -0
- sampling_mining_workflows_dsl/metadata/MetadataList.py +32 -0
- sampling_mining_workflows_dsl/metadata/MetadataNumber.py +29 -0
- sampling_mining_workflows_dsl/metadata/MetadataString.py +13 -0
- sampling_mining_workflows_dsl/metadata/MetadataValue.py +24 -0
- sampling_mining_workflows_dsl/operator/Operator.py +165 -0
- sampling_mining_workflows_dsl/operator/OperatorBuilder.py +191 -0
- sampling_mining_workflows_dsl/operator/OperatorFactory.py +76 -0
- sampling_mining_workflows_dsl/operator/clustering/GroupingOperator.py +36 -0
- sampling_mining_workflows_dsl/operator/clustering/SubWorkflowOperatorBuilder.py +76 -0
- sampling_mining_workflows_dsl/operator/selection/SelectionOperator.py +5 -0
- sampling_mining_workflows_dsl/operator/selection/filter/FilterOperator.py +30 -0
- sampling_mining_workflows_dsl/operator/selection/sampling/SamplingOperator.py +11 -0
- sampling_mining_workflows_dsl/operator/selection/sampling/automatic/AutomaticSamplingOperator.py +9 -0
- sampling_mining_workflows_dsl/operator/selection/sampling/automatic/RandomSelectionOperator.py +24 -0
- sampling_mining_workflows_dsl/operator/selection/sampling/automatic/RandomSelectionPartitionOperator.py +62 -0
- sampling_mining_workflows_dsl/operator/selection/sampling/automatic/SystematicRandomSelectionOperator.py +14 -0
- sampling_mining_workflows_dsl/operator/selection/sampling/automatic/SystematicSelectionOperator.py +31 -0
- sampling_mining_workflows_dsl/operator/selection/sampling/manual/InteractiveManualSamplingOperator.py +40 -0
- sampling_mining_workflows_dsl/operator/selection/sampling/manual/ManualSamplingOperator.py +28 -0
- sampling_mining_workflows_dsl/operator/set_algebra/ExternalSetOperator.py +33 -0
- sampling_mining_workflows_dsl/operator/set_algebra/InternalSetOperator.py +47 -0
- sampling_mining_workflows_dsl/operator/set_algebra/SetOperator.py +30 -0
- sampling_mining_workflows_dsl/operator/set_algebra/external_set_operator/DifferenceOperator.py +17 -0
- sampling_mining_workflows_dsl/operator/set_algebra/external_set_operator/IntersectionOperator.py +19 -0
- sampling_mining_workflows_dsl/operator/set_algebra/external_set_operator/UnionOperator.py +17 -0
- sampling_mining_workflows_dsl/operator/set_algebra/internal_set_operator/DifferenceOperator.py +17 -0
- sampling_mining_workflows_dsl/operator/set_algebra/internal_set_operator/IntersectionOperator.py +18 -0
- sampling_mining_workflows_dsl/operator/set_algebra/internal_set_operator/UnionOperator.py +17 -0
- sampling_mining_workflows_dsl/operator/set_algebra/set_operator/DifferenceOperator.py +15 -0
- sampling_mining_workflows_dsl/operator/set_algebra/set_operator/IntersectionOperator.py +18 -0
- sampling_mining_workflows_dsl/operator/set_algebra/set_operator/UnionOperator.py +17 -0
- sampling_mining_workflows_dsl/test/ __init__.py +0 -0
- sampling_mining_workflows_dsl/test/Workflow_simple.py +38 -0
- sampling_mining_workflows_dsl/test/input.json +401 -0
- sampling_mining_workflows_dsl/toolbox.py +42 -0
- sampling_mining_workflows_dsl-0.0.1.dist-info/METADATA +236 -0
- sampling_mining_workflows_dsl-0.0.1.dist-info/RECORD +79 -0
- sampling_mining_workflows_dsl-0.0.1.dist-info/WHEEL +4 -0
- sampling_mining_workflows_dsl-0.0.1.dist-info/licenses/LICENSE.txt +674 -0
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
from sampling_mining_workflows_dsl.constraint.BoolConstraint import BoolConstraint
|
|
2
|
+
from sampling_mining_workflows_dsl.metadata.Metadata import Metadata
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
class MetadataString(Metadata[str]):
|
|
6
|
+
def __init__(self, name: str):
|
|
7
|
+
super().__init__(name, str)
|
|
8
|
+
|
|
9
|
+
def is_equal(self, value: str) -> BoolConstraint:
|
|
10
|
+
return BoolConstraint(None, lambda x: x == value, self)
|
|
11
|
+
|
|
12
|
+
def is_not_equal(self, value: str) -> BoolConstraint:
|
|
13
|
+
return BoolConstraint(None, lambda x: x != value, self)
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
from typing import TYPE_CHECKING, TypeVar
|
|
2
|
+
|
|
3
|
+
if TYPE_CHECKING:
|
|
4
|
+
from sampling_mining_workflows_dsl.metadata.Metadata import Metadata
|
|
5
|
+
|
|
6
|
+
T = TypeVar("T")
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class MetadataValue[T]:
|
|
10
|
+
def __init__(self, metadata: "Metadata[T]", value: T = None):
|
|
11
|
+
self.metadata = metadata
|
|
12
|
+
self.value = value
|
|
13
|
+
|
|
14
|
+
def get_value(self) -> T:
|
|
15
|
+
return self.value
|
|
16
|
+
|
|
17
|
+
def get_metadata(self) -> "Metadata[T]":
|
|
18
|
+
return self.metadata
|
|
19
|
+
|
|
20
|
+
def __str__(self) -> str:
|
|
21
|
+
return str(self.value)
|
|
22
|
+
|
|
23
|
+
def to_string(self) -> str:
|
|
24
|
+
return self.value.to_string()
|
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import pickle
|
|
3
|
+
from abc import ABC
|
|
4
|
+
from typing import TYPE_CHECKING, Optional
|
|
5
|
+
|
|
6
|
+
from sampling_mining_workflows_dsl.element.Loader import Loader
|
|
7
|
+
from sampling_mining_workflows_dsl.element.Set import Set
|
|
8
|
+
from sampling_mining_workflows_dsl.element.Writer import Writer
|
|
9
|
+
|
|
10
|
+
if TYPE_CHECKING:
|
|
11
|
+
from sampling_mining_workflows_dsl.metadata.MetadataValue import MetadataValue
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class Operator(ABC):
|
|
15
|
+
def __init__(self, worflow):
|
|
16
|
+
self.workflow = worflow
|
|
17
|
+
self._input: Set | None = None
|
|
18
|
+
self._output: Set | None = None
|
|
19
|
+
self._output_writter: Writer | None = None
|
|
20
|
+
self._next_operator: Operator | None = None
|
|
21
|
+
self._previous_operator: Operator | None = None
|
|
22
|
+
# Loader to add metadata on output during the execution
|
|
23
|
+
self._loader: Loader | None = None
|
|
24
|
+
|
|
25
|
+
def add_metadata_loader(self, loader) -> "Operator":
|
|
26
|
+
self._loader = loader
|
|
27
|
+
return self
|
|
28
|
+
|
|
29
|
+
def set_output_set_id(self, set_id: str) -> "Operator":
|
|
30
|
+
if self._output is not None:
|
|
31
|
+
self._output.set_id(set_id)
|
|
32
|
+
return self
|
|
33
|
+
|
|
34
|
+
def set_input_set_id(self, set_id: str) -> "Operator":
|
|
35
|
+
if self._input is not None:
|
|
36
|
+
self._input.set_id(set_id)
|
|
37
|
+
return self
|
|
38
|
+
|
|
39
|
+
def add_metadata(self) -> "Operator":
|
|
40
|
+
# Add the new metadata to the existing workflow metadata list
|
|
41
|
+
for metadata in self._loader.metadatas.values():
|
|
42
|
+
if metadata != self._loader.metadata_id_name:
|
|
43
|
+
self.workflow.add_metadata_type(metadata)
|
|
44
|
+
|
|
45
|
+
# Load new metadata set
|
|
46
|
+
new_metadata_set: Set = self._loader.load_set()
|
|
47
|
+
# Add metadata value to current output set
|
|
48
|
+
for element in self._output.get_elements():
|
|
49
|
+
if not isinstance(element, Set):
|
|
50
|
+
id = element.get_id()
|
|
51
|
+
try:
|
|
52
|
+
new_element = new_metadata_set.get_element(id)
|
|
53
|
+
metadata_values: list[MetadataValue] = (
|
|
54
|
+
new_element.get_all_metadata_values().values()
|
|
55
|
+
)
|
|
56
|
+
metadata_values_filtered = [
|
|
57
|
+
x for x in metadata_values if x.get_value() != id
|
|
58
|
+
]
|
|
59
|
+
element.add_metadata_values(metadata_values_filtered)
|
|
60
|
+
|
|
61
|
+
except ValueError:
|
|
62
|
+
print(f"Element with id {id} not found in the metadata set")
|
|
63
|
+
|
|
64
|
+
def execute(self) -> "Operator":
|
|
65
|
+
if self._loader:
|
|
66
|
+
self.add_metadata()
|
|
67
|
+
|
|
68
|
+
if self._next_operator:
|
|
69
|
+
self._next_operator._input = self._output
|
|
70
|
+
self._next_operator.execute()
|
|
71
|
+
elif self._output_writter:
|
|
72
|
+
self._output_writter.write_set(self._output)
|
|
73
|
+
return self
|
|
74
|
+
|
|
75
|
+
def get_workflow_root_operator(self) -> "Operator":
|
|
76
|
+
if self._previous_operator is None:
|
|
77
|
+
return self
|
|
78
|
+
return self._previous_operator.get_workflow_root_operator()
|
|
79
|
+
|
|
80
|
+
def execute_workflow(self) -> "Operator":
|
|
81
|
+
root = self.get_workflow_root_operator()
|
|
82
|
+
root.input_set(self._input).execute()
|
|
83
|
+
return self
|
|
84
|
+
|
|
85
|
+
def get_next_operator(self) -> Optional["Operator"]:
|
|
86
|
+
return self._next_operator
|
|
87
|
+
|
|
88
|
+
def get_output(self) -> Set | None:
|
|
89
|
+
return self._output
|
|
90
|
+
|
|
91
|
+
def get_merged_output(self) -> Set:
|
|
92
|
+
result = Set()
|
|
93
|
+
for element in self._output.get_elements():
|
|
94
|
+
if isinstance(element, Set):
|
|
95
|
+
# Recursively flatten nested Sets
|
|
96
|
+
result.union(element.flatten_set())
|
|
97
|
+
else:
|
|
98
|
+
# Add non-Set elements directly
|
|
99
|
+
result.add_element(element)
|
|
100
|
+
return result
|
|
101
|
+
|
|
102
|
+
def output(self, writter: Writer) -> "Operator":
|
|
103
|
+
self._output_writter = writter
|
|
104
|
+
return self
|
|
105
|
+
|
|
106
|
+
def input_set(self, input_set: Set) -> "Operator":
|
|
107
|
+
self._input = input_set
|
|
108
|
+
return self
|
|
109
|
+
|
|
110
|
+
def output_set(self, output_set: Set) -> "Operator":
|
|
111
|
+
self._output = output_set
|
|
112
|
+
return self
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def get_input(self) -> Set | None:
|
|
116
|
+
return self._input
|
|
117
|
+
|
|
118
|
+
def input(self, loader: Loader) -> "Operator":
|
|
119
|
+
self._input = loader.load_set()
|
|
120
|
+
return self
|
|
121
|
+
|
|
122
|
+
def chain(self, operator: "Operator") -> "Operator":
|
|
123
|
+
self._next_operator = operator
|
|
124
|
+
operator._previous_operator = self
|
|
125
|
+
return operator
|
|
126
|
+
|
|
127
|
+
def __str__(self) -> str:
|
|
128
|
+
return self.to_string(0)
|
|
129
|
+
|
|
130
|
+
def to_string(self, level: int = 0) -> str:
|
|
131
|
+
indent = " " * level
|
|
132
|
+
double_indent = " " * (level + 1)
|
|
133
|
+
formatted_next_operator = (
|
|
134
|
+
self._next_operator.to_string(level + 3)
|
|
135
|
+
if self._next_operator
|
|
136
|
+
else double_indent
|
|
137
|
+
)
|
|
138
|
+
|
|
139
|
+
class_name = self.__class__.__name__
|
|
140
|
+
return (
|
|
141
|
+
f"{indent}{class_name}\n"
|
|
142
|
+
f"{double_indent}{self.extra_to_string(level)}\n"
|
|
143
|
+
f"{double_indent}input:\n"
|
|
144
|
+
f"{double_indent}{self._input.to_string(level + 2) if self._input else 'None'}\n"
|
|
145
|
+
f"{double_indent}output:\n"
|
|
146
|
+
f"{double_indent}{self._output.to_string(level + 2) if self._output else 'None'}\n"
|
|
147
|
+
f"{double_indent}nextOperator:\n"
|
|
148
|
+
f"{formatted_next_operator}"
|
|
149
|
+
)
|
|
150
|
+
|
|
151
|
+
def extra_to_string(self, level: int) -> str:
|
|
152
|
+
return ""
|
|
153
|
+
|
|
154
|
+
def short_str(self) -> str:
|
|
155
|
+
return f"{self.__class__.__name__} (input size: {self._input.size() if self._input else 'None'}, output size: {self._output.size() if self._output else 'None'})"
|
|
156
|
+
|
|
157
|
+
def serialize(self, path: str):
|
|
158
|
+
os.makedirs(os.path.dirname(path), exist_ok=True)
|
|
159
|
+
with open(path, "wb") as file:
|
|
160
|
+
pickle.dump(self, file)
|
|
161
|
+
|
|
162
|
+
@staticmethod
|
|
163
|
+
def deserialize(path: str) -> "Operator":
|
|
164
|
+
with open(path, "rb") as file:
|
|
165
|
+
return pickle.load(file)
|
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
from typing import TYPE_CHECKING, TypeVar, cast
|
|
2
|
+
|
|
3
|
+
from sampling_mining_workflows_dsl.constraint import NaturalComparator
|
|
4
|
+
from sampling_mining_workflows_dsl.constraint.BoolConstraintString import BoolConstraintString
|
|
5
|
+
from sampling_mining_workflows_dsl.constraint.Comparator import Comparator
|
|
6
|
+
from sampling_mining_workflows_dsl.constraint.Constraint import Constraint
|
|
7
|
+
from sampling_mining_workflows_dsl.element.Loader import Loader
|
|
8
|
+
from sampling_mining_workflows_dsl.operator.clustering.GroupingOperator import GroupingOperator
|
|
9
|
+
from sampling_mining_workflows_dsl.operator.selection.filter.FilterOperator import FilterOperator
|
|
10
|
+
from sampling_mining_workflows_dsl.operator.selection.sampling.automatic.RandomSelectionOperator import (
|
|
11
|
+
RandomSelectionOperator,
|
|
12
|
+
)
|
|
13
|
+
from sampling_mining_workflows_dsl.operator.selection.sampling.automatic.SystematicSelectionOperator import (
|
|
14
|
+
SystematicSelectionOperator,
|
|
15
|
+
)
|
|
16
|
+
from sampling_mining_workflows_dsl.operator.selection.sampling.manual.ManualSamplingOperator import (
|
|
17
|
+
ManualSamplingOperator,
|
|
18
|
+
)
|
|
19
|
+
from sampling_mining_workflows_dsl.operator.set_algebra.set_operator import DifferenceOperator
|
|
20
|
+
|
|
21
|
+
if TYPE_CHECKING:
|
|
22
|
+
from sampling_mining_workflows_dsl.operator.Operator import Operator
|
|
23
|
+
from sampling_mining_workflows_dsl.Workflow import Workflow
|
|
24
|
+
|
|
25
|
+
T = TypeVar("T")
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class OperatorBuilder:
|
|
29
|
+
def __init__(self, workflow: "Workflow | None" = None):
|
|
30
|
+
if workflow is None:
|
|
31
|
+
from sampling_mining_workflows_dsl.Workflow import Workflow as WorkflowClass
|
|
32
|
+
|
|
33
|
+
workflow = WorkflowClass()
|
|
34
|
+
self.workflow = workflow
|
|
35
|
+
|
|
36
|
+
def grouping_operator(self, *workflows: "OperatorBuilder") -> "OperatorBuilder":
|
|
37
|
+
if not workflows:
|
|
38
|
+
raise ValueError("At least one workflow must be provided.")
|
|
39
|
+
|
|
40
|
+
# Retrieve all metadata from the main workflow
|
|
41
|
+
all_metadata = self.workflow.get_all_Metadata()
|
|
42
|
+
|
|
43
|
+
# Ensure all subworkflows have the same metadata
|
|
44
|
+
for w in workflows:
|
|
45
|
+
for metadata in all_metadata:
|
|
46
|
+
w.workflow.add_metadata_type(metadata)
|
|
47
|
+
|
|
48
|
+
# Extract subworkflow objects
|
|
49
|
+
subWorkflows = [w.workflow for w in workflows]
|
|
50
|
+
|
|
51
|
+
# Create and add the GroupingOperator
|
|
52
|
+
grouping_operator = GroupingOperator(self.workflow, subWorkflows)
|
|
53
|
+
self.workflow.add_operator(cast("Operator", grouping_operator))
|
|
54
|
+
return self
|
|
55
|
+
|
|
56
|
+
def add_metadata(self, loader: Loader):
|
|
57
|
+
self.workflow.add_metadata(loader)
|
|
58
|
+
return self
|
|
59
|
+
|
|
60
|
+
def set_output_set_id(self, set_id: str) -> "OperatorBuilder":
|
|
61
|
+
if self.workflow.operators:
|
|
62
|
+
last_operator = self.workflow.operators[-1]
|
|
63
|
+
last_operator.set_output_set_id(set_id)
|
|
64
|
+
return self
|
|
65
|
+
|
|
66
|
+
def set_input_set_id(self, set_id: str) -> "OperatorBuilder":
|
|
67
|
+
if self.workflow.operators:
|
|
68
|
+
first_operator = self.workflow.operators[0]
|
|
69
|
+
first_operator.set_input_set_id(set_id)
|
|
70
|
+
return self
|
|
71
|
+
|
|
72
|
+
def random_selection_operator(
|
|
73
|
+
self, cardinality: int, seed: int = 0
|
|
74
|
+
) -> "OperatorBuilder":
|
|
75
|
+
random_selection_operator = RandomSelectionOperator(
|
|
76
|
+
self.workflow, cardinality=cardinality, seed=seed
|
|
77
|
+
)
|
|
78
|
+
self.workflow.add_operator(cast("Operator", random_selection_operator))
|
|
79
|
+
return self
|
|
80
|
+
|
|
81
|
+
def union_operator(self) -> "OperatorBuilder":
|
|
82
|
+
from sampling_mining_workflows_dsl.operator.set_algebra.internal_set_operator.UnionOperator import UnionOperator
|
|
83
|
+
union_operator = UnionOperator(self.workflow)
|
|
84
|
+
self.workflow.add_operator(union_operator)
|
|
85
|
+
return self
|
|
86
|
+
|
|
87
|
+
def union_with_operator(self, set_name: str) -> "OperatorBuilder":
|
|
88
|
+
from sampling_mining_workflows_dsl.operator.set_algebra.set_operator.UnionOperator import UnionOperator
|
|
89
|
+
union_operator = UnionOperator(self.workflow, set_name)
|
|
90
|
+
self.workflow.add_operator(union_operator)
|
|
91
|
+
return self
|
|
92
|
+
|
|
93
|
+
def union_with_external_set_operator(self, loader:Loader) -> "OperatorBuilder":
|
|
94
|
+
from sampling_mining_workflows_dsl.operator.set_algebra.external_set_operator.UnionOperator import UnionOperator
|
|
95
|
+
|
|
96
|
+
union_operator = UnionOperator(self.workflow, loader)
|
|
97
|
+
self.workflow.add_operator(union_operator)
|
|
98
|
+
return self
|
|
99
|
+
|
|
100
|
+
def difference_with_external_set_operator(self, loader:Loader) -> "OperatorBuilder":
|
|
101
|
+
from sampling_mining_workflows_dsl.operator.set_algebra.external_set_operator.DifferenceOperator import DiferenceOperator
|
|
102
|
+
|
|
103
|
+
union_operator = DifferenceOperator(self.workflow, loader)
|
|
104
|
+
self.workflow.add_operator(union_operator)
|
|
105
|
+
return self
|
|
106
|
+
|
|
107
|
+
def difference_with_operator(self, set_name: str) -> "OperatorBuilder":
|
|
108
|
+
difference_operator = DifferenceOperator(self.workflow, set_name)
|
|
109
|
+
self.workflow.add_operator(difference_operator)
|
|
110
|
+
return self
|
|
111
|
+
|
|
112
|
+
def filter_operator(self, constraint: str | Constraint) -> "OperatorBuilder":
|
|
113
|
+
if isinstance(constraint, str):
|
|
114
|
+
# Handle the case where the constraint is a string
|
|
115
|
+
constraint_obj = BoolConstraintString(self.workflow, constraint)
|
|
116
|
+
elif isinstance(constraint, Constraint):
|
|
117
|
+
# Handle the case where the constraint is already a Constraint object
|
|
118
|
+
constraint_obj = constraint
|
|
119
|
+
else:
|
|
120
|
+
raise TypeError("constraint must be a string or a Constraint object")
|
|
121
|
+
|
|
122
|
+
filter_operator = FilterOperator(self.workflow, constraint_obj)
|
|
123
|
+
self.workflow.add_operator(cast("Operator", filter_operator))
|
|
124
|
+
return self
|
|
125
|
+
|
|
126
|
+
def systematic_selection_operator(
|
|
127
|
+
self,
|
|
128
|
+
cardinality: int,
|
|
129
|
+
metadata_name: str,
|
|
130
|
+
reverse=False,
|
|
131
|
+
step: int = 1,
|
|
132
|
+
order_constraint: Comparator = NaturalComparator,
|
|
133
|
+
) -> "OperatorBuilder":
|
|
134
|
+
systematic_selection_operator = SystematicSelectionOperator(
|
|
135
|
+
self.workflow, cardinality, metadata_name, reverse, step, order_constraint
|
|
136
|
+
)
|
|
137
|
+
self.workflow.add_operator(cast("Operator", systematic_selection_operator))
|
|
138
|
+
return self
|
|
139
|
+
|
|
140
|
+
def manual_sampling_operator(self, *ids: T) -> "OperatorBuilder":
|
|
141
|
+
if not ids:
|
|
142
|
+
raise ValueError(
|
|
143
|
+
"At least one element must be provided for manual sampling."
|
|
144
|
+
)
|
|
145
|
+
|
|
146
|
+
manual_sampling_operator = ManualSamplingOperator(*ids)
|
|
147
|
+
self.workflow.add_operator(cast("Operator", manual_sampling_operator))
|
|
148
|
+
return self
|
|
149
|
+
|
|
150
|
+
def output(self, writer) -> "Workflow":
|
|
151
|
+
return self.workflow.output(writer)
|
|
152
|
+
|
|
153
|
+
def cluster_sampling_operator(self, n_clusters, *cluster_constraints):
|
|
154
|
+
"""
|
|
155
|
+
Adds a cluster sampling grouping operator to the workflow.
|
|
156
|
+
Args:
|
|
157
|
+
n_clusters (int): Number of clusters to select.
|
|
158
|
+
*cluster_constraints: Constraints defining clusters.
|
|
159
|
+
Returns:
|
|
160
|
+
WorkflowBuilder: self (for chaining)
|
|
161
|
+
"""
|
|
162
|
+
return self.grouping_operator(
|
|
163
|
+
*[OperatorBuilder().filter_operator(constraint) for constraint in cluster_constraints]
|
|
164
|
+
).random_selection_operator(n_clusters)
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def stratified_random_operator(self, sample_size, *strata_constraints):
|
|
168
|
+
"""
|
|
169
|
+
Adds a stratified random grouping operator to the workflow.
|
|
170
|
+
Args:
|
|
171
|
+
sample_size (int): Number of samples per stratum.
|
|
172
|
+
* strata_constraints: Constraints defining strata.
|
|
173
|
+
Returns:
|
|
174
|
+
WorkflowBuilder: self (for chaining)
|
|
175
|
+
"""
|
|
176
|
+
return self.grouping_operator(
|
|
177
|
+
*[OperatorBuilder().filter_operator(constraint).random_selection_operator(sample_size) for constraint in strata_constraints]
|
|
178
|
+
)
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def quota_operator(self, *quota_constraints):
|
|
182
|
+
"""
|
|
183
|
+
Adds a quota grouping operator to the workflow.
|
|
184
|
+
Args:
|
|
185
|
+
*quota_constraints: Constraints defining quota groups (can be composed constraints).
|
|
186
|
+
Returns:
|
|
187
|
+
WorkflowBuilder: self (for chaining)
|
|
188
|
+
"""
|
|
189
|
+
return self.grouping_operator(
|
|
190
|
+
*[OperatorBuilder().filter_operator(constraint) for constraint in quota_constraints]
|
|
191
|
+
)
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
from collections.abc import Callable
|
|
2
|
+
from typing import TypeVar
|
|
3
|
+
|
|
4
|
+
from sampling_mining_workflows_dsl.constraint.Comparator import Comparator
|
|
5
|
+
from sampling_mining_workflows_dsl.constraint.Constraint import Constraint
|
|
6
|
+
from sampling_mining_workflows_dsl.operator.clustering.GroupingOperator import GroupingOperator
|
|
7
|
+
from sampling_mining_workflows_dsl.operator.Operator import Operator
|
|
8
|
+
from sampling_mining_workflows_dsl.operator.selection.filter.FilterOperator import FilterOperator
|
|
9
|
+
from sampling_mining_workflows_dsl.operator.selection.sampling.automatic.RandomSelectionOperator import (
|
|
10
|
+
RandomSelectionOperator,
|
|
11
|
+
)
|
|
12
|
+
from sampling_mining_workflows_dsl.operator.selection.sampling.automatic.RandomSelectionPartitionOperator import (
|
|
13
|
+
RandomSelectionPartitionOperator,
|
|
14
|
+
)
|
|
15
|
+
from sampling_mining_workflows_dsl.operator.selection.sampling.automatic.SystematicRandomSelectionOperator import (
|
|
16
|
+
SystematicRandomSelectionOperator,
|
|
17
|
+
)
|
|
18
|
+
from sampling_mining_workflows_dsl.operator.selection.sampling.automatic.SystematicSelectionOperator import (
|
|
19
|
+
SystematicSelectionOperator,
|
|
20
|
+
)
|
|
21
|
+
from sampling_mining_workflows_dsl.operator.selection.sampling.manual.InteractiveManualSamplingOperator import (
|
|
22
|
+
InteractiveManualSamplingOperator,
|
|
23
|
+
)
|
|
24
|
+
from sampling_mining_workflows_dsl.operator.selection.sampling.manual.ManualSamplingOperator import (
|
|
25
|
+
ManualSamplingOperator,
|
|
26
|
+
)
|
|
27
|
+
|
|
28
|
+
T = TypeVar("T")
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class OperatorFactory:
|
|
32
|
+
@staticmethod
|
|
33
|
+
def filter_operator(constraint: Constraint) -> FilterOperator:
|
|
34
|
+
return FilterOperator(constraint)
|
|
35
|
+
|
|
36
|
+
@staticmethod
|
|
37
|
+
def random_selection_operator(
|
|
38
|
+
cardinality: int, seed: int = 0
|
|
39
|
+
) -> RandomSelectionOperator:
|
|
40
|
+
return RandomSelectionOperator(cardinality, seed)
|
|
41
|
+
|
|
42
|
+
@staticmethod
|
|
43
|
+
def random_selection_partition_operator(
|
|
44
|
+
seed: int, cardinality: int
|
|
45
|
+
) -> RandomSelectionPartitionOperator:
|
|
46
|
+
return RandomSelectionPartitionOperator(cardinality=cardinality, seed=seed)
|
|
47
|
+
|
|
48
|
+
@staticmethod
|
|
49
|
+
def grouping_operator(*operators: Operator):
|
|
50
|
+
return GroupingOperator(*operators)
|
|
51
|
+
|
|
52
|
+
@staticmethod
|
|
53
|
+
def manual_sampling_operator(*ids: T) -> ManualSamplingOperator[T]:
|
|
54
|
+
return ManualSamplingOperator(*ids)
|
|
55
|
+
|
|
56
|
+
@staticmethod
|
|
57
|
+
def interactive_manual_sampling_operator() -> InteractiveManualSamplingOperator[T]:
|
|
58
|
+
return InteractiveManualSamplingOperator()
|
|
59
|
+
|
|
60
|
+
@staticmethod
|
|
61
|
+
def systematic_selection_operator(
|
|
62
|
+
cardinality: int, order_constraint: Comparator, pas: int
|
|
63
|
+
) -> SystematicSelectionOperator:
|
|
64
|
+
return SystematicSelectionOperator(cardinality, order_constraint, pas)
|
|
65
|
+
|
|
66
|
+
@staticmethod
|
|
67
|
+
def systematic_random_selection_operator(
|
|
68
|
+
cardinality: int, pas: int
|
|
69
|
+
) -> SystematicRandomSelectionOperator:
|
|
70
|
+
return SystematicRandomSelectionOperator(cardinality, pas)
|
|
71
|
+
|
|
72
|
+
@staticmethod
|
|
73
|
+
def parameterized_operators(
|
|
74
|
+
operator: Callable[[T], Operator], *values: T
|
|
75
|
+
) -> list[Operator]:
|
|
76
|
+
return [operator(value) for value in values]
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
|
|
2
|
+
from typing import TYPE_CHECKING
|
|
3
|
+
|
|
4
|
+
from sampling_mining_workflows_dsl.element.Set import Set
|
|
5
|
+
from sampling_mining_workflows_dsl.operator.Operator import Operator
|
|
6
|
+
|
|
7
|
+
if TYPE_CHECKING:
|
|
8
|
+
from sampling_mining_workflows_dsl.Workflow import Workflow
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class GroupingOperator(Operator):
|
|
12
|
+
def __init__(self, root_workflow, workflows):
|
|
13
|
+
super().__init__(root_workflow)
|
|
14
|
+
|
|
15
|
+
self.workflows = workflows
|
|
16
|
+
|
|
17
|
+
def execute(self) -> Operator:
|
|
18
|
+
self._output = Set()
|
|
19
|
+
for w in self.workflows:
|
|
20
|
+
# The input of the workflow is the input of the grouping operator
|
|
21
|
+
w.set_workflow_input(self._input)
|
|
22
|
+
|
|
23
|
+
w.execute_workflow()
|
|
24
|
+
self._output.add_element(w.get_workflow_output())
|
|
25
|
+
super().execute()
|
|
26
|
+
return self
|
|
27
|
+
|
|
28
|
+
def get_workflows(self) -> list["Workflow"]:
|
|
29
|
+
return self.workflows
|
|
30
|
+
|
|
31
|
+
def extra_to_string(self, level: int) -> str:
|
|
32
|
+
indent = " " * (level + 1)
|
|
33
|
+
res = f"\n{indent}Internal Workflows:"
|
|
34
|
+
for w in self.workflows:
|
|
35
|
+
res += f"\n{w.to_string(level + 1)}"
|
|
36
|
+
return res
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
from typing import TYPE_CHECKING, TypeVar, cast
|
|
2
|
+
|
|
3
|
+
from sampling_mining_workflows_dsl.constraint.BoolConstraintString import BoolConstraintString
|
|
4
|
+
from sampling_mining_workflows_dsl.constraint.Constraint import Constraint
|
|
5
|
+
from sampling_mining_workflows_dsl.operator.clustering.GroupingOperator import GroupingOperator
|
|
6
|
+
from sampling_mining_workflows_dsl.operator.OperatorBuilder import OperatorBuilder
|
|
7
|
+
from sampling_mining_workflows_dsl.operator.selection.filter.FilterOperator import FilterOperator
|
|
8
|
+
from sampling_mining_workflows_dsl.operator.selection.sampling.automatic.RandomSelectionOperator import (
|
|
9
|
+
RandomSelectionOperator,
|
|
10
|
+
)
|
|
11
|
+
from sampling_mining_workflows_dsl.operator.selection.sampling.manual.ManualSamplingOperator import (
|
|
12
|
+
ManualSamplingOperator,
|
|
13
|
+
)
|
|
14
|
+
from sampling_mining_workflows_dsl.Workflow import Workflow
|
|
15
|
+
|
|
16
|
+
if TYPE_CHECKING:
|
|
17
|
+
from sampling_mining_workflows_dsl.operator.Operator import Operator
|
|
18
|
+
|
|
19
|
+
T = TypeVar("T")
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class SubWorkflowOperatorBuilder:
|
|
23
|
+
@staticmethod
|
|
24
|
+
def grouping_operator(*workflows: "OperatorBuilder") -> "OperatorBuilder":
|
|
25
|
+
workflows = [
|
|
26
|
+
w.workflow for w in workflows
|
|
27
|
+
] # Extract the Workflow objects from OperatorBuilder instances
|
|
28
|
+
if not workflows:
|
|
29
|
+
raise ValueError("At least one workflow must be provided.")
|
|
30
|
+
|
|
31
|
+
# Create a GroupingOperator with the provided sub workflows
|
|
32
|
+
grouping_operator = GroupingOperator(*workflows)
|
|
33
|
+
|
|
34
|
+
# Add the grouping operator to the current workflow
|
|
35
|
+
# self.workflow.add_operator(cast(Operator, grouping_operator))
|
|
36
|
+
subWorkflow = Workflow()
|
|
37
|
+
subWorkflow.add_operator(grouping_operator)
|
|
38
|
+
return OperatorBuilder(subWorkflow)
|
|
39
|
+
|
|
40
|
+
@staticmethod
|
|
41
|
+
def random_selection_operator(cardinality: int, seed: int = 0) -> "OperatorBuilder":
|
|
42
|
+
subWorkflow = Workflow()
|
|
43
|
+
random_selection_operator = RandomSelectionOperator(
|
|
44
|
+
subWorkflow, cardinality=cardinality, seed=seed
|
|
45
|
+
)
|
|
46
|
+
subWorkflow.add_operator(cast("Operator", random_selection_operator))
|
|
47
|
+
return OperatorBuilder(subWorkflow)
|
|
48
|
+
|
|
49
|
+
@staticmethod
|
|
50
|
+
def filter_operator(constraint: str | Constraint) -> "OperatorBuilder":
|
|
51
|
+
subWorkflow = Workflow()
|
|
52
|
+
|
|
53
|
+
if isinstance(constraint, str):
|
|
54
|
+
# Handle the case where the constraint is a string
|
|
55
|
+
constraint_obj = BoolConstraintString(subWorkflow, constraint)
|
|
56
|
+
elif isinstance(constraint, Constraint):
|
|
57
|
+
# Handle the case where the constraint is already a Constraint object
|
|
58
|
+
constraint_obj = constraint
|
|
59
|
+
else:
|
|
60
|
+
raise TypeError("constraint must be a string or a Constraint object")
|
|
61
|
+
|
|
62
|
+
filter_operator = FilterOperator(subWorkflow, constraint_obj)
|
|
63
|
+
subWorkflow.add_operator(cast("Operator", filter_operator))
|
|
64
|
+
return OperatorBuilder(subWorkflow)
|
|
65
|
+
|
|
66
|
+
@staticmethod
|
|
67
|
+
def manual_sampling_operator(*ids: T) -> "OperatorBuilder":
|
|
68
|
+
if not ids:
|
|
69
|
+
raise ValueError(
|
|
70
|
+
"At least one element must be provided for manual sampling."
|
|
71
|
+
)
|
|
72
|
+
|
|
73
|
+
subWorkflow = Workflow()
|
|
74
|
+
manual_sampling_operator = ManualSamplingOperator(*ids)
|
|
75
|
+
subWorkflow.add_operator(cast("Operator", manual_sampling_operator))
|
|
76
|
+
return OperatorBuilder(subWorkflow)
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
from sampling_mining_workflows_dsl.constraint.Constraint import Constraint
|
|
2
|
+
from sampling_mining_workflows_dsl.element.Set import Set
|
|
3
|
+
from sampling_mining_workflows_dsl.operator.Operator import Operator
|
|
4
|
+
from tqdm import tqdm
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class FilterOperator(Operator):
|
|
8
|
+
def __init__(self, workflow, constraint: Constraint):
|
|
9
|
+
super().__init__(workflow)
|
|
10
|
+
self._constraint = constraint
|
|
11
|
+
constraint.set_workflow(workflow)
|
|
12
|
+
|
|
13
|
+
def execute(self):
|
|
14
|
+
self._output = Set()
|
|
15
|
+
elements = self._input.get_elements()
|
|
16
|
+
|
|
17
|
+
# Add progress bar for filtering operation
|
|
18
|
+
with tqdm(elements, desc="Filtering elements", unit="element") as pbar:
|
|
19
|
+
for element in pbar:
|
|
20
|
+
if self._constraint.is_satisfied(element):
|
|
21
|
+
self._output.add_element(element)
|
|
22
|
+
|
|
23
|
+
super().execute()
|
|
24
|
+
return self
|
|
25
|
+
|
|
26
|
+
def get_constraint(self) -> Constraint:
|
|
27
|
+
return self._constraint
|
|
28
|
+
|
|
29
|
+
def short_str(self) -> str:
|
|
30
|
+
return str(self._constraint)+" "+super().short_str()
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
|
|
2
|
+
from sampling_mining_workflows_dsl.operator.selection.SelectionOperator import SelectionOperator
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
class SamplingOperator(SelectionOperator):
|
|
6
|
+
def __init__(self, workflow, cardinality: int):
|
|
7
|
+
super().__init__(workflow)
|
|
8
|
+
self._cardinality = cardinality
|
|
9
|
+
def get_cardinality(self):
|
|
10
|
+
return self._cardinality
|
|
11
|
+
|
sampling_mining_workflows_dsl/operator/selection/sampling/automatic/RandomSelectionOperator.py
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import random
|
|
2
|
+
|
|
3
|
+
from sampling_mining_workflows_dsl.operator.Operator import Operator
|
|
4
|
+
from sampling_mining_workflows_dsl.operator.selection.sampling.automatic.AutomaticSamplingOperator import (
|
|
5
|
+
AutomaticSamplingOperator,
|
|
6
|
+
)
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class RandomSelectionOperator(AutomaticSamplingOperator):
|
|
10
|
+
def __init__(self, workflow, cardinality: int, seed: int = -1):
|
|
11
|
+
super().__init__(workflow, cardinality)
|
|
12
|
+
# If seed is -1, a random seed will be generated
|
|
13
|
+
if seed == -1:
|
|
14
|
+
seed = random.randint(0, 1000000)
|
|
15
|
+
|
|
16
|
+
self._seed = seed
|
|
17
|
+
|
|
18
|
+
def execute(self) -> Operator:
|
|
19
|
+
self._output = self._input.get_random_subset(self._cardinality, self._seed)
|
|
20
|
+
super().execute()
|
|
21
|
+
return self
|
|
22
|
+
|
|
23
|
+
def short_str(self) -> str:
|
|
24
|
+
return super().short_str() + f" (cardinality: {self._cardinality}, seed: {self._seed})"
|