sampling-mining-workflows-dsl 0.0.1__py3-none-any.whl

sampling_mining_workflows_dsl/operator/selection/sampling/automatic/RandomSelectionPartitionOperator.py

@@ -0,0 +1,62 @@
+
+from sampling_mining_workflows_dsl.element.Set import Set
+from sampling_mining_workflows_dsl.operator.Operator import Operator
+from sampling_mining_workflows_dsl.operator.selection.sampling.automatic.AutomaticSamplingOperator import (
+    AutomaticSamplingOperator,
+)
+from sampling_mining_workflows_dsl.Workflow import Workflow
+
+
+class RandomSelectionPartitionOperator(AutomaticSamplingOperator):
+    def __init__(self, workflow: Workflow, cardinality: int, seed: int = 0):
+        super().__init__(workflow, cardinality)
+        self.seed = seed
+
+    def execute(self) -> Operator:
+        from sampling_mining_workflows_dsl.operator.OperatorFactory import (
+            OperatorFactory,
+        )  # Local import
+
+        random_selection_operator = OperatorFactory.random_selection_operator
+
+        self._output = Set()
+        is_set_of_set = all(isinstance(x, Set) for x in self._input.get_elements())
+
+        if is_set_of_set:
+            input_sets: list[Set] = [
+                x for x in self._input.get_elements() if isinstance(x, Set)
+            ]
+            total_size = sum(len(s.get_elements()) for s in input_sets)
+
+            # Compute the number of elements per list
+            sample_sizes = [
+                round((len(sublist.get_elements()) / total_size) * self._cardinality)
+                for sublist in input_sets
+            ]
+
+            # Correction to ensure the total sample size matches the cardinality
+            diff = self._cardinality - sum(sample_sizes)
+            i = 0
+            while diff != 0:
+                index = i % len(input_sets)
+                if diff > 0:
+                    sample_sizes[index] += 1
+                    diff -= 1
+                else:
+                    sample_sizes[index] -= 1
+                    diff += 1
+                i += 1
+
+            # Generate sampled sets
+            for i, sublist in enumerate(input_sets):
+                cardinality = sample_sizes[i]
+                sampled_set = (
+                    random_selection_operator(cardinality)
+                    .input_set(sublist)
+                    .execute()
+                    .get_output()
+                )
+                self._output.add_element(sampled_set)
+
+            return self
+        raise RuntimeError("Input is not a set of sets")

sampling_mining_workflows_dsl/operator/selection/sampling/automatic/SystematicRandomSelectionOperator.py

@@ -0,0 +1,14 @@
+from sampling_mining_workflows_dsl.operator.Operator import Operator
+from sampling_mining_workflows_dsl.operator.selection.sampling.automatic.AutomaticSamplingOperator import (
+    AutomaticSamplingOperator,
+)
+from sampling_mining_workflows_dsl.Workflow import Workflow
+
+
+class SystematicRandomSelectionOperator(AutomaticSamplingOperator):
+    def __init__(self, workflow: Workflow, cardinality: int, pas: int):
+        super().__init__(workflow, cardinality)
+        self.pas = pas
+
+    def execute(self) -> Operator:
+        raise NotImplementedError("Unimplemented method 'execute'")

sampling_mining_workflows_dsl/operator/selection/sampling/automatic/SystematicSelectionOperator.py

@@ -0,0 +1,31 @@
+from sampling_mining_workflows_dsl.constraint import NaturalComparator
+from sampling_mining_workflows_dsl.constraint.Comparator import Comparator
+from sampling_mining_workflows_dsl.operator.Operator import Operator
+from sampling_mining_workflows_dsl.operator.selection.sampling.automatic.AutomaticSamplingOperator import (
+    AutomaticSamplingOperator,
+)
+from sampling_mining_workflows_dsl.Workflow import Workflow
+
+
+class SystematicSelectionOperator(AutomaticSamplingOperator):
+    def __init__(
+        self,
+        workflow: Workflow,
+        cardinality: int,
+        metadata_name: str,
+        reverse=False,
+        step: int = 1,
+        order_constraint: Comparator = NaturalComparator,
+    ):
+        super().__init__(workflow, cardinality)
+
+        self.metadata_name = metadata_name
+        self.step = step
+        self.order_constraint = order_constraint
+
+    def execute(self) -> Operator:
+        self._input.sort_by_metadata(
+            self.metadata_name, self.order_constraint, self.reverse
+        )
+        self._output = self._input.get_systematic_subset(self._cardinality, self.step)
+        return self

sampling_mining_workflows_dsl/operator/selection/sampling/manual/InteractiveManualSamplingOperator.py

@@ -0,0 +1,40 @@
+from typing import TypeVar
+
+from sampling_mining_workflows_dsl.element.Repository import Repository
+from sampling_mining_workflows_dsl.element.Set import Set
+from sampling_mining_workflows_dsl.operator.Operator import Operator
+from sampling_mining_workflows_dsl.operator.selection.sampling.manual.ManualSamplingOperator import (
+    ManualSamplingOperator,
+)
+from sampling_mining_workflows_dsl.Workflow import Workflow
+
+T = TypeVar("T")
+
+
+class InteractiveManualSamplingOperator[T](ManualSamplingOperator):
+    def __init__(self, workflow: Workflow):
+        super().__init__(workflow)
+
+    def execute(self) -> Operator:
+        self._output = Set()
+
+        print(f"input : \n{self._input.to_string() if self._input else 'None'}")
+        for element in self._input.get_elements():
+            if isinstance(element, Repository):
+                user_input = (
+                    input(
+                        f"Do you want to keep the project with ID {element.get_id()}? (yes/no/stop): "
+                    )
+                    .strip()
+                    .lower()
+                )
+                if user_input in {"yes", "y"}:
+                    self._output.add_element(element)
+                elif user_input in {"stop"}:
+                    return self
+            else:
+                raise RuntimeError(
+                    "Interactive manual sampling not implemented for sets"
+                )
+
+        return self

sampling_mining_workflows_dsl/operator/selection/sampling/manual/ManualSamplingOperator.py

@@ -0,0 +1,28 @@
+from typing import TypeVar
+
+from sampling_mining_workflows_dsl.element.Repository import Repository
+from sampling_mining_workflows_dsl.element.Set import Set
+from sampling_mining_workflows_dsl.operator.Operator import Operator
+from sampling_mining_workflows_dsl.operator.selection.sampling.SamplingOperator import (
+    SamplingOperator,
+)
+
+T = TypeVar("T")
+
+
+class ManualSamplingOperator[T](SamplingOperator):
+    def __init__(self, workflow, *ids: T):
+        super().__init__(workflow, len(ids))
+        self.ids = ids
+
+    def execute(self) -> Operator:
+        self._output = Set()
+
+        for element in self._input.get_elements():
+            if isinstance(element, Repository):
+                if any(element.get_id() == id_ for id_ in self.ids):
+                    self._output.add_element(element)
+            else:
+                raise RuntimeError("Manual sampling not implemented for sets")
+        super().execute()
+        return self

sampling_mining_workflows_dsl/operator/set_algebra/ExternalSetOperator.py

@@ -0,0 +1,33 @@
+from typing import TypeVar
+
+from sampling_mining_workflows_dsl.element.Loader import Loader
+from sampling_mining_workflows_dsl.element.Repository import Repository
+from sampling_mining_workflows_dsl.element.Set import Set
+from sampling_mining_workflows_dsl.operator.Operator import Operator
+from sampling_mining_workflows_dsl.operator.selection.sampling.SamplingOperator import (
+    SamplingOperator,
+)
+
+T = TypeVar("T")
+
+
+class ExternalSetOperator(Operator):
+    set_function = None
+    def __init__(self, workflow,loader:Loader):
+        self.loader = loader
+        super().__init__(workflow)
+        
+      
+    def execute(self) -> Operator:
+        if not self._output.size()==0:
+            print("Warning: output set is not empty, clearing it")
+            self.output_set.remove_all_elements()
+        set= self.loader.load()
+        self.output_set.add(self.input_set)
+        self.set_function(self.output_set,set)
+        super().execute()
+        return self
+    
+        
+    
+   

sampling_mining_workflows_dsl/operator/set_algebra/InternalSetOperator.py

@@ -0,0 +1,47 @@
+from typing import TypeVar
+
+from sampling_mining_workflows_dsl.element.Repository import Repository
+from sampling_mining_workflows_dsl.element.Set import Set
+from sampling_mining_workflows_dsl.operator.Operator import Operator
+from sampling_mining_workflows_dsl.operator.selection.sampling.SamplingOperator import (
+    SamplingOperator,
+)
+
+T = TypeVar("T")
+
+
+class InternalSetOperator(Operator):
+    set_function = None
+    def __init__(self, workflow,indexes=[-1]):
+        self.indexes=indexes
+        super().__init__(workflow)
+        
+      
+    def execute(self) -> Operator:
+
+        if self._input.get_depth() < 2:
+            raise ValueError(f" Internal set Operator need a set of depth >= 2")
+        if not self._output:
+            self._output = Set()
+        if not self._output.size()==0:
+            print("Warning: output set is not empty, clearing it")
+            self._output.remove_all_elements()
+
+        if self.indexes == [-1]:
+            self.indexes = list(range(self._input.size()))
+
+        #get the first set and delete it from the list
+        set_res = self._input.get_element_by_index(self.indexes.pop(0)).clone()
+        
+        for i in self.indexes:
+            set = self._input.get_element_by_index(i)
+            if not isinstance(set, Set):
+                raise ValueError("Internal set Operator need need a set of depth >= 2")
+            self.set_function(set_res,set)
+        for element in set_res.elements.values():
+            self._output.add_element(element)
+        super().execute()
+        return self
+    
+    
+   

sampling_mining_workflows_dsl/operator/set_algebra/SetOperator.py

@@ -0,0 +1,30 @@
+from typing import TypeVar
+
+from sampling_mining_workflows_dsl.element.Repository import Repository
+from sampling_mining_workflows_dsl.element.Set import Set
+from sampling_mining_workflows_dsl.operator.Operator import Operator
+from sampling_mining_workflows_dsl.operator.selection.sampling.SamplingOperator import (
+    SamplingOperator,
+)
+
+T = TypeVar("T")
+
+
+class SetOperator(Operator):
+    set_function = None
+    def __init__(self, workflow,other:Set):
+        self.other = other
+        super().__init__(workflow)
+        
+      
+    def execute(self) -> Operator:
+        if not self._output.size()==0:
+            print("Warning: output set is not empty, clearing it")
+            self.output_set.remove_all_elements()
+
+        self.output_set.add(self.input_set)
+        self.set_function(self.output_set,set)
+        super().execute()
+        return self
+    
+   

sampling_mining_workflows_dsl/operator/set_algebra/external_set_operator/DifferenceOperator.py

@@ -0,0 +1,17 @@
+from typing import TypeVar
+
+from sampling_mining_workflows_dsl.element.Repository import Repository
+from sampling_mining_workflows_dsl.element.Set import Set
+from sampling_mining_workflows_dsl.operator.Operator import Operator
+from sampling_mining_workflows_dsl.operator.selection.sampling.SamplingOperator import (
+    SamplingOperator,
+)
+from sampling_mining_workflows_dsl.operator.set_algebra import ExternalSetOperator
+
+T = TypeVar("T")
+
+
+class DifferenceOperator(ExternalSetOperator):
+        def __init__(self, workflow,indexes=[-1]):
+            super().__init__(workflow)
+            self.set_function = Set.difference      

sampling_mining_workflows_dsl/operator/set_algebra/external_set_operator/IntersectionOperator.py

@@ -0,0 +1,19 @@
+from typing import TypeVar
+
+from sampling_mining_workflows_dsl.element.Repository import Repository
+from sampling_mining_workflows_dsl.element.Set import Set
+from sampling_mining_workflows_dsl.operator.Operator import Operator
+from sampling_mining_workflows_dsl.operator.selection.sampling.SamplingOperator import (
+    SamplingOperator,
+)
+from sampling_mining_workflows_dsl.operator.set_algebra import ExternalSetOperator
+from sampling_mining_workflows_dsl.operator.set_algebra.InternalSetOperator import InternalSetOperator
+
+T = TypeVar("T")
+
+
+class IntersectionOperator(ExternalSetOperator):
+    def __init__(self, workflow,indexes=[-1]):
+        super().__init__(workflow)
+        self.set_function = Set.intersection      
+    

sampling_mining_workflows_dsl/operator/set_algebra/external_set_operator/UnionOperator.py

@@ -0,0 +1,17 @@
+from typing import TypeVar
+
+from sampling_mining_workflows_dsl.element.Repository import Repository
+from sampling_mining_workflows_dsl.element.Set import Set
+from sampling_mining_workflows_dsl.operator.Operator import Operator
+from sampling_mining_workflows_dsl.operator.selection.sampling.SamplingOperator import (
+    SamplingOperator,
+)
+from sampling_mining_workflows_dsl.operator.set_algebra import ExternalSetOperator
+
+T = TypeVar("T")
+
+
+class UnionOperator(ExternalSetOperator):
+    def __init__(self, workflow, indexes=[-1]):
+        super().__init__(workflow)
+        self.set_function = Set.union

sampling_mining_workflows_dsl/operator/set_algebra/internal_set_operator/DifferenceOperator.py

@@ -0,0 +1,17 @@
+from typing import TypeVar
+
+from sampling_mining_workflows_dsl.element.Repository import Repository
+from sampling_mining_workflows_dsl.element.Set import Set
+from sampling_mining_workflows_dsl.operator.Operator import Operator
+from sampling_mining_workflows_dsl.operator.selection.sampling.SamplingOperator import (
+    SamplingOperator,
+)
+from sampling_mining_workflows_dsl.operator.set_algebra.InternalSetOperator import InternalSetOperator
+
+T = TypeVar("T")
+
+
+class DifferenceOperator(InternalSetOperator):
+        def __init__(self, workflow,indexes=[-1]):
+            super().__init__(workflow)
+            self.set_function = Set.difference      

sampling_mining_workflows_dsl/operator/set_algebra/internal_set_operator/IntersectionOperator.py

@@ -0,0 +1,18 @@
+from typing import TypeVar
+
+from sampling_mining_workflows_dsl.element.Repository import Repository
+from sampling_mining_workflows_dsl.element.Set import Set
+from sampling_mining_workflows_dsl.operator.Operator import Operator
+from sampling_mining_workflows_dsl.operator.selection.sampling.SamplingOperator import (
+    SamplingOperator,
+)
+from sampling_mining_workflows_dsl.operator.set_algebra.InternalSetOperator import InternalSetOperator
+
+T = TypeVar("T")
+
+
+class IntersectionOperator(InternalSetOperator):
+    def __init__(self, workflow,indexes=[-1]):
+        super().__init__(workflow)
+        self.set_function = Set.intersection      
+    

sampling_mining_workflows_dsl/operator/set_algebra/internal_set_operator/UnionOperator.py

@@ -0,0 +1,17 @@
+from typing import TypeVar
+
+from sampling_mining_workflows_dsl.element.Repository import Repository
+from sampling_mining_workflows_dsl.element.Set import Set
+from sampling_mining_workflows_dsl.operator.Operator import Operator
+from sampling_mining_workflows_dsl.operator.selection.sampling.SamplingOperator import (
+    SamplingOperator,
+)
+from sampling_mining_workflows_dsl.operator.set_algebra.InternalSetOperator import InternalSetOperator
+
+T = TypeVar("T")
+
+
+class UnionOperator(InternalSetOperator):
+    def __init__(self, workflow, indexes=[-1]):
+        super().__init__(workflow)
+        self.set_function = Set.union

sampling_mining_workflows_dsl/operator/set_algebra/set_operator/DifferenceOperator.py

@@ -0,0 +1,15 @@
+from typing import TypeVar
+
+from sampling_mining_workflows_dsl.element.Repository import Repository
+from sampling_mining_workflows_dsl.element.Set import Set
+from sampling_mining_workflows_dsl.operator.Operator import Operator
+
+from sampling_mining_workflows_dsl.operator.set_algebra.SetOperator import SetOperator
+
+T = TypeVar("T")
+
+
+class DifferenceOperator(SetOperator):
+        def __init__(self, workflow,indexes=[-1]):
+            super().__init__(workflow)
+            self.set_function = Set.difference      

sampling_mining_workflows_dsl/operator/set_algebra/set_operator/IntersectionOperator.py

@@ -0,0 +1,18 @@
+from typing import TypeVar
+
+from sampling_mining_workflows_dsl.element.Repository import Repository
+from sampling_mining_workflows_dsl.element.Set import Set
+from sampling_mining_workflows_dsl.operator.Operator import Operator
+from sampling_mining_workflows_dsl.operator.selection.sampling.SamplingOperator import (
+    SamplingOperator,
+)
+from sampling_mining_workflows_dsl.operator.set_algebra.SetOperator import SetOperator
+
+T = TypeVar("T")
+
+
+class IntersectionOperator(SetOperator):
+    def __init__(self, workflow,indexes=[-1]):
+        super().__init__(workflow)
+        self.set_function = Set.intersection      
+    

sampling_mining_workflows_dsl/operator/set_algebra/set_operator/UnionOperator.py

@@ -0,0 +1,17 @@
+from typing import TypeVar
+
+from sampling_mining_workflows_dsl.element.Repository import Repository
+from sampling_mining_workflows_dsl.element.Set import Set
+from sampling_mining_workflows_dsl.operator.Operator import Operator
+from sampling_mining_workflows_dsl.operator.selection.sampling.SamplingOperator import (
+    SamplingOperator,
+)
+from sampling_mining_workflows_dsl.operator.set_algebra.SetOperator import SetOperator
+
+T = TypeVar("T")
+
+
+class UnionOperator(SetOperator):
+    def __init__(self, workflow, indexes=[-1]):
+        super().__init__(workflow)
+        self.set_function = Set.union

sampling_mining_workflows_dsl/test/Workflow_simple.py

@@ -0,0 +1,38 @@
+from sampling_mining_workflows_dsl.element.loader.LoaderFactory import LoaderFactory
+from sampling_mining_workflows_dsl.element.writer.WriterFactory import WritterFactory
+from sampling_mining_workflows_dsl.exec_visualizer.WorkflowVisualizer import WorkflowVisualizer
+from sampling_mining_workflows_dsl.metadata.Metadata import Metadata
+from sampling_mining_workflows_dsl.operator.clustering.SubWorkflowOperatorBuilder import (
+    SubWorkflowOperatorBuilder,
+)
+from sampling_mining_workflows_dsl.WorkflowBuilder import WorkflowBuilder
+
+json_loader = LoaderFactory.json_loader
+json_writer = WritterFactory.json_writer
+filter_operator = SubWorkflowOperatorBuilder.filter_operator
+
+
+def main():
+    import os
+
+    input_path = os.path.join(os.path.dirname(__file__), "input.json")
+
+    language = Metadata.of_string("language")
+    id_ = Metadata.of_string("id")
+    commit_nb = Metadata.of_integer("commitNb")
+
+    w = (
+        WorkflowBuilder()
+        .input(json_loader(input_path, id_, commit_nb, language))
+        .filter_operator("commitNb > 2000 and commitNb < 7000")
+        .random_selection_operator(40)
+        .output(json_writer("out.json"))
+    )
+
+
+    w.execute_workflow()
+    
+
+if __name__ == "__main__":
+    main()
+