sampling-mining-workflows-dsl 0.0.1__py3-none-any.whl

sampling_mining_workflows_dsl/CompleteWorkflow.py

@@ -0,0 +1,23 @@
+from typing import TypeVar
+
+T = TypeVar("T")
+
+
+class CompleteWorkflow:
+    def __init__(self, workflow):
+        if not workflow.is_complete():
+            raise ValueError(
+                "Workflow is incomplete. Ensure it has an input, output, and at least one operator."
+            )
+        self._workflow = workflow
+
+    def execute_workflow(self):
+        self._workflow.execute_workflow()
+        return self
+
+    def analyze_workflow(self, metadata):
+        self._workflow.analyze_workflow(metadata)
+        return self
+
+    def __str__(self):
+        return str(self._workflow)

sampling_mining_workflows_dsl/Workflow.py

@@ -0,0 +1,242 @@
+from typing import TypeVar, cast
+
+from sampling_mining_workflows_dsl import CompleteWorkflow
+from sampling_mining_workflows_dsl.analysis.DistributionWorkflowAnalysis import (
+    DistributionWorkflowAnalysis,
+)
+from sampling_mining_workflows_dsl.analysis.HistWorkflowAnalysis import HistWorkflowAnalysis
+from sampling_mining_workflows_dsl.analysis.YamaneWorkflowAnalysis import YamaneWorkflowAnalysis
+from sampling_mining_workflows_dsl.constraint.Constraint import Constraint
+from sampling_mining_workflows_dsl.element.Element import Element
+from sampling_mining_workflows_dsl.element.Loader import Loader
+from sampling_mining_workflows_dsl.element.Set import Set
+from sampling_mining_workflows_dsl.element.Writer import Writer
+from sampling_mining_workflows_dsl.metadata.Metadata import Metadata
+from sampling_mining_workflows_dsl.operator.clustering.GroupingOperator import GroupingOperator
+from sampling_mining_workflows_dsl.operator.Operator import Operator
+from sampling_mining_workflows_dsl.operator.selection.filter.FilterOperator import FilterOperator
+from sampling_mining_workflows_dsl.operator.selection.sampling.automatic.RandomSelectionOperator import (
+    RandomSelectionOperator,
+)
+from sampling_mining_workflows_dsl.operator.selection.sampling.manual.ManualSamplingOperator import (
+    ManualSamplingOperator,
+)
+
+T = TypeVar("T")
+
+
+class Workflow:
+    def __init__(self):
+        self._input: Set | None = None
+        self._output: Set | None = None
+        self._output_writer: Writer | None = None
+        self._root: Operator | None = None
+        self._last_operator: Operator | None = None
+        self._metadata: list[Metadata] = []
+
+    def get_all_Metadata(self) -> list[Metadata]:
+        return self._metadata
+
+    def add_metadata_type(self, metadata: Metadata) -> "Workflow":
+        self._metadata.append(metadata)
+        return self
+
+    def grouping_operator(self, *workflows: "Workflow") -> "Workflow":
+        if not workflows:
+            raise ValueError("At least one workflow must be provided.")
+
+        # Create a GroupingOperator with the provided sub workflows
+        grouping_operator = GroupingOperator(self, *workflows)
+
+        # Add the grouping operator to the current workflow
+        self.add_operator(cast("Operator", grouping_operator))
+        return self
+
+    def random_selection_operator(self, cardinality: int, seed: int = -1) -> "Workflow":
+        random_selection_operator = RandomSelectionOperator(
+            self, cardinality=cardinality, seed=seed
+        )
+        self.add_operator(cast("Operator", random_selection_operator))
+        return self
+
+    def filter_operator(self, constraint: Constraint):
+        filter_operator = FilterOperator(self, constraint)
+        self.add_operator(cast("Operator", filter_operator))
+        return self
+
+    def manual_sampling_operator(self, *ids: T) -> "Workflow":
+        if not ids:
+            raise ValueError(
+                "At least one element must be provided for manual sampling."
+            )
+
+        manual_sampling_operator = ManualSamplingOperator(self, *ids)
+        self.add_operator(cast("Operator", manual_sampling_operator))
+        return self
+    
+    def get_all_set_from_workflow(self, index=0):
+        sets = {}
+        if index==0:
+            sets[index] = (self._input,None)
+            index += 1
+
+        op = self.get_root()
+        while op is not None:
+            if not isinstance(op, GroupingOperator):
+                if hasattr(op, "_output") and op._output is not None:
+                    sets[index] = (op._output,op)  # Fix: use index as key, not the Set object
+                    index += 1
+            else:
+                for internal_w in op.get_workflows():
+                    grouping_sets = internal_w.get_all_set_from_workflow(index)
+                    index = index + len(grouping_sets)
+                    sets.update(grouping_sets)  # Fix: use update() instead of extend()
+            op = op.get_next_operator()
+        return sets
+    
+    def get_internal_set_by_index(self,index):
+        return self.get_all_set_from_workflow()[index]
+        
+    def get_internal_set_by_id(self, set_id: str) -> Set | None:
+        current: Operator= self._root
+        while current is not None:
+            output:Set = current.get_output()
+            if not output is None and output.get_id() == set_id:
+                return output
+            if current.isinstance(GroupingOperator):
+                for w in current.get_workflows():
+                    internal_set = w.get_internal_set_by_id(set_id)
+                    if internal_set is not None:
+                        return internal_set
+            current = current._next_operator
+        return None
+
+    def input(self, loader: Loader) -> "Workflow":
+        self._metadata = list(loader.metadatas.values())
+        self._input = loader.load_set()
+        return self
+
+    def add_metadata(self, loader: Loader) -> "Workflow":
+        # Add metadata value to last declared operator
+        current_operator: Operator = self.get_last_operator()
+        if current_operator:
+            current_operator.add_metadata_loader(loader)
+        return self
+
+    def output(self, writer: Writer) -> "Workflow":
+        self._output_writer = writer
+        last_operator = self.get_last_operator()
+        if last_operator:
+            last_operator.output(writer)
+        return self
+
+    def is_complete(self) -> bool:
+        return (
+            self._root is not None
+            and self._input is not None
+            and self._output_writer is not None
+        )
+
+    def add_operator(self, operator: Operator):
+        # If the workflow is empty, set the root operator
+        if self._root is None:
+            self._root = operator
+            self._last_operator = operator
+
+        # If the workflow already has operators, append the new operator to the end
+        else:
+            self._last_operator._next_operator = operator
+            operator._previous_operator = self._last_operator
+            operator.input_set(self._last_operator.get_output())
+            self._last_operator.output_set(operator.get_output())
+            self._last_operator = operator
+
+        if self.is_complete():
+            return CompleteWorkflow(self)
+        return None
+
+    def set_workflow_input(self, input_set: Set | None) -> "Workflow":
+        self._input = input_set
+        if self._root is not None:
+            self._root.input_set(input_set)
+        return self
+
+    def set_root_input(self, input_set: Set | None) -> "Workflow":
+        if self._root is None:
+            raise ValueError("Cannot set root input when no root operator is defined.")
+        self._root.input_set(input_set)
+        return self
+
+    def get_workflow_input(self) -> Element | None:
+        return self._input
+
+    def set_workflow_output(self, output_element: Element) -> "Workflow":
+        self._output = output_element
+        return self
+
+    def get_workflow_output(self) -> Set | None:
+        return self._output
+
+    def get_root(self) -> Operator | None:
+        return self._root
+
+    def get_last_operator(self) -> Operator | None:
+        if self._root is None:
+            return None
+        current = self._root
+        while current._next_operator is not None:
+            current = current._next_operator
+        return current
+
+    def get_operator_by_position(self, position: int) -> Operator | None:
+        if position < 0:
+            return None
+        current = self._root
+        index = 0
+        while current is not None:
+            if index == position:
+                return current
+            current = current._next_operator
+            index += 1
+        return None
+
+    def execute_workflow(self) -> "Workflow":
+        root = self._root
+        root.input_set(self._input)
+        root.execute()
+        self._output = self._last_operator.get_output()
+        return self
+
+    def print(self) -> "Workflow":
+        print(self)
+        return self
+
+    def analyze_workflow(self, metadata: Metadata[T]) -> "Workflow":
+        # Perform analysis on a given metadata
+        # workflow_analysis = HistWorkflowAnalysis(metadata)
+        workflow_distrib_analysis = DistributionWorkflowAnalysis(metadata=metadata)
+        workflow_distrib_analysis.analyze(self)
+
+        workflow_yamane_analysis = YamaneWorkflowAnalysis()
+        workflow_yamane_analysis.analyze(self)
+
+        workflow_hist_analysis = HistWorkflowAnalysis(metadata=metadata)
+        workflow_hist_analysis.analyze(self)
+        return self
+
+    # --- Methods for workflow printing ---
+
+    def __str__(self) -> str:
+        return self.to_string(0)
+
+    def to_string(self, level: int):
+        indent = "    " * level
+        res = f"{indent}Workflow: [[[\n"
+        # res = ""
+        if self._root is not None:
+            res += self._root.to_string(level)
+        else:
+            res += f"{indent}No operators defined in this workflow.\n"
+
+        res += "\n" + indent + "]]]\n"
+        return res

sampling_mining_workflows_dsl/WorkflowBuilder.py

@@ -0,0 +1,11 @@
+from sampling_mining_workflows_dsl.operator.OperatorBuilder import OperatorBuilder
+from sampling_mining_workflows_dsl.Workflow import Workflow
+
+
+class WorkflowBuilder:
+    def __init__(self):
+        self.workflow = Workflow()
+
+    def input(self, loader) -> "OperatorBuilder":
+        self.workflow.input(loader)
+        return OperatorBuilder(self.workflow)

sampling_mining_workflows_dsl/analysis/ChiSquareAnalysis.py

@@ -0,0 +1,37 @@
+from collections import Counter
+
+from scipy.stats import chisquare
+
+from sampling_mining_workflows_dsl.element.Set import Set
+from sampling_mining_workflows_dsl.metadata.Metadata import Metadata
+
+class ChiSquareAnalysis:
+    def __init__(self, metadata: Metadata[str]):
+        self.metadata = metadata
+
+    def analyze(self, set_1: Set, set_2: Set):
+        # Extract keywords from both sets
+        keywords_1 = self.extract_keywords(set_1)
+        keywords_2 = self.extract_keywords(set_2)
+
+        all_labels = sorted(set(keywords_1 + keywords_2))
+
+        pop_counts = [Counter(keywords_1).get(label, 0) for label in all_labels]
+        sample_counts = [Counter(keywords_2).get(label, 0) for label in all_labels]
+
+        total_pop = sum(pop_counts)
+        total_sample = sum(sample_counts)
+        scaled_pop = [x * (total_sample / total_pop) for x in pop_counts]
+
+        chi2, p = chisquare(f_obs=sample_counts, f_exp=scaled_pop)
+
+        print("Chi2:", chi2)
+        print("p-value:", p)
+
+    def extract_keywords(self, s: Set):
+        keywords = []
+        for element in s.get_elements():
+            metadata_value = element.get_metadata_value(self.metadata)
+            if metadata_value:
+                keywords.extend(metadata_value.get_value())
+        return keywords

sampling_mining_workflows_dsl/analysis/CochranTest.py

@@ -0,0 +1,44 @@
+import math
+
+from sampling_mining_workflows_dsl.operator.selection.sampling.automatic.RandomSelectionOperator import RandomSelectionOperator
+from scipy.stats import norm
+
+class CochranTest:
+    def __init__(self, op :RandomSelectionOperator,confidence_level=0.95,margin_error=0.05,p=0.5):
+        self.op: RandomSelectionOperator  = op
+        self.confidence_level=confidence_level
+        self.e=margin_error
+        self.p=p
+    """
+        Calculate sample size using Cochran's formula.
+
+        Parameters:
+            confidence_level (float): Confidence level (e.g. 0.95 for 95%)
+            p (float): Estimated proportion of the population (use 0.5 if unknown)
+            e (float): Margin of error (e.g. 0.05 for 5%)
+            population (int or None): Population size, optional for finite correction
+
+        Returns:
+            float: Required sample size
+    """
+    def cochran_sample_size(self):
+        sampling_frame_size = self.op.get_input().size()
+
+        # Get Z-value for given confidence level
+        z = norm.ppf(1 - (1 - self.confidence_level) / 2)
+
+        # Cochran's initial sample size
+        n0 = (z**2 * self.p * (1 - self.p)) / (self.e**2)
+
+        # Adjust for finite population if given
+        if sampling_frame_size:
+            n = n0 / (1 + (n0 - 1) / sampling_frame_size)
+            return n
+        else:
+            return n0
+
+
+    def is_representative(self) -> bool:
+        required_sample_size = self.cochran_sample_size()
+        sample_size = self.op.get_cardinality()
+        return sample_size >= required_sample_size

sampling_mining_workflows_dsl/analysis/CochranWorkflowAnalysis.py

@@ -0,0 +1,37 @@
+from sampling_mining_workflows_dsl.Workflow import Workflow
+from sampling_mining_workflows_dsl.analysis.WorkflowAnalysis import WorkflowAnalysis
+from sampling_mining_workflows_dsl.operator.selection.sampling.automatic.RandomSelectionOperator import RandomSelectionOperator
+from sampling_mining_workflows_dsl.analysis.CochranTest import CochranTest
+
+
+class CochranWorkflowAnalysis(WorkflowAnalysis):
+    def __init__(self,confidence_level=0.95,margin_error=0.05,p=0.5):
+        self.confidence_level=confidence_level
+        self.margin_error=margin_error
+        self.p=p
+
+    def analyze(self,workflow : Workflow):
+        #dict set_num --> (set,op) 
+        set_op_dict = workflow.get_all_set_from_workflow()
+        for (set_nb,value) in set_op_dict.items():
+            set_,op = value
+            if isinstance(op,RandomSelectionOperator):
+                print(f"Set #{set_nb} produced by a random selection operator, checking sample size")
+                cochran_test = CochranTest(op,self.confidence_level,self.margin_error,self.p)
+                required_sample_size = cochran_test.cochran_sample_size()
+                actual_sample_size = op.get_cardinality()
+                print(f"Cochran's Test Analysis for set {set_nb}:")
+                print("Required Sample Size:", required_sample_size)
+                print("Actual Sample Size:", actual_sample_size)
+
+                if cochran_test.is_representative():
+                    print("The sample pass Cochran's test.")
+                else:
+                    print("The sample do not pass Cochran's test.")
+
+                print("------------------------------------------")
+
+        
+ 
+    
+        

sampling_mining_workflows_dsl/analysis/CoverageTest.py

@@ -0,0 +1,60 @@
+from collections import Counter
+
+from sampling_mining_workflows_dsl.element.Set import Set
+from sampling_mining_workflows_dsl.metadata.Metadata import Metadata
+
+
+class CoverageTest:
+    def __init__(self, metadata: Metadata, set_1: Set, set_2: Set):
+        self.metadata = metadata
+        self.set_1 = set_1
+        self.set_2 = set_2
+
+    def compute_coverage(self, top_x: int = None) -> float:
+        counter = Counter()
+        for element in self.set_1.get_elements():
+            values = element.get_metadata_value(self.metadata).get_value()
+            counter.update(self.flatten(values))
+
+        if top_x is not None:
+            top_values = {item for item, _ in counter.most_common(top_x)}
+        else:
+            top_values = set(counter.elements())
+
+        set_1_unique = {
+            v
+            for element in self.set_1.get_elements()
+            for v in self.flatten(element.get_metadata_value(self.metadata).get_value())
+            if v in top_values
+        }
+
+        set_2_unique = {
+            v
+            for element in self.set_2.get_elements()
+            for v in self.flatten(element.get_metadata_value(self.metadata).get_value())
+        }
+
+        if not set_1_unique:
+            print(
+                f"No value for {self.metadata.name}. Couverture = 0.0"
+            )
+            return 0.0
+
+        intersection = set_1_unique.intersection(set_2_unique)
+        coverage = len(intersection) / len(set_1_unique)
+
+        print(
+            f"Coverage for {self.metadata.name} (top {top_x if top_x is not None else 'all'}): {coverage:.2f}"
+        )
+        return coverage
+
+    def flatten(self, values):
+        if isinstance(values, list):
+            result = []
+            for v in values:
+                if isinstance(v, list):
+                    result.extend(v)
+                else:
+                    result.append(v)
+            return result
+        return [values]

sampling_mining_workflows_dsl/analysis/DistributionWorkflowAnalysis.py

@@ -0,0 +1,57 @@
+
+from typing import TYPE_CHECKING
+
+from sampling_mining_workflows_dsl.analysis.kolmogorov_smirnov import kolmogorov_smirnov
+from sampling_mining_workflows_dsl.metadata.Metadata import Metadata
+from sampling_mining_workflows_dsl.operator.clustering.GroupingOperator import GroupingOperator
+
+if TYPE_CHECKING:
+    from sampling_mining_workflows_dsl.element.Set import Set
+
+
+class DistributionWorkflowAnalysis:
+    def __init__(self, metadata: Metadata[int]):
+        self.map: dict[str, Set] = {}
+        self.metadata = metadata
+
+    def analyze(self, workflow):
+        # Retrieve the root operator from the workflow
+        root_operator = workflow.get_root()
+        self.populate_map(root_operator, "")
+        analyzer = kolmogorov_smirnov(self.metadata)
+        entries = list(self.map.items())
+
+        print("Kolmogorov-Smirnov Test:")
+
+        for i in range(len(entries) - 1):
+            for j in range(i + 1, len(entries)):
+                name = f"{entries[i][0]} /// {entries[j][0]}"
+                result = analyzer.analyze(entries[i][1], entries[j][1])
+
+                print(name)
+                print(result)
+                print("------------------------------------------")
+
+    def populate_map(self, workflow, prefix: str):
+        op = workflow.get_workflow_root_operator()
+        self.map[f"{prefix}input"] = op.get_input()
+        count = 1
+
+        while op is not None:
+            if isinstance(op, GroupingOperator):
+                grouping_operator: GroupingOperator = op
+                self.map[f"{prefix}operator_{count}_merged"] = (
+                    grouping_operator.get_merged_output()
+                )
+
+                child_count = 1
+                for internal_w in grouping_operator.get_workflows():
+                    self.populate_map(
+                        internal_w, f"{prefix}operator_{count}_Child_{child_count}_"
+                    )
+                    child_count += 1
+            else:
+                self.map[f"{prefix}operator_{count}"] = op.get_output()
+
+            count += 1
+            op = op.get_next_operator()