sdk-seshat-python 0.3.4__py3-none-any.whl

seshat/transformer/deriver/from_database.py

@@ -0,0 +1,66 @@
+from typing import Dict, Callable
+
+from pandas import DataFrame
+from pyspark.sql import DataFrame as PySparkDataFrame
+
+from seshat.source.database import SQLDBSource
+from seshat.transformer.deriver.base import SFrameDeriver
+
+
+class FromSQLDBDeriver(SFrameDeriver):
+    """
+    This transformer derive new column by fetching data from sql database source find
+    """
+
+    DEFAULT_GROUP_KEYS = {"default": "default", "result": "result"}
+
+    def __init__(
+        self,
+        source: SQLDBSource,
+        base_col: str = None,
+        query: str = None,
+        get_query_fn: Callable = None,
+        get_query_fn_kwargs=None,
+        filters: dict = None,
+        merge_result: bool = True,
+        merge_how: str = "left",
+        group_keys=None,
+        *args,
+        **kwargs,
+    ):
+        super().__init__(group_keys, *args, **kwargs)
+        if get_query_fn_kwargs is None:
+            get_query_fn_kwargs = {}
+        if filters is None:
+            filters = {}
+        self.get_query_fn_kwargs = get_query_fn_kwargs
+        self.source = source
+        self.get_query_fn = get_query_fn
+        self.query = query
+        self.filters = filters
+        self.base_col = base_col
+        self.merge_result = merge_result
+        self.merge_how = merge_how
+
+    def derive_df(self, default: DataFrame, *args, **kwargs) -> Dict["str", DataFrame]:
+        db_result = self.get_from_source(default, *args, **kwargs)
+        if self.merge_result:
+            default = default.merge(
+                right=db_result, on=self.base_col, how=self.merge_how
+            )
+            return {"default": default}
+        return {"default": default, "result": db_result}
+
+    def derive_spf(self, default: PySparkDataFrame, *args, **kwargs):
+        db_result = self.get_from_source(default, *args, **kwargs)
+        if self.merge_result:
+            default = default.join(db_result, on=self.base_col, how=self.merge_how)
+            return {"default": default}
+        return {"default": default, "result": db_result}
+
+    def get_from_source(self, default, *args, **kwargs):
+        self.source.query = self.query or self.get_query_fn(
+            default, *args, **kwargs, **self.get_query_fn_kwargs
+        )
+        self.source.filters = self.filters
+        return self.source().to_raw()

seshat/transformer/imputer/base.py

@@ -0,0 +1,6 @@
+from seshat.transformer import Transformer
+
+
+class Imputer(Transformer):
+    def impute(self):
+        pass

seshat/transformer/merger/__init__.py

@@ -0,0 +1 @@
+from .base import Merger

seshat/transformer/merger/base.py

@@ -0,0 +1,221 @@
+from typing import Iterable, List, Dict
+from typing import Type
+
+from pyspark.sql import DataFrame as PySparkDataFrame
+from pyspark.sql.dataframe import DataFrame
+
+from seshat.data_class import DFrame, GroupSFrame
+from seshat.data_class import SFrame, SPFrame
+from seshat.general import configs
+from seshat.general.exceptions import InvalidArgumentsError
+from seshat.transformer import Transformer
+from seshat.transformer.schema import Schema
+
+
+class SFrameMerger(Transformer):
+    HANDLER_NAME = "merge"
+    ONLY_GROUP = True
+
+
+class Merger(SFrameMerger):
+    """
+    This transformer merge two sframe based on columns names.
+
+    Parameters:
+    axis: int, optional
+        Show that merging operation should operate on which axis. Default is 1.
+    left_on: str, optional
+        Column name that is base of merging for default sf
+    right_on: str, optional
+        Column name that is base of merging for right sf
+    on: str, optional
+        Column name that is base of merging for both default & right sf.
+        This is same when left_on and right_on are same.
+    right_schema: Schema, optional
+        This schema will be ran on right sf before merging happened.
+        Useful when keep only some columns of right sf in default sf is required.
+    inplace: bool, optional
+        Show that weather default sf should changed or merged result save on new sf.
+        Default is False.
+    merge_how: str, optional
+        Show how to merge two sframes. For example: left, right, inner.
+        Default is left.
+    drop_unmerged: bool, optional
+        If true, two sframe that used for merging will drop from result.
+    group_key
+        Group keys for the parent Transformer class.
+    *args
+        Additional positional arguments.
+    **kwargs
+        Additional keyword arguments.
+    """
+
+    DEFAULT_GROUP_KEYS = {
+        "default": configs.DEFAULT_SF_KEY,
+        "other": configs.OTHER_SF_KEY,
+        "merged": configs.MERGED_SF_KEY,
+    }
+    on: str | Iterable[str]
+    axis: int
+
+    def __init__(
+        self,
+        axis: int = 1,
+        left_on: str | List[str] = None,
+        right_on: str | List[str] = None,
+        on: str | List[str] = None,
+        right_schema: Schema = None,
+        inplace: bool = False,
+        group_keys=None,
+        merge_how: str = "left",
+        drop_unmerged: bool = False,
+        *args,
+        **kwargs,
+    ):
+        if isinstance(left_on, list) != isinstance(right_on, list):
+            raise InvalidArgumentsError(
+                "left_on and right_on must both be either lists or strings and cannot be of different types."
+            )
+        if isinstance(left_on, list) and len(left_on) != len(right_on):
+            raise InvalidArgumentsError("len(right_on) must equal len(left_on)")
+
+        super().__init__(group_keys, *args, **kwargs)
+        self.axis = axis
+        self.left_on = left_on
+        self.right_on = right_on
+        self.on = on
+        self.right_schema = right_schema
+        self.inplace = inplace
+        self.merge_how = merge_how
+        self.drop_unmerged = drop_unmerged
+
+    def set_raw(self, sf: GroupSFrame, result: Dict[str, object]):
+        if self.drop_unmerged:
+            for _, v in self.group_keys.items():
+                sf.children.pop(v, None)
+        return super().set_raw(sf, result)
+
+    def merge_df(self, default: DataFrame, other: DataFrame, *args, **kwargs):
+        merging_kwargs = self.get_merging_kwargs()
+        merged = self.merge(default, other, DFrame, merging_kwargs)
+        if self.axis == 1 and "right_on" in merging_kwargs:
+            merged = merged.drop(self.get_drop_cols(), axis=1)
+        merged = merged.drop_duplicates().reset_index(drop=True)
+        return self.choose_return_result(default, other, merged)
+
+    def merge_spf(
+        self, default: PySparkDataFrame, other: PySparkDataFrame, *args, **kwargs
+    ):
+        merging_kwargs = self.get_merging_kwargs()
+        merged = self.merge(default, other, SPFrame, merging_kwargs)
+        if self.axis == 1 and "right_on" in merging_kwargs:
+            merged: PySparkDataFrame = merged.drop(self.get_drop_cols())
+        merged = merged.dropDuplicates()
+        return self.choose_return_result(default, other, merged)
+
+    def merge(
+        self,
+        default: object,
+        other: object,
+        sf_class: Type[SFrame],
+        merging_kwargs: dict,
+    ) -> object:
+        default_sf = sf_class.from_raw(default)
+        other_sf = sf_class.from_raw(other)
+        if self.right_schema:
+            other_sf = self.right_schema(other_sf)
+        result = default_sf.extend(other_sf.to_raw(), axis=self.axis, **merging_kwargs)
+        return result
+
+    def get_merging_kwargs(self):
+        if self.on:
+            return {"on": self.on, "how": self.merge_how}
+        return {
+            "left_on": self.left_on,
+            "right_on": self.right_on,
+            "how": self.merge_how,
+        }
+
+    def choose_return_result(self, default, other, merged):
+        if self.drop_unmerged:
+            result_kwargs = {}
+        else:
+            result_kwargs = {"default": default, "other": other}
+
+        if self.inplace:
+            result_kwargs["default"] = merged
+        else:
+            result_kwargs["merged"] = merged
+        return result_kwargs
+
+    def get_drop_cols(self):
+        left_on_set = (
+            {self.left_on} if isinstance(self.left_on, str) else set(self.left_on)
+        )
+        right_on_set = (
+            {self.right_on} if isinstance(self.right_on, str) else set(self.right_on)
+        )
+        return right_on_set - left_on_set
+
+
+class ListMerger(SFrameMerger):
+    """
+    Get the list of sframe either they are group or non-group and at last return
+    only one group with all input sf.
+    To avoid duplication in sf keys in result group sf, this format have been used:
+    If sf is group, then key is prefix + index of sf in input sf_list + child key
+    Otherwise key is prefix + index of sf in input sf_list
+
+    Parameters:
+    ----------
+    sf_prefix: str
+        The prefix that all sf keys in output starts with it.
+    sf_list: List[SFrame]
+        The List of sframes, each one can be group or non-group. The merging operation
+        is performed on this list.
+
+    Examples
+    --------
+    >>> sf1: SFrame, sf2: GroupSFrame
+    >>> list(sf2.keys)
+    ['default', 'address']
+
+    >>> merger = ListMerger(sf_prefix="sf_")
+    >>> result_sf = merger(sf_list=[sf1, sf2])
+    >>> list(result_sf.keys)
+    ['sf_0', 'sf_1_default', 'sf_1_address']
+    """
+
+    def __init__(self, sf_prefix="sf", *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.sf_prefix = sf_prefix
+
+    def __call__(self, sf_list: List[SFrame], *args, **kwargs):
+        result_sf = GroupSFrame()
+        for i, sf in enumerate(sf_list):
+            if isinstance(sf, GroupSFrame):
+                for child_key, child_sf in sf.children.items():
+                    result_sf[f"{self.sf_prefix}{i}_{child_key}"] = child_sf
+            else:
+                result_sf[f"{self.sf_prefix}{i}"] = sf
+        return result_sf
+
+
+class MultiMerger(SFrameMerger):
+    """
+    This merger apply multiple merger on input sf.
+    """
+
+    mergers = List[Merger]
+    ONLY_GROUP = True
+
+    def __init__(self, mergers, group_keys=None, *args, **kwargs):
+        super().__init__(group_keys, *args, **kwargs)
+        self.mergers = mergers
+
+    def __call__(
+        self, sf_input: GroupSFrame, *args: object, **kwargs: object
+    ) -> SFrame:
+        for merger in self.mergers:
+            sf_input = merger(sf_input)
+        return sf_input

seshat/transformer/pipeline/__init__.py

@@ -0,0 +1 @@
+from .base import Pipeline

seshat/transformer/pipeline/base.py

@@ -0,0 +1,60 @@
+from typing import List
+
+from seshat.transformer import Transformer
+
+
+class Pipeline(Transformer):
+    """
+    A data processing pipeline that sequentially applies a list of transformers to input data.
+    Each transformer in the `pipes` list processes the data and passes the output to the next transformer
+    in the sequence. The result from the last transformer in the list is the final output of the pipeline.
+
+    The transformers in the `pipes` list must be capable of applying a transformation and
+    producing an output that can be handled by the next transformer in the list, if there is one.
+    This allows for a flexible and modular design where different transformations can be chained
+    together to achieve complex data processing workflows.
+
+    Parameters
+    ----------
+    pipes : list of Transformer instances
+        A list of transformer objects through which the data will be passed in sequence.
+        The output of one transformer becomes the input to the next.
+
+    Examples
+    --------
+    >>> pipeline = Pipeline(pipes=[transformer1, transformer2]
+    >>> pipeline(input_data)
+
+    Notes
+    -----
+    This pipeline design is particularly useful for data transformations where multiple
+    discrete processing steps are required. Each transformer should be designed to handle
+    the output from the previous transformer in the sequence, ensuring compatibility between transformations.
+    """
+
+    pipes: List[Transformer]
+
+    def __init__(self, pipes: List[Transformer]):
+        self.pipes = pipes
+
+    def __call__(self, data, *args, **kwargs):
+        result = data
+        for pipe in self.pipes:
+            result = pipe(result, *args, **kwargs)
+        return result
+
+    def replace(self, transformer: Transformer, index: int):
+        self.pipes[index] = transformer
+        return self
+
+    def append(self, transformer: Transformer):
+        self.pipes.append(transformer)
+        return self
+
+    def insert(self, transformer: Transformer, index: int):
+        self.pipes.insert(index, transformer)
+        return self
+
+    def remove(self, index: int):
+        self.pipes.pop(index)
+        return self

seshat/transformer/pipeline/branch.py

@@ -0,0 +1,162 @@
+from concurrent.futures.process import ProcessPoolExecutor
+from typing import List, Dict
+
+from seshat.data_class import SFrame, GroupSFrame
+from seshat.general.config import DEFAULT_SF_KEY
+from seshat.general.exceptions import InvalidArgumentsError
+from seshat.transformer import Transformer
+from seshat.transformer.merger import Merger
+from seshat.transformer.pipeline import Pipeline
+
+
+def process_pipeline(sf, sf_key, pipe, index, *args, **kwargs):
+    if index is not None:
+        return sf_key + f"__{index}", pipe(sf.get(sf_key), *args, **kwargs)
+    return sf_key, pipe(sf.get(sf_key), *args, **kwargs)
+
+
+class Branch(Transformer):
+    """
+    Branch is a Transformer that runs multiple pipelines simultaneously and merges their outputs.
+    It can execute pipelines in parallel using multiple processes.
+
+    Parameters
+    ----------
+    pipe_map : Dict[str, Pipeline | List[Pipeline]], optional
+        A dictionary mapping SFrames to pipelines or lists of pipelines.
+    merger : Merger, optional
+        An object to merge the pipeline outputs.
+    parallel : bool, optional
+        Whether to run pipelines in parallel. Default is False.
+    max_workers : int, optional
+        Maximum number of workers for parallel execution. Default is 16.
+    group_keys
+        Group keys for the parent Transformer class.
+    *args
+        Additional positional arguments.
+    **kwargs
+        Additional keyword arguments.
+
+    Example usage
+    ----------
+    1. Creating a Branch instance with a list of pipelines:
+
+        >>> sf_input = GroupSFrame(children={"default": DFrame(transaction_df), "address": DFrame(address_df)})
+        ... pipe_map = {
+        ... "default": [Pipeline(pipes=[ZeroAddressTrimmer(), LowTransactionTrimmer()]),
+        ...             Pipeline(pipes=[OperationOnColsDeriver(
+        ...                 cols=("sent_count", "received_count"),
+        ...                 result_col="tx_count",
+        ...                 agg_func="sum",
+        ...                 is_numeric=True,
+        ...             )])],
+        ... }
+        ... branch = Branch(pipe_map=pipe_map, merger=Merger())
+        ... result = branch(sf_input)
+
+    2. Creating a Branch instance with a single pipeline for each SFrame:
+
+        >>> sf_input = GroupSFrame(children={"default": DFrame(transaction_df), "address": DFrame(address_df)})
+        ... pipe_map = {
+        ...     "default": Pipeline(pipes=[ZeroAddressTrimmer(), LowTransactionTrimmer()]),
+        ...     "address": Pipeline(pipes=[FeatureForAddressDeriver(
+        ...         value_col="ez_token_transfers_id",
+        ...         result_col="received_count",
+        ...         default_index_col="to_address",
+        ...         agg_func="nunique",
+        ...         is_numeric=False,
+        ...     )])
+        ... }
+        ... branch = Branch(pipe_map=pipe_map, merger=Merger())
+        ... result = branch(sf_input)
+    """
+
+    merger: Merger
+    parallel: bool
+    max_workers: int
+    HANDLER_NAME = "run"
+
+    def __init__(
+        self,
+        pipe_map: Dict[str, Pipeline | List[Pipeline]] = None,
+        merger: Merger = None,
+        parallel: bool = False,
+        max_workers: int = 16,
+        group_keys=None,
+        *args,
+        **kwargs,
+    ):
+        super().__init__(group_keys, *args, **kwargs)
+
+        self.pipe_map = pipe_map
+        self.merger = merger
+        self.parallel = parallel
+        self.max_workers = max_workers
+
+    def __call__(self, sf_input: SFrame, *args, **kwargs) -> SFrame:
+        result = super().__call__(sf_input, *args, **kwargs)
+        if self.merger:
+            result = self.merger(result)
+        return result
+
+    def call_handler(self, sf: SFrame, *args, **kwargs) -> Dict[str, object]:
+        return self.run(sf, *args, **kwargs)
+
+    def run(self, sf: SFrame, *args, **kwargs):
+        if isinstance(sf, GroupSFrame):
+            if set(self.pipe_map.keys()).difference(set(sf.children.keys())):
+                raise InvalidArgumentsError(
+                    "Pipeline map keys must exist in passed SFrame"
+                )
+        else:
+            sf = sf.make_group(DEFAULT_SF_KEY)
+        result = (
+            self.run_parallel_pipes(sf, *args, **kwargs)
+            if self.parallel
+            else self.run_pipes(sf, *args, **kwargs)
+        )
+        return result
+
+    def run_pipes(self, sf: SFrame, *args, **kwargs):
+        results = {}
+        for sf_key, pipes in self.pipe_map.items():
+            if isinstance(pipes, list):
+                for i in range(len(pipes)):
+                    results[sf_key + f"__{i}"] = pipes[i](
+                        sf.get(sf_key), *args, **kwargs
+                    )
+            else:
+                results[sf_key] = pipes(sf.get(sf_key), *args, **kwargs)
+        return results
+
+    def run_parallel_pipes(self, sf: SFrame, *args, **kwargs):
+        results = {}
+
+        with ProcessPoolExecutor(max_workers=self.max_workers) as executor:
+            futures = []
+            for sf_key, pipes in self.pipe_map.items():
+                if isinstance(pipes, list):
+                    for i in range(len(pipes)):
+                        futures.append(
+                            executor.submit(
+                                process_pipeline,
+                                sf,
+                                sf_key,
+                                pipes[i],
+                                i,
+                                *args,
+                                **kwargs,
+                            )
+                        )
+                else:
+                    futures.append(
+                        executor.submit(
+                            process_pipeline, sf, sf_key, pipes, None, *args, **kwargs
+                        )
+                    )
+
+            for future in futures:
+                key, result = future.result()
+                results[key] = result
+
+        return results

seshat/transformer/pipeline/recommendation/address_pipeline.py

@@ -0,0 +1,79 @@
+from seshat.general import configs
+from seshat.transformer.deriver.base import (
+    SFrameFromColsDeriver,
+    FeatureForAddressDeriver,
+    OperationOnColsDeriver,
+    PercentileTransactionValueDeriver,
+    InteractedSymbolsToSentenceDeriver,
+)
+from seshat.transformer.pipeline import Pipeline
+
+address_pipeline = Pipeline(
+    [
+        SFrameFromColsDeriver(
+            cols=(configs.FROM_ADDRESS_COL, configs.TO_ADDRESS_COL),
+            result_col="address",
+        ),
+        FeatureForAddressDeriver(
+            value_col="SYMBOL",
+            is_numeric=False,
+            result_col="unique_tokens_count",
+            agg_func="count",
+        ),
+        FeatureForAddressDeriver(
+            value_col="AMOUNT_USD",
+            is_numeric=False,
+            result_col="sent_amount",
+        ),
+        FeatureForAddressDeriver(
+            value_col="AMOUNT_USD",
+            is_numeric=False,
+            address_col=configs.TO_ADDRESS_COL,
+            result_col="received_amount",
+        ),
+        FeatureForAddressDeriver(
+            value_col="SYMBOL",
+            is_numeric=False,
+            result_col="sent_symbols",
+            agg_func="unique",
+        ),
+        FeatureForAddressDeriver(
+            value_col="SYMBOL",
+            is_numeric=False,
+            address_col=configs.TO_ADDRESS_COL,
+            result_col="received_symbols",
+            agg_func="unique",
+        ),
+        FeatureForAddressDeriver(
+            value_col=configs.FROM_ADDRESS_COL,
+            is_numeric=False,
+            address_col=configs.TO_ADDRESS_COL,
+            result_col="received_partners",
+            agg_func="nunique",
+        ),
+        FeatureForAddressDeriver(
+            value_col=configs.TO_ADDRESS_COL,
+            is_numeric=False,
+            result_col="sent_partners",
+            agg_func="nunique",
+        ),
+        OperationOnColsDeriver(
+            cols=("received_amount", "sent_amount"),
+            result_col="interacted_avg_amount",
+        ),
+        OperationOnColsDeriver(
+            group_keys={"default": "address"},
+            cols=("received_amount", "sent_amount"),
+            agg_func="sum",
+            result_col="interacted_total_amount",
+        ),
+        PercentileTransactionValueDeriver(
+            group_keys={"default": "address"},
+            value_col="interacted_avg_amount",
+        ),
+        InteractedSymbolsToSentenceDeriver(
+            symbol_col="SYMBOL",
+            result_col="sentence_symbols",
+        ),
+    ]
+)

seshat/transformer/reducer/base.py

@@ -0,0 +1,6 @@
+from seshat.transformer import Transformer
+
+
+class Reducer(Transformer):
+    def reduce(self):
+        pass

seshat/transformer/scaler/base.py

@@ -0,0 +1,6 @@
+from seshat.transformer import Transformer
+
+
+class Scaler(Transformer):
+    def scale(self):
+        pass

seshat/transformer/schema/__init__.py

@@ -0,0 +1 @@
+from .base import Schema, Col