graphreduce 1.7.3__py3-none-any.whl

graphreduce/__main__.py

@@ -0,0 +1,15 @@
+#!/usr/bin/env python
+
+import sys
+
+from .cli.entry_point import entrypoint_cli
+
+
+def main():
+    if sys.version_info[:3] == (3, 8):
+        pass
+    entrypoint_cli()
+
+
+if __name__ == '__main__':
+    main()

graphreduce/cli/auto_fe.py

@@ -0,0 +1,85 @@
+#!/usr/bin/env python
+import sqlite3
+import json
+import os
+import typing
+import datetime
+
+import typer
+from typer import Argument, Option
+import pandas as pd
+
+# examples for using SQL engines and dialects
+from graphreduce.node import SQLNode, DynamicNode
+from graphreduce.graph_reduce import GraphReduce
+from graphreduce.enum import SQLOpType, ComputeLayerEnum, PeriodUnit
+from graphreduce.models import sqlop
+from graphreduce.context import method_requires
+
+
+auto_fe_cli = typer.Typer(name="auto_fe", help="Perform automated feature engineering", no_args_is_help=True)
+
+
+@auto_fe_cli.command("autofefs")
+def autofe_filesystem (
+            data_path: str = Argument(help="Path to data"),
+            fmt: str = Argument(help="File format"),
+            prefixes: str = Argument(help="json dict of filenames with prefixes (e.g., `{'test.csv':'test'}`)"),
+            date_keys: str = Argument(help="json dict of filenames with associated date key (e.g., `{'test.csv': 'ts'}`)"),
+            relationships: str = Argument(
+                help="json of relationships (e.g., `[{'from_node':'fname', 'from_key':'cust_id', 'to_node':'tname', 'to_key'}]`)"),
+            parent_node: str = Argument(
+                help="parent/root node to which to aggregate all of the data"
+                ),
+            cut_date: str = Argument(str(datetime.datetime.today())),
+            compute_layer: str = Argument("pandas"),
+            hops_front: int = Option(1, '-hf', '--hops-front', help='number of front hops to peform'),
+            hops_back: int = Option(3, '-hb', '--hops-back', help='number of back hops to perform'),
+            output_path: str = Option('-op', '--output-path', help='output path for the data')
+            ):
+    """
+Main automated feature engineering function.
+    """
+    prefixes = json.loads(prefixes)
+    date_keys = json.loads(date_keys)
+    relationships = json.loads(relationships)
+
+    if isinstance(cut_date, str):
+        cut_date = datetime.datetime.strptime(cut_date, '%Y-%m-%d')
+
+    nodes = {}
+    if fmt in ['csv', 'parquet', 'delta', 'iceberg']:
+        for f in os.listdir(data_path):
+            print(f"adding file {f}")
+            nodes[f] = DynamicNode(
+                    fpath=f"{data_path}/{f}",
+                    fmt=f.split('.')[1],
+                    prefix=prefixes.get(f),
+                    compute_layer=getattr(ComputeLayerEnum, compute_layer),
+                    date_key=date_keys.get(f, None)
+                    )
+    gr = GraphReduce(
+            name='autofe',
+            parent_node=nodes[parent_node],
+            fmt=fmt,
+            cut_date=cut_date,
+            compute_layer=getattr(ComputeLayerEnum, compute_layer),
+            auto_features=True,
+            auto_feature_hops_front=hops_front,
+            auto_feature_hops_back=hops_back
+            )
+    for rel in relationships:
+        gr.add_entity_edge(
+                parent_node=nodes[rel['to_node']],
+                parent_key=rel['to_key'],
+                relation_node=nodes[rel['from_node']],
+                relation_key=rel['from_key'],
+                reduce=rel.get('reduce', True)
+                )
+    gr.do_transformations()
+    if not output_path:
+        output_path = os.path.join(
+                os.path.expanduser("~"),
+                "graphreduce_outputs/test.csv"
+                )
+    getattr(gr.parent_node.df, f"to_{fmt}")(output_path)

graphreduce/cli/entry_point.py

@@ -0,0 +1,26 @@
+#!/usr/bin/env python
+
+import typing
+
+import typer
+
+from .auto_fe import auto_fe_cli
+
+
+entrypoint_cli_typer = typer.Typer(
+        no_args_is_help=True,
+        add_completion=False,
+        rich_markup_mode="markdown",
+        help="""
+        See examples at https://github.com/wesmadrigal/graphreduce
+        """
+)
+
+# Automated feature engineering
+entrypoint_cli_typer.add_typer(auto_fe_cli, rich_help_panel="autofe")
+entrypoint_cli = typer.main.get_command(entrypoint_cli_typer)
+entrypoint_cli.list_commands(None)
+
+
+if __name__ == '__main__':
+    entrypoint_cli()

graphreduce/common.py

@@ -0,0 +1,100 @@
+#!/usr/bin/env python
+
+import pytz
+from datetime import datetime
+import pandas as pd
+import dask.dataframe as dd
+from pyspark.sql import functions as F
+import pyspark
+from torch_frame import stype
+
+
+stype_map = {
+        'numerical': [
+            'min',
+            'max',
+            'median',
+            'mean',
+            'sum',
+            ],
+        'categorical': [
+            'nunique',
+            'count',
+            'mode',
+            ],
+        'text_embedded': [
+            'length'
+            ],
+        'text_tokenized': [
+            'length'
+            ],
+        'multicategorical': [
+            'length'
+            ],
+        'sequence_numerical': [
+            'sum',
+            'min',
+            'max',
+            'median',
+            ],
+        'timestamp': [
+            'min',
+            'max',
+            'delta'
+            ],
+        'image_embedded': [],
+        'embedding': []
+}
+
+
+def clean_datetime_pandas(df: pd.DataFrame, col: str) -> pd.DataFrame:
+    df[col] = pd.to_datetime(df[col], errors="coerce", utc=True)
+
+    # Count the number of rows before removing invalid dates
+    total_before = len(df)
+
+    # Remove rows where timestamp is NaT (indicating parsing failure)
+    df = df.dropna(subset=[col])
+
+    # Count the number of rows after removing invalid dates
+    total_after = len(df)
+
+    # Calculate the percentage of rows removed
+    percentage_removed = ((total_before - total_after) / total_before) * 100
+
+    # Print the percentage of comments removed
+    print(
+        f"Percentage of rows removed due to invalid dates: "
+        f"{percentage_removed:.2f}%"
+    )
+    return df
+
+
+def clean_datetime_dask(df: dd.DataFrame, col: str) -> dd.DataFrame:
+    df[col] = dd.to_datetime(df[col])
+    total_before = len(df)
+    df = df.dropna(subset=[col])
+    total_after = len(df)
+    percentage_removed = ((total_before - total_after) / total_before) * 100
+    return df
+
+
+def clean_datetime_spark(df, col: str) -> pyspark.sql.DataFrame:
+    pass
+
+
+
+def convert_to_utc(dt):
+  """Converts a datetime object to UTC.
+
+  Args:
+    dt: The datetime object to convert.
+
+  Returns:
+    The datetime object converted to UTC.
+  """
+  if dt.tzinfo is None:  # Naive datetime
+    # Assuming the original timezone is the local system time
+    local_tz = pytz.timezone('US/Pacific')  # Replace with the actual timezone if known
+    dt = local_tz.localize(dt)
+  return dt.astimezone(pytz.UTC)

graphreduce/context.py

@@ -0,0 +1,73 @@
+#!/usr/bin/env python
+"""A module for helping with context
+in GraphReduce compute graphs.
+"""
+
+# standard library
+import typing
+
+# third party
+import pandas as pd
+import dask.dataframe as dd
+import pyspark
+from structlog import get_logger
+
+# internal
+from graphreduce.node import GraphReduceNode
+
+
+logger = get_logger('graphreduce.context')
+
+
+def method_requires (
+        nodes: typing.List[GraphReduceNode] = [],
+        checkpoint: bool = False,
+        ) -> callable:
+    """
+A decorator for ensuring the function
+only runs when the calling Node has
+a merged edge to the required `nodes` list.
+
+Arguments
+    nodes: list of GraphReduceNode classes to require
+           for function execution
+
+    checkpoint: boolean of whether or not to checkpoint
+
+Usage:
+    @requires(nodes=[CustomerNode, OrderNode])
+    def do_post_join_annotate(self):
+        self.df = self.df.withColumn(...)
+    """
+    def wrapit(func, nodes=nodes):
+        def newfunc(inst, *args, **kwargs):
+            for x in nodes:
+                if x not in inst._merged:
+                    return None
+            res = func(inst, *args, **kwargs)
+            if hasattr(inst, '_storage_client') and checkpoint:
+                if not isinstance(res, None.__class__):
+                    if res.__class__ in [pd.DataFrame, dd.DataFrame, pyspark.sql.dataframe.DataFrame]:
+                        df = res
+                else:
+                    df = inst.df
+
+                fname = func.__name__
+                if hasattr(inst, '_checkpoints') and fname in inst._checkpoints:
+                    return res
+                else:
+                    name = f"{inst.__class__.__name__}_{fname}.{inst.fmt}"
+                    # checkpoint 
+                    inst._storage_client.offload(
+                            df,
+                            name
+                            )
+                    path = inst._storage_client.get_path(name)
+                    # reload 
+                    inst.df = inst._storage_client.load(path)
+
+                    # add function to list of checkpoints.
+                    inst._checkpoints.append(fname)
+            return res
+        return newfunc
+    return wrapit

graphreduce/enum.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env
+
+import enum
+
+class PeriodUnit(enum.Enum):
+    second = 'second'
+    minute = 'minute'
+    hour = 'hour'
+    day = 'day'
+    week = 'week'
+    month = 'month'
+    year = 'year'
+
+class ComputeLayerEnum(enum.Enum):
+    # File-based compute layers.
+    pandas = 'pandas'
+    polars = 'polars'
+    dask = 'dask'
+    spark = 'spark'
+    ray = 'ray'
+    # SQL dialects.
+    athena = 'athena'
+    snowflake = 'snowflake'
+    redshift = 'redshift'
+    postgres = 'postgres'
+    mysql = 'mysql'
+    sqlite = 'sqlite'
+    databricks = 'databricks'
+    daft = 'daft'
+
+
+class StorageFormatEnum(enum.Enum):
+    csv = 'csv'
+    parquet = 'parquet'
+    tsv = 'tsv'
+    delta = 'delta'
+
+class ProviderEnum(enum.Enum):
+    local = 'local'
+    s3 = 's3'
+    blog = 'blob'
+    gcs = 'gcs'
+    hdfs = 'hdfs'
+
+
+class SQLOpType(enum.Enum):
+    # All things related to selecting
+    # data get housed under this op type.
+    # Even case statements, if in the select
+    # part of the query, should go here.
+    select = 'select'
+    from_ = 'from'
+    # All aspects of where clauses use this.
+    where = 'where'
+    # The anatomy of a given aggregation
+    # typically consists of a select
+    # AND a group by statement.
+    # This op should be used only for the
+    # actual grouping portion and a separate
+    # `select` op type should be used
+    # for the columns and aggregation functions.
+    agg = 'group by'
+    aggfunc = 'aggfunc'
+    order = 'order by'
+    having = 'having'
+    window = 'window'
+    custom = 'custom'