rand-engine 0.2.0__tar.gz → 0.3.0__tar.gz

{rand_engine-0.2.0 → rand_engine-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: rand-engine
-Version: 0.2.0
+Version: 0.3.0
 Summary: Rand Engine v2. Package with some methods to generate random data in different formats. Great to mock data while testing or developing.
 Home-page: https://github.com/marcoaureliomenezes/rand_engine
 Author: marcoaureliomenezes

{rand_engine-0.2.0 → rand_engine-0.3.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "rand-engine"
-version = "0.2.0"
+version = "0.3.0"
 description = "Rand Engine v2. Package with some methods to generate random data in different formats. Great to mock data while testing or developing."
 authors = ["marcoaureliomenezes <marcoaurelioreislima@gmail.com>"]
 repository = "https://github.com/marcoaureliomenezes/rand_engine"
@@ -12,7 +12,6 @@ numpy = "^2.1.1"
 pandas = "^2.2.2"
 
 
-
 [tool.poetry.group.test.dependencies]
 pytest = "^8.3.3"
 faker = "^28.4.1"

rand_engine-0.3.0/rand_engine/core/distinct_utils.py

@@ -0,0 +1,54 @@
+
+import itertools
+
+
+class DistinctUtils:
+
+  @classmethod
+  def handle_distincts_lvl_1(self, distinct_prop, precision):
+    return [ key for key, value in distinct_prop.items() for i in range(value * precision )]
+
+  @classmethod
+  def handle_distincts_lvl_2(self, distincts, sep=";"):
+    data_flatted = [f"{j}{sep}{i}" for j in distincts for i in distincts[j]]
+    return data_flatted
+
+
+  @classmethod
+  def handle_distincts_lvl_3(self, distincts, sep=";"):
+    parm_paired_distincts = {k: list(map(lambda x: f"{x[0]}@!{x[1]}", v)) for k, v in distincts.items()}
+    data_flatted = self.handle_distincts_lvl_2(parm_paired_distincts, sep)
+    result = []
+    for i in data_flatted:
+      value, size = i.split("@!")
+      result.extend([value for _ in range(int(size))])
+    return result
+  
+  @classmethod
+  def handle_distincts_lvl_4(self, distincts, sep=";"):
+    combinations = [list(itertools.product([k], *v)) for k, v in distincts.items()]
+    result = [sep.join(i) for i in list(itertools.chain(*combinations))]
+    return result
+  
+
+
+if __name__ == '__main__':
+  distincts = {"OPC": ["C_OPC","V_OPC"], "SWP": ["C_SWP", "V_SWP"]}
+
+  distinct_1 = {"OPC": [["C_OPC","V_OPC"], ["PF", "PJ"], ["IN"]], "SWP": [["C_SWP", "V_SWP"], ["AF", "ME"], ["NULL"]]}
+  distinct_2 = {"OPC": [{"C_OPC": ["PF", "PJ"]}, {"V_OPC": ["NA"]}], "SWP": [{"C_SWP": ["AP"]}, {"V_SWP": ["MA", "ME"]}]}
+  #print(DistinctUtils.handle_distincts_lvl_5(distinct_2)[]
+
+
+  def rec(structure):
+    if isinstance(structure, list):
+      return [rec(i) for i in structure]
+    if isinstance(structure, dict):
+      return [[[k], rec(v)] for k, v in structure.items()]
+    return structure
+
+  import numpy as np
+
+  result = rec(distinct_2)
+
+  combinations = np.array(list(itertools.product(*result)))

rand_engine-0.3.0/rand_engine/main/data_generator.py

@@ -0,0 +1,74 @@
+import pandas as pd
+from typing import List, Dict, Optional, Generator, Callable
+from pyspark.sql import SparkSession, DataFrame
+import json
+import os
+import time
+import random
+
+
+class DataGenerator:
+      
+
+  def handle_splitable(self, metadata, df):
+    for key, value in metadata.items():
+      if value.get("splitable"):
+        sep = value.get("sep", ";")
+        cols = value.get("cols")
+        df[cols] = df[key].str.split(sep, expand=True)
+        df.drop(columns=[key], inplace=True)
+    return df
+
+
+  def __convert_dt_to_str(self, dataframe: pd.DataFrame) -> pd.DataFrame:
+      df_result = dataframe.copy()
+      for column in df_result.columns:
+          if 'datetime64' in str(df_result[column].dtype):
+              df_result[column] = df_result[column].astype(str)
+      return df_result
+  
+      
+  def __sleep_to_contro_throughput(self, min_throughput: int, max_throughput: int):
+    sleep_time = 1 / random.uniform(min_throughput, max_throughput)
+    time.sleep(sleep_time)
+
+
+  def create_pandas_df(self, size: int, metadata: dict, transformer: Optional[Callable]=None) -> pd.DataFrame:
+    dict_data = {key: value["method"](size, **value["parms"]) for key, value in metadata.items()}
+    df_data = pd.DataFrame(dict_data)
+    df_data_final = self.handle_splitable(metadata, df_data)
+    if transformer: df_data_final = transformer(df_data_final)
+    return df_data_final
+  
+
+  def create_spark_df(self, spark: SparkSession, size: int, metadata: dict, transformer: Optional[Callable]=None) -> DataFrame:
+    df_data = self.create_pandas_df(size=size, metadata=metadata)
+    df_final = spark.createDataFrame(df_data)
+    return df_final
+
+
+  def create_streaming_records(self, microbatch_size:int, metadata: dict, transformer: Optional[Callable]=None, min_throughput: int=1, max_throughput: int = 10) -> Generator:
+    while True:
+      df_data_microbatch = self.create_pandas_df(size=microbatch_size, metadata=metadata, transformer=transformer)
+      df_data_parsed = self.__convert_dt_to_str(df_data_microbatch)
+      list_of_records = df_data_parsed.to_dict('records')
+      for record in list_of_records:
+          record["timestamp_created"] = round(time.time(), 3)
+          yield record
+          self.__sleep_to_contro_throughput(min_throughput, max_throughput)
+  
+
+  def create_csv_file(self, microbatch_size: int, size_in_mb: int, metadata: dict, path: str, transformer: Optional[Callable]=None) -> None:
+    os.makedirs(os.path.dirname(path), exist_ok=True)
+    while True:
+      df = self.create_pandas_df(size=microbatch_size, metadata=metadata, transformer=transformer)
+      df.to_csv(path, mode='a', header=False, index=False)
+      if os.path.getsize(path) > size_in_mb * 1024 * 1024: break
+
+
+
+
+   
+
+if __name__ == '__main__':
+  pass

rand_engine-0.2.0/rand_engine/core/distinct_utils.py

@@ -1,23 +0,0 @@
-
-
-class DistinctUtils:
-
-  @classmethod
-  def handle_distincts_lvl_1(self, distinct_prop, precision):
-    return [ key for key, value in distinct_prop.items() for i in range(value * precision )]
-
-  @classmethod
-  def handle_distincts_lvl_2(self, distincts, sep=";"):
-    data_flatted = [f"{j}{sep}{i}" for j in distincts for i in distincts[j]]
-    return data_flatted
-
-  @classmethod
-  def handle_distincts_lvl_3(self, distincts, sep=";"):
-    parm_paired_distincts = {k: list(map(lambda x: f"{x[0]}@!{x[1]}", v)) for k, v in distincts.items()}
-    data_flatted = self.handle_distincts_lvl_2(parm_paired_distincts, sep)
-    result = []
-    for i in data_flatted:
-      value, size = i.split("@!")
-      result.extend([value for _ in range(int(size))])
-    return result
-  

rand_engine-0.2.0/rand_engine/main/dataframe_builder.py

@@ -1,64 +0,0 @@
-import pandas as pd
-import json
-import os
-
-
-class BulkRandEngine:
-      
-
-  def handle_splitable(self, metadata, df):
-    for key, value in metadata.items():
-      if value.get("splitable"):
-        sep = value.get("sep", ";")
-        cols = value.get("cols")
-        df[cols] = df[key].str.split(sep, expand=True)
-        df.drop(columns=[key], inplace=True)
-    return df
-
-      
-
-  def create_pandas_df(self, size, metadata):
-    df_pandas = pd.DataFrame({key: value["method"](size, **value["parms"]) for key, value in metadata.items()})
-    df_pandas = self.handle_splitable(metadata, df_pandas)
-    return df_pandas
-  
-  @classmethod
-  def convert_datetimes_to_string(self, pandas_df):
-    for column in pandas_df.columns:
-      if pandas_df[column].dtype == 'datetime64[ns]':
-        pandas_df[column] = pandas_df[column].astype(str)
-    return pandas_df
-  
-
-  @classmethod
-  def create_streaming_df(self, pandas_df):
-    pandas_df = self.convert_datetimes_to_string(pandas_df)
-    list_of_dicts = pandas_df.to_dict('records')
-    for record in list_of_dicts:
-      yield json.dumps(record)
-
-  @classmethod
-  def create_streaming_series(self, pandas_series):
-    for record in pandas_series:
-      yield record
-
-  @classmethod
-  def create_file(path, word, limit_size):
-    os.makedirs(os.path.dirname(path), exist_ok=True)
-    with open(path, 'w') as file:
-      while file.tell() < limit_size:
-        file.write(word + '\n')
-    return True
-  
-
-  def microbatch_file_with_streaming(self, path, metadata, df_transformer, microbatch_size, total_size):
-    os.makedirs(os.path.dirname(path), exist_ok=True)
-    while True:
-      df = self.create_pandas_df(size=microbatch_size, metadata=metadata)
-      df = df_transformer(df)
-      df.to_csv(path, mode='a', header=False, index=False)
-      if os.path.getsize(path) > total_size:
-        break
-      
-if __name__ == '__main__':
-  pass