rand-engine 0.4.4__tar.gz → 0.4.7__tar.gz

{rand_engine-0.4.4 → rand_engine-0.4.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rand-engine
-Version: 0.4.4
+Version: 0.4.7
 Summary: Rand Engine v2. Package with some methods to generate random data in different formats. Great to mock data while testing or developing.
 Author: marcoaureliomenezes
 Author-email: marcoaurelioreislima@gmail.com

{rand_engine-0.4.4 → rand_engine-0.4.7}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "rand-engine"
-version = "0.4.4"
+version = "0.4.7"
 description = "Rand Engine v2. Package with some methods to generate random data in different formats. Great to mock data while testing or developing."
 authors = ["marcoaureliomenezes <marcoaurelioreislima@gmail.com>"]
 repository = "https://github.com/marcoaureliomenezes/rand_engine"

rand_engine-0.4.7/rand_engine/core.py

@@ -0,0 +1,95 @@
+
+from typing import List, Any
+import numpy as np
+from datetime import datetime as dt
+from functools import reduce
+
+
+
+class Core:
+    
+  @classmethod
+  def gen_distincts(self, size: int, distinct: List[Any]) -> np.ndarray:
+    assert len(list(set([type(x) for x in distinct]))) == 1
+    return np.random.choice(distinct, size)
+  
+
+  @classmethod
+  def gen_distincts_untyped(self, size: int, distinct: List[Any]) -> List[Any]:
+    return list(map(lambda x: distinct[x], np.random.randint(0, len(distinct), size)))
+  
+
+  @classmethod
+  def gen_complex_distincts(self, size: int, pattern="x.x.x-x", replacement="x", templates=[]):
+    assert pattern.count(replacement) == len(templates)
+    list_of_lists, counter = [], 0
+    for replacer_cursor in range(len(pattern)):
+      if pattern[replacer_cursor] == replacement:
+        list_of_lists.append(templates[counter]["method"](size, **templates[counter]["parms"]))
+        counter += 1
+      else:
+        list_of_lists.append(np.array([pattern[replacer_cursor] for i in range(size)]))
+    return reduce(lambda a, b: a.astype('str') + b.astype('str'), list_of_lists)
+  
+  
+  @classmethod
+  def gen_ints(self, size: int, min: int, max: int) -> np.ndarray:
+    return np.random.randint(min, max + 1, size)
+
+
+  @classmethod
+  def gen_ints_zfilled(self, size: int, length: int) -> np.ndarray:
+    str_arr = np.random.randint(0, 10**length, size).astype('str')
+    return np.char.zfill(str_arr, length)
+  
+  
+  @classmethod
+  def gen_floats(self, size: int, min: int, max: int, round: int = 2) -> np.ndarray:
+    sig_part = np.random.randint(min, max, size)
+    decimal = np.random.randint(0, 10 ** round, size)
+    return sig_part + (decimal / 10 ** round) if round > 0 else sig_part
+
+
+  @classmethod
+  def gen_floats_normal(self, size: int, mean: int, std: int, round: int = 2) -> np.ndarray:
+    return np.round(np.random.normal(mean, std, size), round)
+  
+
+  @classmethod
+  def gen_unix_timestamps(self, size: int, start: str, end: str, format: str) -> np.ndarray:
+    dt_start, dt_end = dt.strptime(start, format), dt.strptime(end, format)
+    if dt_start < dt(1970, 1, 1): dt_start = dt(1970, 1, 1)
+    timestamp_start, timestamp_end = dt_start.timestamp(), dt_end.timestamp()
+    int_array = np.random.randint(timestamp_start, timestamp_end, size)
+    return int_array
+  
+
+  @classmethod
+  def gen_unique_identifiers(self, size: int, strategy="zint", length=12) -> np.ndarray:
+    import uuid
+    if strategy == "uuid4":
+      return np.array([str(uuid.uuid4()) for _ in range(size)])
+    elif strategy == "uuid1":
+      return np.array([str(uuid.uuid1()) for _ in range(size)])
+    elif strategy == "zint":
+      return self.gen_ints_zfilled(size, length)
+    else:
+      raise ValueError("Method not recognized. Use 'uuid4', 'uuid1', 'shortuuid' or 'random'.")
+  # @classmethod
+  # def gen_timestamps(self, size: int, start: str, end: str, format: str) -> np.ndarray:
+  #   """
+  #   This method generates an array of random timestamps.
+  #   :param size: int: Number of elements to be generated.
+  #   :param start: str: Start date of the generated timestamps.
+  #   :param end: str: End date of the generated timestamps.
+  #   :param format: str: Format of the input dates.
+  #   :return: np.ndarray: Array of random timestamps."""
+  #   date_array = self.gen_unix_timestamps(size, start, end, format).astype('datetime64[s]')
+  #   return date_array
+  
+  
+  # @classmethod
+  # def gen_datetimes(self, size: int, start: str, end: str, format_in: str, format_out: str):
+  #   timestamp_array = self.gen_unix_timestamps(size, start, end, format_in)
+  #   vectorized_func = np.vectorize(lambda x: dt.fromtimestamp(x).strftime(format_out))
+  #   return vectorized_func(timestamp_array)

{rand_engine-0.4.4/rand_engine/main → rand_engine-0.4.7/rand_engine}/file_writer.py

@@ -86,7 +86,8 @@ class FileWriter:
     :return: Callable: Function to write the Pandas DataFrame to a csv file.
     """
     if self.write_options.get("compression"):
-      full_path= full_path.replace("csv", f"csv.{self.write_options['compression']}")
+      # Add compression extension to the end of the filename
+      full_path = f"{full_path}.{self.write_options['compression']}"
     writer = lambda: dataframe().to_csv(full_path, index=False, **self.write_options)
     return writer
   
@@ -98,7 +99,8 @@ class FileWriter:
     :return: Callable: Function to write the Pandas DataFrame to a json file.
     """
     if self.write_options.get("compression"):
-      full_path= full_path.replace("json", f"json.{self.write_options['compression']}")
+      # Add compression extension to the end of the filename
+      full_path = f"{full_path}.{self.write_options['compression']}"
     def writer():
       dataframe().to_json(full_path, orient='records', lines=True)
     return writer
@@ -132,13 +134,13 @@ class FileWriter:
     :param path: str: Path of the file to be written.
     :param size_in_mb: int: Size in MB of the file to be written.
     """
-    self.__handle_fs(path, flag=False)
+    self.__handle_fs(path, flag=True)
     counter = 0
     while True:
       full_path = f"{path}/part-{str(counter).zfill(6)}.{self.write_format}"
       dataframe = self.microbatch_def()
       self.dict_format[self.write_format](dataframe, full_path)()
       size_bytes = self.__get_dir_size(path)
-      if counter % 100 == 0: print(f"Size: {size_bytes/2**20:.2f} MB")
+      #if counter % 100 == 0: print(f"Size: {size_bytes/2**20:.2f} MB")
       if self.__get_dir_size(path) >= size_in_mb*2**20: break
       counter += 1

{rand_engine-0.4.4 → rand_engine-0.4.7}/rand_engine/main/cdc_generator.py

@@ -8,7 +8,7 @@ import pandas as pd
 from pandas import DataFrame as PandasDF
 
 from rand_engine.main.i_random_spec import IRandomSpec
-from rand_engine.main.data_generator import DataGenerator
+from rand_engine.main import RandGenerator
 from rand_engine.main.fs_utils import FSUtils, DBFSUtils
 
 from pyspark.sql.functions import coalesce
@@ -34,10 +34,10 @@ class FilesGenerator:
   def _get_file_path(self) -> str:
     return f"{self.base_path}/{self.file_name}_{str(uuid4())[:8]}.{self.ext}"
   
-
+  
   def generate_sample(self, size: int=100) -> PandasDF:
     return (
-      DataGenerator(self.footprint.metadata())
+      RandGenerator(self.footprint.metadata())
         .generate_pandas_df(size, transformer=self.footprint.transformer())
         .get_df()
     )
@@ -55,7 +55,7 @@ class FilesGenerator:
   def write_file(self, size: int=100, const_cols={}):
     file_path = self._get_file_path()
     _ = (
-      DataGenerator(self.footprint.metadata()) \
+      RandGenerator(self.footprint.metadata()) \
         .generate_pandas_df(size, transformer=self.footprint.transformer(**const_cols))
         .write() \
         .mode("overwrite") \
@@ -108,7 +108,7 @@ class CDCGenerator(FilesGenerator):
     metadata = self.footprint.metadata()
     size = df_pks_to_change.shape[0]
     transformer = self.footprint.transformer_cdc_update(null_rate=null_rate, **const_cols)
-    df_data = DataGenerator(metadata).generate_pandas_df(size, transformer).get_df()
+    df_data = RandGenerator(metadata).generate_pandas_df(size, transformer).get_df()
     for coluna in self.pk_cols: df_data[coluna] = df_pks_to_change[coluna]
     if null_rate != 1: 
       cols_to_check = [col for col in df_data.columns if col not in self.pk_cols + list(const_cols.keys())]

{rand_engine-0.4.4 → rand_engine-0.4.7}/rand_engine/main/fs_utils.py

@@ -131,7 +131,7 @@ class DBFSUtils(FSUtils):
 
     def rm(self, path: str, recursive: bool = False) -> None:
         try:
-            result = self.dbutils.fs.rm(path, recurse=recursive)
+            result = self.dbutils.fs.rm(path, recursive)
             if not result:
                 raise Exception(f"Failed to delete file {path}")
         except Exception as e:

rand_engine-0.4.7/rand_engine/main.py

@@ -0,0 +1,102 @@
+import os
+import time
+import pandas as pd
+import numpy as np
+from typing import List, Dict, Optional, Generator, Callable, Any
+from rand_engine.rand_generator import RandGenerator
+from rand_engine.file_writer import FileWriter
+from rand_engine.utils.stream_handler import StreamHandler
+
+  
+
+
+class RandEngine:
+      
+  def __init__(self, random_spec, seed: bool = False):
+    np.random.seed(42) if seed else np.random.seed(None)
+    self.actual_dataframe: Optional[Callable[[], pd.DataFrame]] = None
+    self.data_generator = RandGenerator(random_spec)
+    self._mode = "pandas"
+    self._size = 1000
+    self._transformers: List[Optional[Callable]] = []
+
+ 
+  def generate_pandas_df(self, size: int) -> pd.DataFrame:
+    """
+    This method generates a pandas DataFrame based on random data specified in the metadata parameter.
+    :param size: int: Number of rows to be generated.
+    :param transformer: Optional[Callable]: Function to transform the generated data.
+    :return: pd.DataFrame: DataFrame with the generated data.
+    """
+    def wrapped_lazy_dataframe():
+      df_pandas = self.data_generator.generate_first_level(size=size)
+      df_pandas = self.data_generator.handle_splitable(df_pandas)
+      df_pandas = self.data_generator.apply_embedded_transformers(df_pandas)
+      df_pandas = self.data_generator.apply_global_transformers(df_pandas, self._transformers)
+      return df_pandas
+    self.actual_dataframe = wrapped_lazy_dataframe
+  
+
+  def transformers(self, transformers: List[Optional[Callable]]):
+    self._transformers = transformers
+    return self
+
+  def generate_spark_df(self, spark, size: int) -> Any:
+    """
+    This method generates a Spark DataFrame based on random data specified in the random_spec parameter.
+    :param spark: SparkSession: SparkSession object.
+    :param size: int: Number of rows to be generated.
+    :param transformer: Optional[Callable]: Function to transform the generated data."""
+    def wrapped_lazy_dataframe():
+      self.generate_pandas_df(size=size)
+      df_spark = spark.createDataFrame(self.actual_dataframe())
+      return df_spark
+    self.actual_dataframe = wrapped_lazy_dataframe
+
+  def mode(self, mode: str):
+    assert mode in ["pandas", "spark"], "Mode not recognized. Use 'pandas' or 'spark'."
+    self._mode = mode
+    return self
+
+  def size(self, size: int):
+    self._size = size
+    return self
+
+  def get_df(self, spark=None):
+    if self._mode == "pandas":
+      self.generate_pandas_df(size=self._size)
+    elif self._mode == "spark":
+      self.generate_spark_df(spark=spark, size=self._size)
+    assert self.actual_dataframe is not None, "You need to generate a DataFrame first."
+    return self.actual_dataframe()
+
+
+  def stream_dict(self, min_throughput: int=1, max_throughput: int = 10) -> Generator:
+    """
+    This method creates a generator of records to be used in a streaming context.
+    :param min_throughput: int: Minimum throughput to be generated.
+    :param max_throughput: int: Maximum throughput to be generated.
+    :return: Generator: Generator of records.
+    """
+    self.generate_pandas_df(size=self._size)
+    assert self.actual_dataframe is not None, "You need to generate a DataFrame first."
+    while True:
+      df_data_microbatch = self.actual_dataframe()
+      df_data_parsed = StreamHandler.convert_dt_to_str(df_data_microbatch)
+      list_of_records = df_data_parsed.to_dict('records')
+      for record in list_of_records:
+        record["timestamp_created"] = round(time.time(), 3)
+        yield record
+        StreamHandler.sleep_to_contro_throughput(min_throughput, max_throughput)
+  
+
+  def write(self, size):
+    self.generate_pandas_df(size=size)
+    microbatch_def = lambda: self.actual_dataframe
+    return FileWriter(microbatch_def)
+   
+
+
+if __name__ == '__main__':
+
+  pass

rand_engine-0.4.7/rand_engine/rand_generator.py

@@ -0,0 +1,46 @@
+from typing import List, Optional, Callable
+import pandas as pd
+
+
+class RandGenerator:
+
+
+  def __init__(self, random_spec):
+    self.random_spec = random_spec
+
+
+  def generate_first_level(self, size: int):
+    dict_data = {}
+    for k, v in self.random_spec.items():
+      try:
+        if "args" in v: dict_data[k] = v["method"](size , *v["args"]) 
+        else: dict_data[k] = v["method"](size , **v.get("kwargs", {}))
+      except Exception as e:
+        raise Exception(f"Error generating data for column '{k}': {e}")
+    df_pandas = pd.DataFrame(dict_data)
+    return df_pandas
+
+
+  def apply_embedded_transformers(self, df):
+
+    cols_with_transformers = {key: value["transformers"] for key, value in self.random_spec.items() if value.get("transformers")}
+    for col, transformers in cols_with_transformers.items():
+      for transformer in transformers:
+        df[col] = df[col].apply(transformer)
+    return df
+  
+  def apply_global_transformers(self, df, transformers: List[Optional[Callable]]):
+    if transformers:
+      if len(transformers) > 0: 
+        for transformer in transformers:
+          df = transformer(df)
+    return df
+ 
+  def handle_splitable(self, df):
+    for key, value in self.random_spec.items():
+      if value.get("splitable"):
+        sep = value.get("sep", ";")   
+        cols = value.get("cols")
+        df[cols] = df[key].str.split(sep, expand=True)
+        df.drop(columns=[key], inplace=True)
+    return df

rand_engine-0.4.7/rand_engine/spark/spark_core.py

@@ -0,0 +1,40 @@
+
+import pyspark.sql.functions as F
+from pyspark.sql.functions import randn, rand, randstr
+from pyspark.sql import DataFrame, SparkSession
+import pandas as pd
+
+class RandSpark:
+
+    def __init__(self, spark, df: DataFrame):
+        self.spark = spark
+        self._df = df
+
+
+    def withColumnRandInt(self, col_name="rand_int", min_size=0, max_size=10):
+        return RandSpark(
+            self._df.withColumn(col_name, (F.rand() * (max_size - min_size) + min_size).cast("int"))
+        )
+    
+    def withColumnRandFloat(self, col_name="rand_float", min_size=0.0, max_size=10.0, decimals=2):
+      
+        return RandSpark(
+            self._df.withColumn(col_name, F.round(F.rand() * (max_size - min_size) + min_size, decimals))
+        )
+    
+    def withColumnRandChoice(self, col_name="rand_choice", distincts=[]):
+        df_columns = self._df.columns
+        aux_col = f"{col_name}_aux"
+        
+        df_pd = pd.DataFrame(distincts, columns=[col_name])
+        df_pd[aux_col] = range(len(distincts))
+        df_spark = self.spark.createDataFrame(df_pd)
+        df = RandSpark(self._df.withColumn(aux_col, (F.rand() * (len(distincts) - 0) + 0).cast("int")))
+        return (
+            df.alias("a").join(F.broadcast(df_spark).alias("b"), on=aux_col, how="left") \
+            .select(*df_columns, f"b.{col_name}"))
+
+
+    def __getattr__(self, name):
+        """Delegate unknown methods to the original DataFrame"""
+        return getattr(self._df, name)

rand_engine-0.4.4/rand_engine/core/distinct_utils.py → rand_engine-0.4.7/rand_engine/utils/distincts.py

@@ -5,7 +5,7 @@ import itertools
 class DistinctUtils:
 
   @classmethod
-  def handle_distincts_lvl_1(self, distinct_prop, precision):
+  def handle_distincts_lvl_1(self, distinct_prop, precision=1):
     """
     This method generates a list of distinct values based on a dictionary of distinct values and their respective frequencies.
     :param distinct_prop: dict: Dictionary containing the distinct values and their respective frequencies.
@@ -22,6 +22,13 @@ class DistinctUtils:
     data_flatted = [f"{j}{sep}{i}" for j in distincts for i in distincts[j]]
     return data_flatted
 
+  @classmethod
+  def handle_distincts_lvl_22(self, distincts):
+    """
+    This method generates a list of distinct values based on a dictionary of distinct values and their respective frequencies.
+    :param distincts: dict: Dictionary containing the distinct values and their respective frequencies."""
+    data_flatted = [(j, i) for j in distincts for i in distincts[j]]
+    return data_flatted
 
   @classmethod
   def handle_distincts_lvl_3(self, distincts, sep=";"):

rand_engine-0.4.4/rand_engine/main/stream_handle.py → rand_engine-0.4.7/rand_engine/utils/stream_handler.py

@@ -2,7 +2,7 @@ import random
 import time
 import pandas as pd
 
-class StreamHandle:
+class StreamHandler:
   
   @staticmethod
   def convert_dt_to_str(dataframe: pd.DataFrame) -> pd.DataFrame:

rand_engine-0.4.7/rand_engine/utils/update.py

@@ -0,0 +1,33 @@
+import numpy as np
+from random import randint
+
+
+class Changer:
+
+    def __init__(self, cols_to_change):
+        self.cols_to_change = cols_to_change
+
+    def __transformer_numeric(self, df):
+        numeric_types = ['int64', 'int32', 'float32', 'float64']
+        for col in df.columns:
+            if col in self.cols_to_change and df[col].dtype in numeric_types:
+                np.random.seed(None)
+                df[col + '_random'] = np.random.randint(df[col].min(), df[col].max(), size=len(df))
+                df[col + '_random_signal'] = np.random.choice([-1, 1], size=len(df))
+                df[col + '_random'] = df[col + '_random'] * df[col + '_random_signal']
+                df[col] = df[col] + df[col + '_random']
+                df.drop(columns=[col + '_random', col + '_random_signal'], inplace=True)
+        return df
+
+
+    def __transformer_object(self, df):
+        for col in df.columns:
+            if col in self.cols_to_change and df[col].dtype == 'object':
+                df[col] = np.roll(df[col], randint(1, len(df)))
+        return df
+
+    def updater(self, df):
+        transformers = [self.__transformer_numeric, self.__transformer_object]
+        for transform in transformers:
+            df = transform(df)
+        return df

rand_engine-0.4.4/rand_engine/core/datetime_core.py

@@ -1,45 +0,0 @@
-import numpy as np
-import random
-from datetime import datetime as dt, timedelta
-
-
-class DatetimeCore:
-  
-  @classmethod
-  def gen_unix_timestamps(self, size: int, start: str, end: str, format: str) -> np.ndarray:
-    """
-    This method generates an array of random unix timestamps.
-    :param size: int: Number of elements to be generated.
-    :param start: str: Start date of the generated timestamps.
-    :param end: str: End date of the generated timestamps.
-    :param format: str: Format of the input dates."""
-    dt_start, dt_end = dt.strptime(start, format), dt.strptime(end, format)
-    if dt_start < dt(1970, 1, 1): dt_start = dt(1970, 1, 1)
-    timestamp_start, timestamp_end = dt_start.timestamp(), dt_end.timestamp()
-    int_array = np.random.randint(timestamp_start, timestamp_end, size)
-    return int_array
-
-
-  @classmethod
-  def gen_timestamps(self, size: int, start: str, end: str, format: str) -> np.ndarray:
-    """
-    This method generates an array of random timestamps.
-    :param size: int: Number of elements to be generated.
-    :param start: str: Start date of the generated timestamps.
-    :param end: str: End date of the generated timestamps.
-    :param format: str: Format of the input dates.
-    :return: np.ndarray: Array of random timestamps."""
-
-    date_array = self.gen_unix_timestamps(size, start, end, format).astype('datetime64[s]')
-    return date_array
-  
-  
-  @classmethod
-  def gen_datetimes(self, size: int, start: str, end: str, format_in: str, format_out: str):
-    timestamp_array = self.gen_unix_timestamps(size, start, end, format_in)
-    return [dt.fromtimestamp(i).strftime(format_out) for i in timestamp_array]
-
-
-
-if __name__ == '__main__':
-  pass

rand_engine-0.4.4/rand_engine/core/distinct_core.py

@@ -1,65 +0,0 @@
-from functools import reduce
-from typing import List, Any, Iterator
-import numpy as np
-
-
-class DistinctCore:
-    
-  @classmethod
-  def gen_distincts_typed(self, size: int, distinct: List[Any]) -> np.ndarray:
-    """
-    This method generates an array of random distinct values.
-    :param size: int: Number of elements to be generated.
-    :param distinct: List[Any]: List of distinct values to be generated.
-    :return: np.ndarray: Array of random distinct values.
-    """
-    assert len(list(set([type(x) for x in distinct]))) == 1
-    return np.random.choice(distinct, size)
-    
-
-  @classmethod
-  def gen_distincts_untyped(self, size: int, distinct: List[Any]) -> List[Any]:
-    """
-    This method generates an array of random distinct values.
-    :param size: int: Number of elements to be generated.
-    :param distinct: List[Any]: List of distinct values to be generated.
-    :return: Iterator: Iterator of random distinct values.
-    """
-    return list(map(lambda x: distinct[x], np.random.randint(0, len(distinct), size)))
-  
-  @classmethod
-  def gen_complex_distincts(self, size: int, pattern="x.x.x-x", replacement="x", templates=[]):
-    """
-    This method generates an array of random distinct values.
-    :param size: int: Number of elements to be generated.
-    :param pattern: str: Pattern to be replaced.
-    :param replacement: str: Replacement of the pattern.
-    :param templates: List[Dict]: List of dictionaries containing the method and parameters to be used in the replacement.
-    :return: np.ndarray: Array of random distinct values.
-    """
-    assert pattern.count(replacement) == len(templates)
-    list_of_lists, counter = [], 0
-    for replacer_cursor in range(len(pattern)):
-      if pattern[replacer_cursor] == replacement:
-        list_of_lists.append(templates[counter]["method"](size, **templates[counter]["parms"]))
-        counter += 1
-      else:
-        list_of_lists.append(np.array([pattern[replacer_cursor] for i in range(size)]))
-    return reduce(lambda a, b: a.astype('str') + b.astype('str'), list_of_lists)
-  
-  
-  
-if __name__ == '__main__':
-  pass
-
-
-
-# def replace_duplicate(array_input, replace):
-#     result = list(set(array_input))
-#     result.extend([replace for i in range(len(array_input)-len(list(set(array_input))))])
-#     random.shuffle(result)
-#     return result
-
-# def handle_string_format(array_input, **kwargs): 
-#     return replace_duplicate(array_input, np.nan) \
-#                 if kwargs.get("rm_dupl") else array_input

rand_engine-0.4.4/rand_engine/core/general_utils.py

@@ -1,14 +0,0 @@
-import numpy as np
-
-def expand_array(size=10, base_array=[]):
-    return [base_array[int(i % len(base_array))] for i in range(size)]
-
-def reduce_array(size=10, base_array=[]):
-    int_array = [int(i) for i in np.linspace(0, size-1, len(base_array))]
-    reduced = [int_array.index(i) for i in range(size)]
-    result = [base_array[i] for i in reduced]
-    return result
-
-
-def spaced_array(interval, num_part=2):
-    return list(np.linspace(interval[0], interval[1], num_part))

rand_engine-0.4.4/rand_engine/core/numeric_core.py

@@ -1,62 +0,0 @@
-import numpy as np
-
-
-class NumericCore:
-
-
-  @classmethod
-  def gen_ints(self, size: int, min: int, max: int) -> np.ndarray:
-    """
-    This method generates an array of random integers.
-    :param size: int: Number of elements to be generated.
-    :param min: int: Minimum value of the generated numbers.
-    :param max: int: Maximum value of the generated numbers.
-    :return: np.ndarray: Array of random integers.
-    """
-    return np.random.randint(min, max + 1, size)
-
-
-  @classmethod
-  def gen_ints_zfilled(self, size: int, length: int) -> np.ndarray:
-    """
-    This method generates an array of random integers with a fixed length.
-    :param size: int: Number of elements to be generated.
-    :param length: int: Length of the generated numbers.
-    :return: np.ndarray: Array of random integers.
-    """
-    str_arr = np.random.randint(0, 10**length, size).astype('str')
-    return np.char.zfill(str_arr, length)
-  
-  
-  @classmethod
-  def gen_floats(self, size: int, min: int, max: int, round: int = 2):
-    """
-    This method generates an array of random floats.
-    :param size: int: Number of elements to be generated.
-    :param min: int: Minimum value of the generated numbers.
-    :param max: int: Maximum value of the generated numbers.
-    :param round: int: Number of decimal places to round the generated numbers. Default is 2.
-    :return: np.ndarray: Array of random floats.
-    """
-    sig_part = np.random.randint(min, max, size)
-    decimal = np.random.randint(0, 10 ** round, size)
-    return sig_part + (decimal / 10 ** round) if round > 0 else sig_part
-
-
-  @classmethod
-  def gen_floats_normal(self, size: int, mean: int, std: int, round: int = 2):
-    """
-    This method generates an array of random floats with a normal distribution.
-    :param size: int: Number of elements to be generated.
-    :param mean: int: Mean of the distribution.
-    :param std: int: Standard deviation of the distribution.
-    :param round: int: Number of decimal places to round the generated numbers. Default is 2.
-    :return: np.ndarray: Array of random floats.
-    """
-    return np.round(np.random.normal(mean, std, size), round)
-
-
-
-
-if __name__ == '__main__':
-  pass

rand_engine-0.4.4/rand_engine/main/data_generator.py

@@ -1,88 +0,0 @@
-import os
-import time
-import pandas as pd
-from typing import List, Dict, Optional, Generator, Callable, Any
-from rand_engine.main.file_writer import FileWriter
-from rand_engine.main.stream_handle import StreamHandle
-
-class DataGenerator:
-      
-  def __init__(self, random_spec):
-    self.random_spec = random_spec
-    self.actual_dataframe = None
-
-
-  def handle_splitable(self, metadata, df):
-    for key, value in metadata.items():
-      if value.get("splitable"):
-        sep = value.get("sep", ";")
-        cols = value.get("cols")
-        df[cols] = df[key].str.split(sep, expand=True)
-        df.drop(columns=[key], inplace=True)
-    return df
-
-
-  def generate_pandas_df(self, size: int, transformer: Optional[Callable]=None) -> pd.DataFrame:
-    """
-    This method generates a pandas DataFrame based on random data specified in the metadata parameter.
-    :param size: int: Number of rows to be generated.
-    :param transformer: Optional[Callable]: Function to transform the generated data.
-    :return: pd.DataFrame: DataFrame with the generated data.
-    """
-    assert type(self.random_spec) is dict, "You need to pass a random_spec parameter to generate the random data."
-    def first_level():
-      dict_data = {key: value["method"](size, **value["parms"]) for key, value in self.random_spec.items()}
-      df_pandas = pd.DataFrame(dict_data)
-      df_pandas = self.handle_splitable(self.random_spec, df_pandas)
-      if transformer: return transformer(df_pandas)
-      return df_pandas
-    self.actual_dataframe = first_level
-    return self
-  
-
-  def generate_spark_df(self, spark, size: int, transformer: Optional[Callable]=None) -> Any:
-    """
-    This method generates a Spark DataFrame based on random data specified in the random_spec parameter.
-    :param spark: SparkSession: SparkSession object.
-    :param size: int: Number of rows to be generated.
-    :param transformer: Optional[Callable]: Function to transform the generated data."""
-    def second_level():
-      self.generate_pandas_df(size=size, transformer=transformer)
-      df_spark = spark.createDataFrame(self.actual_dataframe())
-      return df_spark
-    self.actual_dataframe = second_level
-    return self
-
-
-  def get_df(self):
-    assert self.actual_dataframe is not None, "You need to generate a DataFrame first."
-    return self.actual_dataframe()
-
-
-  def stream_dict(self, min_throughput: int=1, max_throughput: int = 10) -> Generator:
-    """
-    This method creates a generator of records to be used in a streaming context.
-    :param min_throughput: int: Minimum throughput to be generated.
-    :param max_throughput: int: Maximum throughput to be generated.
-    :return: Generator: Generator of records.
-    """
-    assert self.actual_dataframe is not None, "You need to generate a DataFrame first."
-    while True:
-      df_data_microbatch = self.actual_dataframe()
-      df_data_parsed = StreamHandle.convert_dt_to_str(df_data_microbatch)
-      list_of_records = df_data_parsed.to_dict('records')
-      for record in list_of_records:
-        record["timestamp_created"] = round(time.time(), 3)
-        yield record
-        StreamHandle.sleep_to_contro_throughput(min_throughput, max_throughput)
-  
-
-  def write(self):
-    microbatch_def = lambda: self.actual_dataframe
-    return FileWriter(microbatch_def)
-   
-
-
-if __name__ == '__main__':
-  
-  pass