rand-engine 0.3.3__tar.gz → 0.3.7__tar.gz

{rand_engine-0.3.3 → rand_engine-0.3.7}/PKG-INFO

@@ -1,8 +1,7 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.3
 Name: rand-engine
-Version: 0.3.3
+Version: 0.3.7
 Summary: Rand Engine v2. Package with some methods to generate random data in different formats. Great to mock data while testing or developing.
-Home-page: https://github.com/marcoaureliomenezes/rand_engine
 Author: marcoaureliomenezes
 Author-email: marcoaurelioreislima@gmail.com
 Requires-Python: >=3.10,<4.0
@@ -11,6 +10,8 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
+Requires-Dist: fastavro (>=1.10.0,<2.0.0)
+Requires-Dist: fastparquet (>=2024.11.0,<2025.0.0)
 Requires-Dist: numpy (>=2.1.1,<3.0.0)
 Requires-Dist: pandas (>=2.2.2,<3.0.0)
 Project-URL: Repository, https://github.com/marcoaureliomenezes/rand_engine
@@ -42,4 +43,6 @@ CoreDistinct().randint(0, 100, 10)
 
 ```
 
+git tag 0.0.1
 
+git push origin --tags

{rand_engine-0.3.3 → rand_engine-0.3.7}/README.md

@@ -24,3 +24,6 @@ CoreDistinct().randint(0, 100, 10)
 
 ```
 
+git tag 0.0.1
+
+git push origin --tags

{rand_engine-0.3.3 → rand_engine-0.3.7}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "rand-engine"
-version = "0.3.3"
+version = "0.3.7"
 description = "Rand Engine v2. Package with some methods to generate random data in different formats. Great to mock data while testing or developing."
 authors = ["marcoaureliomenezes <marcoaurelioreislima@gmail.com>"]
 repository = "https://github.com/marcoaureliomenezes/rand_engine"
@@ -10,7 +10,8 @@ readme = "README.md"
 python = "^3.10"
 numpy = "^2.1.1"
 pandas = "^2.2.2"
-
+fastavro = "^1.10.0"
+fastparquet = "^2024.11.0"
 
 [tool.poetry.group.test.dependencies]
 pytest = "^8.3.3"

{rand_engine-0.3.3 → rand_engine-0.3.7}/rand_engine/core/datetime_core.py

@@ -6,7 +6,13 @@ from datetime import datetime as dt, timedelta
 class DatetimeCore:
   
   @classmethod
-  def gen_unix_timestamps(self, size: int, start: str, end: str, format: str):
+  def gen_unix_timestamps(self, size: int, start: str, end: str, format: str) -> np.ndarray:
+    """
+    This method generates an array of random unix timestamps.
+    :param size: int: Number of elements to be generated.
+    :param start: str: Start date of the generated timestamps.
+    :param end: str: End date of the generated timestamps.
+    :param format: str: Format of the input dates."""
     dt_start, dt_end = dt.strptime(start, format), dt.strptime(end, format)
     if dt_start < dt(1970, 1, 1): dt_start = dt(1970, 1, 1)
     timestamp_start, timestamp_end = dt_start.timestamp(), dt_end.timestamp()
@@ -15,7 +21,15 @@ class DatetimeCore:
 
 
   @classmethod
-  def gen_timestamps(self, size: int, start: str, end: str, format: str):
+  def gen_timestamps(self, size: int, start: str, end: str, format: str) -> np.ndarray:
+    """
+    This method generates an array of random timestamps.
+    :param size: int: Number of elements to be generated.
+    :param start: str: Start date of the generated timestamps.
+    :param end: str: End date of the generated timestamps.
+    :param format: str: Format of the input dates.
+    :return: np.ndarray: Array of random timestamps."""
+
     date_array = self.gen_unix_timestamps(size, start, end, format).astype('datetime64[s]')
     return date_array
   
@@ -25,16 +39,7 @@ class DatetimeCore:
     timestamp_array = self.gen_unix_timestamps(size, start, end, format_in)
     return [dt.fromtimestamp(i).strftime(format_out) for i in timestamp_array]
 
-if __name__ == '__main__':
-
-  pass
 
 
-# def format_date_array(date_array, format):
-#     return [datetime.fromtimestamp(i).strftime(format) for i in date_array]
-# def handle_format(format):
-#     return format[randint(0, len(format))] if format == list else \
-#             format if format == str else "%d-%m-%Y"
-# def get_interval(start, end, date_format):
-#     return datetime.timestamp(datetime.strptime(start, date_format)), \
-#             datetime.timestamp(datetime.strptime(end, date_format))
+if __name__ == '__main__':
+  pass

{rand_engine-0.3.3 → rand_engine-0.3.7}/rand_engine/core/distinct_core.py

@@ -7,15 +7,36 @@ class DistinctCore:
     
   @classmethod
   def gen_distincts_typed(self, size: int, distinct: List[Any]) -> np.ndarray:
+    """
+    This method generates an array of random distinct values.
+    :param size: int: Number of elements to be generated.
+    :param distinct: List[Any]: List of distinct values to be generated.
+    :return: np.ndarray: Array of random distinct values.
+    """
     assert len(list(set([type(x) for x in distinct]))) == 1
     return np.random.choice(distinct, size)
     
+
   @classmethod
-  def gen_distincts_untyped(self, size: int, distinct: List[Any]) -> Iterator:
+  def gen_distincts_untyped(self, size: int, distinct: List[Any]) -> List[Any]:
+    """
+    This method generates an array of random distinct values.
+    :param size: int: Number of elements to be generated.
+    :param distinct: List[Any]: List of distinct values to be generated.
+    :return: Iterator: Iterator of random distinct values.
+    """
     return list(map(lambda x: distinct[x], np.random.randint(0, len(distinct), size)))
   
   @classmethod
   def gen_complex_distincts(self, size: int, pattern="x.x.x-x", replacement="x", templates=[]):
+    """
+    This method generates an array of random distinct values.
+    :param size: int: Number of elements to be generated.
+    :param pattern: str: Pattern to be replaced.
+    :param replacement: str: Replacement of the pattern.
+    :param templates: List[Dict]: List of dictionaries containing the method and parameters to be used in the replacement.
+    :return: np.ndarray: Array of random distinct values.
+    """
     assert pattern.count(replacement) == len(templates)
     list_of_lists, counter = [], 0
     for replacer_cursor in range(len(pattern)):
@@ -27,6 +48,7 @@ class DistinctCore:
     return reduce(lambda a, b: a.astype('str') + b.astype('str'), list_of_lists)
   
   
+  
 if __name__ == '__main__':
   pass
 

{rand_engine-0.3.3 → rand_engine-0.3.7}/rand_engine/core/distinct_utils.py

@@ -6,10 +6,19 @@ class DistinctUtils:
 
   @classmethod
   def handle_distincts_lvl_1(self, distinct_prop, precision):
+    """
+    This method generates a list of distinct values based on a dictionary of distinct values and their respective frequencies.
+    :param distinct_prop: dict: Dictionary containing the distinct values and their respective frequencies.
+    :param precision: int: Precision of the distinct values.
+    :return: List: List of distinct values.
+    """
     return [ key for key, value in distinct_prop.items() for i in range(value * precision )]
 
   @classmethod
   def handle_distincts_lvl_2(self, distincts, sep=";"):
+    """
+    This method generates a list of distinct values based on a dictionary of distinct values and their respective frequencies.
+    :param distincts: dict: Dictionary containing the distinct values and their respective frequencies."""
     data_flatted = [f"{j}{sep}{i}" for j in distincts for i in distincts[j]]
     return data_flatted
 
@@ -39,16 +48,12 @@ if __name__ == '__main__':
   distinct_2 = {"OPC": [{"C_OPC": ["PF", "PJ"]}, {"V_OPC": ["NA"]}], "SWP": [{"C_SWP": ["AP"]}, {"V_SWP": ["MA", "ME"]}]}
   #print(DistinctUtils.handle_distincts_lvl_5(distinct_2)[]
 
-
   def rec(structure):
     if isinstance(structure, list):
       return [rec(i) for i in structure]
     if isinstance(structure, dict):
       return [[[k], rec(v)] for k, v in structure.items()]
     return structure
-
   import numpy as np
-
   result = rec(distinct_2)
-
   combinations = np.array(list(itertools.product(*result)))

rand_engine-0.3.7/rand_engine/core/numeric_core.py

@@ -0,0 +1,62 @@
+import numpy as np
+
+
+class NumericCore:
+
+
+  @classmethod
+  def gen_ints(self, size: int, min: int, max: int) -> np.ndarray:
+    """
+    This method generates an array of random integers.
+    :param size: int: Number of elements to be generated.
+    :param min: int: Minimum value of the generated numbers.
+    :param max: int: Maximum value of the generated numbers.
+    :return: np.ndarray: Array of random integers.
+    """
+    return np.random.randint(min, max + 1, size)
+
+
+  @classmethod
+  def gen_ints_zfilled(self, size: int, length: int) -> np.ndarray:
+    """
+    This method generates an array of random integers with a fixed length.
+    :param size: int: Number of elements to be generated.
+    :param length: int: Length of the generated numbers.
+    :return: np.ndarray: Array of random integers.
+    """
+    str_arr = np.random.randint(0, 10**length, size).astype('str')
+    return np.char.zfill(str_arr, length)
+  
+  
+  @classmethod
+  def gen_floats(self, size: int, min: int, max: int, round: int = 2):
+    """
+    This method generates an array of random floats.
+    :param size: int: Number of elements to be generated.
+    :param min: int: Minimum value of the generated numbers.
+    :param max: int: Maximum value of the generated numbers.
+    :param round: int: Number of decimal places to round the generated numbers. Default is 2.
+    :return: np.ndarray: Array of random floats.
+    """
+    sig_part = np.random.randint(min, max, size)
+    decimal = np.random.randint(0, 10 ** round, size)
+    return sig_part + (decimal / 10 ** round) if round > 0 else sig_part
+
+
+  @classmethod
+  def gen_floats_normal(self, size: int, mean: int, std: int, round: int = 2):
+    """
+    This method generates an array of random floats with a normal distribution.
+    :param size: int: Number of elements to be generated.
+    :param mean: int: Mean of the distribution.
+    :param std: int: Standard deviation of the distribution.
+    :param round: int: Number of decimal places to round the generated numbers. Default is 2.
+    :return: np.ndarray: Array of random floats.
+    """
+    return np.round(np.random.normal(mean, std, size), round)
+
+
+
+
+if __name__ == '__main__':
+  pass

rand_engine-0.3.7/rand_engine/main/data_generator.py

@@ -0,0 +1,88 @@
+import os
+import time
+import pandas as pd
+from typing import List, Dict, Optional, Generator, Callable, Any
+from rand_engine.main.file_writer import FileWriter
+from rand_engine.main.stream_handle import StreamHandle
+
+class DataGenerator:
+      
+  def __init__(self, random_spec):
+    self.random_spec = random_spec
+    self.actual_dataframe = None
+
+
+  def handle_splitable(self, metadata, df):
+    for key, value in metadata.items():
+      if value.get("splitable"):
+        sep = value.get("sep", ";")
+        cols = value.get("cols")
+        df[cols] = df[key].str.split(sep, expand=True)
+        df.drop(columns=[key], inplace=True)
+    return df
+
+
+  def generate_pandas_df(self, size: int, transformer: Optional[Callable]=None) -> pd.DataFrame:
+    """
+    This method generates a pandas DataFrame based on random data specified in the metadata parameter.
+    :param size: int: Number of rows to be generated.
+    :param transformer: Optional[Callable]: Function to transform the generated data.
+    :return: pd.DataFrame: DataFrame with the generated data.
+    """
+    assert type(self.random_spec) is dict, "You need to pass a random_spec parameter to generate the random data."
+    def first_level():
+      dict_data = {key: value["method"](size, **value["parms"]) for key, value in self.random_spec.items()}
+      df_pandas = pd.DataFrame(dict_data)
+      df_pandas = self.handle_splitable(self.random_spec, df_pandas)
+      if transformer: df_pandas = transformer(df_pandas)
+      return df_pandas
+    self.actual_dataframe = first_level
+    return self
+  
+
+  def generate_spark_df(self, spark, size: int, transformer: Optional[Callable]=None) -> Any:
+    """
+    This method generates a Spark DataFrame based on random data specified in the random_spec parameter.
+    :param spark: SparkSession: SparkSession object.
+    :param size: int: Number of rows to be generated.
+    :param transformer: Optional[Callable]: Function to transform the generated data."""
+    def second_level():
+      self.generate_pandas_df(size=size, transformer=transformer)
+      df_spark = spark.createDataFrame(self.actual_dataframe())
+      return df_spark
+    self.actual_dataframe = second_level
+    return self
+
+
+  def get_df(self):
+    assert self.actual_dataframe is not None, "You need to generate a DataFrame first."
+    return self.actual_dataframe()
+
+
+  def stream_dict(self, min_throughput: int=1, max_throughput: int = 10) -> Generator:
+    """
+    This method creates a generator of records to be used in a streaming context.
+    :param min_throughput: int: Minimum throughput to be generated.
+    :param max_throughput: int: Maximum throughput to be generated.
+    :return: Generator: Generator of records.
+    """
+    assert self.actual_dataframe is not None, "You need to generate a DataFrame first."
+    while True:
+      df_data_microbatch = self.actual_dataframe()
+      df_data_parsed = StreamHandle.convert_dt_to_str(df_data_microbatch)
+      list_of_records = df_data_parsed.to_dict('records')
+      for record in list_of_records:
+        record["timestamp_created"] = round(time.time(), 3)
+        yield record
+        StreamHandle.sleep_to_contro_throughput(min_throughput, max_throughput)
+  
+
+  def write(self):
+    microbatch_def = lambda: self.actual_dataframe
+    return FileWriter(microbatch_def)
+   
+
+
+if __name__ == '__main__':
+  
+  pass

rand_engine-0.3.7/rand_engine/main/file_writer.py

@@ -0,0 +1,131 @@
+import os
+from typing import Callable
+
+
+class FileWriter:
+
+
+  def __init__(self, microbatch_def):
+    self.microbatch_def = microbatch_def
+    self.write_format = "csv"
+    self.write_mode = "overwrite"
+    self.write_options = {}
+    self.dict_format = {
+        "csv": self.to_csv,
+        "parquet": self.to_parquet
+    }
+
+  def __handle_fs(self, path, flag=True) -> None:
+    """
+    This method handles the file system operations.
+    :param path: str: Path of the file to be written.
+    """
+    if self.write_mode == "overwrite":
+      try:
+        if os.path.exists(path):
+          for file in os.listdir(path):
+            os.remove(os.path.join(path, file))
+      except Exception as e: pass
+    if flag == True: to_create = os.path.dirname(path)
+    else: to_create = path
+    os.makedirs(to_create, exist_ok=True)  
+
+
+  def __get_dir_size(self, folder_path: str) -> int:
+    """
+    This method calculates the size in bytes of a directory.
+    :param folder_path: str: Path of the directory.
+    :return: int: Size of the directory in bytes.
+    """
+    total_size = 0
+    for dirpath, dirnames, filenames in os.walk(folder_path):
+      for filename in filenames:
+        file_path = os.path.join(dirpath, filename)
+        if not os.path.islink(file_path):
+          total_size += os.path.getsize(file_path)
+    return total_size
+  
+
+  def mode(self, write_mode: str) -> Callable:
+    """
+    This method sets the write mode of the file.
+    :param write_mode: str: Write mode of the file. Default is overwrite.
+    :return: FileWriter: Instance of the fileWriter class for method chaining.
+    """
+    self.write_mode = write_mode
+    return self
+
+
+  def format(self, format):
+    """
+    This method sets the write format of the file.
+    :param format: str: Write format of the file. Default is csv. Supported formats are csv and parquet.
+    :return: FileWriter: Instance of the fileWriter class for method chaining.
+    """
+    self.write_format = format
+    return self
+  
+
+  def option(self, key, value):
+    """
+    This method sets the write options of the file.
+    :param key: str: Key of the write option.
+    :param value: Any: Value of the write option.
+    :return: FileWriter: Instance of the fileWriter class for method chaining
+    """
+    self.write_options[key] = value
+    return self
+  
+
+  def to_csv(self, dataframe, full_path) -> Callable:
+    """
+    This method writes a pandas DataFrame to a csv file.
+    :param dataframe: pd.DataFrame: DataFrame to be written.
+    :param full_path: str: Full path of the file to be written.
+    :return: Callable: Function to write the Pandas DataFrame to a csv file.
+    """
+    if self.write_options.get("compression"):
+      full_path= full_path.replace("csv", f"csv.{self.write_options['compression']}")
+    writer = lambda: dataframe().to_csv(full_path, index=False, **self.write_options)
+    return writer
+  
+
+  def to_parquet(self, dataframe, full_path):
+    """
+    This method writes a pandas DataFrame to a parquet file.
+    :param dataframe: pd.DataFrame: DataFrame to be written.
+    :param full_path: str: Full path of the file to be written.
+    :return: Callable: Function to write the Pandas DataFrame to a parquet file.
+    """
+    if self.write_options.get("compression"):
+      full_path= full_path.replace(".parquet", f".{self.write_options['compression']}.parquet")
+    writer = lambda: dataframe().to_parquet(full_path, index=False, engine='pyarrow', **self.write_options)
+    return writer
+
+
+  def load(self, path: str) -> None:
+    """
+    This method writes a pandas DataFrame to a file.
+    :param path: str: Path of the file to be written.
+    """
+    self.__handle_fs(path)
+    dataframe = self.microbatch_def()
+    self.dict_format[self.write_format](dataframe, path)()
+
+
+  def incr_load(self, path, size_in_mb=4):
+    """
+    This method writes a pandas DataFrame to a file in incremental mode.
+    :param path: str: Path of the file to be written.
+    :param size_in_mb: int: Size in MB of the file to be written.
+    """
+    self.__handle_fs(path, flag=False)
+    counter = 0
+    while True:
+      full_path = f"{path}/part-{str(counter).zfill(6)}.{self.write_format}"
+      dataframe = self.microbatch_def()
+      self.dict_format[self.write_format](dataframe, full_path)()
+      size_bytes = self.__get_dir_size(path)
+      if counter % 100 == 0: print(f"Size: {size_bytes/2**20:.2f} MB")
+      if self.__get_dir_size(path) >= size_in_mb*2**20: break
+      counter += 1

rand_engine-0.3.7/rand_engine/main/stream_handle.py

@@ -0,0 +1,18 @@
+import random
+import time
+import pandas as pd
+
+class StreamHandle:
+  
+  @staticmethod
+  def convert_dt_to_str(dataframe: pd.DataFrame) -> pd.DataFrame:
+    df_result = dataframe.copy()
+    for column in df_result.columns:
+      if 'datetime64' in str(df_result[column].dtype):
+        df_result[column] = df_result[column].astype(str)
+    return df_result
+  
+  @staticmethod
+  def sleep_to_contro_throughput(min_throughput: int, max_throughput: int):
+    sleep_time = 1 / random.uniform(min_throughput, max_throughput)
+    time.sleep(sleep_time)

rand_engine-0.3.3/rand_engine/core/numeric_core.py

@@ -1,33 +0,0 @@
-import numpy as np
-
-
-class NumericCore:
-
-
-  @classmethod
-  def gen_ints(self, size: int, min: int, max: int):
-    return np.random.randint(min, max + 1, size)
-
-
-  @classmethod
-  def gen_ints_zfilled(self, size: int, length: int) -> np.ndarray:
-    str_arr = np.random.randint(0, 10**length, size).astype('str')
-    return np.char.zfill(str_arr, length)
-  
-  
-  @classmethod
-  def gen_floats(self, size: int, min: int, max: int, round: int = 2):
-    sig_part = np.random.randint(min, max, size)
-    decimal = np.random.randint(0, 10 ** round, size)
-    return sig_part + (decimal / 10 ** round) if round > 0 else sig_part
-
-
-  @classmethod
-  def gen_floats_normal(self, size: int, mean: int, std: int, round: int = 2):
-    return np.round(np.random.normal(mean, std, size), round)
-
-
-
-
-if __name__ == '__main__':
-  pass

rand_engine-0.3.3/rand_engine/main/data_generator.py

@@ -1,72 +0,0 @@
-import os
-import time
-import random
-import pandas as pd
-from typing import List, Dict, Optional, Generator, Callable, Any
-
-
-class DataGenerator:
-      
-
-  def handle_splitable(self, metadata, df):
-    for key, value in metadata.items():
-      if value.get("splitable"):
-        sep = value.get("sep", ";")
-        cols = value.get("cols")
-        df[cols] = df[key].str.split(sep, expand=True)
-        df.drop(columns=[key], inplace=True)
-    return df
-
-
-  def __convert_dt_to_str(self, dataframe: pd.DataFrame) -> pd.DataFrame:
-      df_result = dataframe.copy()
-      for column in df_result.columns:
-          if 'datetime64' in str(df_result[column].dtype):
-              df_result[column] = df_result[column].astype(str)
-      return df_result
-  
-      
-  def __sleep_to_contro_throughput(self, min_throughput: int, max_throughput: int):
-    sleep_time = 1 / random.uniform(min_throughput, max_throughput)
-    time.sleep(sleep_time)
-
-
-  def create_pandas_df(self, size: int, metadata: dict, transformer: Optional[Callable]=None) -> pd.DataFrame:
-    dict_data = {key: value["method"](size, **value["parms"]) for key, value in metadata.items()}
-    df_data = pd.DataFrame(dict_data)
-    df_data_final = self.handle_splitable(metadata, df_data)
-    if transformer: df_data_final = transformer(df_data_final)
-    return df_data_final
-  
-
-  def create_spark_df(self, spark, size: int, metadata: dict, transformer: Optional[Callable]=None) -> Any:
-    df_data = self.create_pandas_df(size=size, metadata=metadata, transformer=transformer)
-    df_final = spark.createDataFrame(df_data)
-    return df_final
-
-
-  def create_streaming_records(self, microbatch_size:int, metadata: dict, transformer: Optional[Callable]=None, min_throughput: int=1, max_throughput: int = 10) -> Generator:
-    while True:
-      df_data_microbatch = self.create_pandas_df(size=microbatch_size, metadata=metadata, transformer=transformer)
-      df_data_parsed = self.__convert_dt_to_str(df_data_microbatch)
-      list_of_records = df_data_parsed.to_dict('records')
-      for record in list_of_records:
-          record["timestamp_created"] = round(time.time(), 3)
-          yield record
-          self.__sleep_to_contro_throughput(min_throughput, max_throughput)
-  
-
-  def create_csv_file(self, microbatch_size: int, size_in_mb: int, metadata: dict, path: str, transformer: Optional[Callable]=None) -> None:
-    os.makedirs(os.path.dirname(path), exist_ok=True)
-    while True:
-      df = self.create_pandas_df(size=microbatch_size, metadata=metadata, transformer=transformer)
-      df.to_csv(path, mode='a', header=False, index=False)
-      if os.path.getsize(path) > size_in_mb * 1024 * 1024: break
-
-
-
-
-   
-
-if __name__ == '__main__':
-  pass