pipelet 0.0.2__tar.gz → 0.0.3__tar.gz

{pipelet-0.0.2 → pipelet-0.0.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pipelet
-Version: 0.0.2
+Version: 0.0.3
 Summary: 
 Author: Daniel Naumowich
 Author-email: cskteam4@gmail.com
@@ -69,7 +69,7 @@ A processor for parsing CSV files using the `pandas` library. It supports numero
 ### 7. **JsonParser**
 A processor for parsing JSON data from strings or files into Python dictionaries. It supports configuring how numbers, constants (e.g., NaN), and JSON objects are handled with custom functions.
 
-### 8. **BaseUnzipProcessor**
+### 8. **UnzipProcessor**
 A processor for extracting files from `.zip` archives. It supports extracting data into files, managing chunk sizes, and deleting the original archive after processing.
 
 ### 9. **RetryProcessor**
@@ -91,7 +91,7 @@ A processor designed to handle transient errors by retrying operations based on
 from pipelet.processors.http import HttpDataExtractProcessor
 from pipelet.processors.csv_parser import CsvParser
 from pipelet.processors.json_parser import JsonParser
-from pipelet.processors.unzip import BaseUnzipProcessor
+from pipelet.processors.unzip import UnzipProcessor
 from pipelet.processors.file_system import AbstractFileSystemManager
 
 # Create file system manager
@@ -101,7 +101,7 @@ file_system_manager = AbstractFileSystemManager()
 http_processor = HttpDataExtractProcessor(file_system_manager)
 csv_parser = CsvParser(file_system_manager)
 json_parser = JsonParser(file_system_manager)
-unzip_processor = BaseUnzipProcessor(file_system_manager)
+unzip_processor = UnzipProcessor(file_system_manager)
 
 # Create processor chain
 http_processor.set_next(csv_parser).set_next(json_parser).set_next(unzip_processor)
@@ -118,7 +118,7 @@ for output in http_processor.process(input_data):
 from pipelet.processors.http import HttpDataExtractProcessor
 from pipelet.processors.csv_parser import CsvParser
 from pipelet.processors.json_parser import JsonParser
-from pipelet.processors.unzip import BaseUnzipProcessor
+from pipelet.processors.unzip import UnzipProcessor
 from pipelet.processors.file_system import AbstractFileSystemManager
 
 # Create file system manager
@@ -128,7 +128,7 @@ file_system_manager = AbstractFileSystemManager()
 http_processor = HttpDataExtractProcessor(file_system_manager)
 csv_parser = CsvParser(file_system_manager)
 json_parser = JsonParser(file_system_manager)
-unzip_processor = BaseUnzipProcessor(file_system_manager)
+unzip_processor = UnzipProcessor(file_system_manager)
 
 # Create processor chain using `>>`
 pipeline = http_processor >> csv_parser >> json_parser >> unzip_processor
@@ -147,23 +147,35 @@ from pipelet.processors.file_system import FileModeEnum
 
 def main():
     # Define the pipeline with parallel processing for parsing and unzipping
+
     pipeline = [
         Op(
             BaseOpEnum.downloading, 
             kwargs={
-                "file_system_manager": {...},
+                "file_system_manager": {
+                    "path": ..., # Here can be your custom file system manager (as an example S3)
+                    "kwargs": {...},
+                },
                 "retry_args": {
+                    "retry_processor": ..., # Here can be your custom retry processor
                     "max_retries": 3,
                     "delay": 1,
                     "delay_step": 1,
                     "retry_with_white_exc": True,
                 },
+                "logging_args": {
+                    "logger": ..., # Here can be your custom logger
+                    "logging": True, # Logging intup data
+                },
             }
         ),
         Op(
             BaseOpEnum.upzip,
             kwargs={
-                "file_system_manager": {...}
+                "file_system_manager": {
+                    "path": ..., # Here can be your custom file system manager (as an example S3)
+                    "kwargs": {...},
+                },
             }
         ),
         OpAny(
@@ -171,13 +183,19 @@ def main():
                 Op(
                     BaseOpEnum.json_parsing, 
                     kwargs={
-                        "file_system_manager": {...}
+                        "file_system_manager": {
+                            "path": ..., # Here can be your custom file system manager (as an example S3)
+                            "kwargs": {...},
+                        },
                     }
                 ),
                 Op(
                     BaseOpEnum.csv_parsing, 
                     kwargs={
-                        "file_system_manager": {...},
+                        "file_system_manager": {
+                            "path": ..., # Here can be your custom file system manager (as an example S3)
+                            "kwargs": {...},
+                        },,
                         "white_exceptions": [
                             TypeError,
                             BaseExceptionsEnum.csv_parsing_error,
@@ -213,8 +231,10 @@ The library is developer-friendly, making it easy to extend and integrate custom
 
 ```python
 from pipelet.processors.base import BaseProcessor
+from pipelet.decorators import logging_decorator
 
 class CustomProcessor(BaseProcessor):
+    @logging_decorator(custom_logger=...) # If you need this you can use either a custom logger or a default logger
     def process(self, input_data):
         # Custom data processing logic
         yield transformed_data

{pipelet-0.0.2 → pipelet-0.0.3}/README.md

@@ -49,7 +49,7 @@ A processor for parsing CSV files using the `pandas` library. It supports numero
 ### 7. **JsonParser**
 A processor for parsing JSON data from strings or files into Python dictionaries. It supports configuring how numbers, constants (e.g., NaN), and JSON objects are handled with custom functions.
 
-### 8. **BaseUnzipProcessor**
+### 8. **UnzipProcessor**
 A processor for extracting files from `.zip` archives. It supports extracting data into files, managing chunk sizes, and deleting the original archive after processing.
 
 ### 9. **RetryProcessor**
@@ -71,7 +71,7 @@ A processor designed to handle transient errors by retrying operations based on
 from pipelet.processors.http import HttpDataExtractProcessor
 from pipelet.processors.csv_parser import CsvParser
 from pipelet.processors.json_parser import JsonParser
-from pipelet.processors.unzip import BaseUnzipProcessor
+from pipelet.processors.unzip import UnzipProcessor
 from pipelet.processors.file_system import AbstractFileSystemManager
 
 # Create file system manager
@@ -81,7 +81,7 @@ file_system_manager = AbstractFileSystemManager()
 http_processor = HttpDataExtractProcessor(file_system_manager)
 csv_parser = CsvParser(file_system_manager)
 json_parser = JsonParser(file_system_manager)
-unzip_processor = BaseUnzipProcessor(file_system_manager)
+unzip_processor = UnzipProcessor(file_system_manager)
 
 # Create processor chain
 http_processor.set_next(csv_parser).set_next(json_parser).set_next(unzip_processor)
@@ -98,7 +98,7 @@ for output in http_processor.process(input_data):
 from pipelet.processors.http import HttpDataExtractProcessor
 from pipelet.processors.csv_parser import CsvParser
 from pipelet.processors.json_parser import JsonParser
-from pipelet.processors.unzip import BaseUnzipProcessor
+from pipelet.processors.unzip import UnzipProcessor
 from pipelet.processors.file_system import AbstractFileSystemManager
 
 # Create file system manager
@@ -108,7 +108,7 @@ file_system_manager = AbstractFileSystemManager()
 http_processor = HttpDataExtractProcessor(file_system_manager)
 csv_parser = CsvParser(file_system_manager)
 json_parser = JsonParser(file_system_manager)
-unzip_processor = BaseUnzipProcessor(file_system_manager)
+unzip_processor = UnzipProcessor(file_system_manager)
 
 # Create processor chain using `>>`
 pipeline = http_processor >> csv_parser >> json_parser >> unzip_processor
@@ -127,23 +127,35 @@ from pipelet.processors.file_system import FileModeEnum
 
 def main():
     # Define the pipeline with parallel processing for parsing and unzipping
+
     pipeline = [
         Op(
             BaseOpEnum.downloading, 
             kwargs={
-                "file_system_manager": {...},
+                "file_system_manager": {
+                    "path": ..., # Here can be your custom file system manager (as an example S3)
+                    "kwargs": {...},
+                },
                 "retry_args": {
+                    "retry_processor": ..., # Here can be your custom retry processor
                     "max_retries": 3,
                     "delay": 1,
                     "delay_step": 1,
                     "retry_with_white_exc": True,
                 },
+                "logging_args": {
+                    "logger": ..., # Here can be your custom logger
+                    "logging": True, # Logging intup data
+                },
             }
         ),
         Op(
             BaseOpEnum.upzip,
             kwargs={
-                "file_system_manager": {...}
+                "file_system_manager": {
+                    "path": ..., # Here can be your custom file system manager (as an example S3)
+                    "kwargs": {...},
+                },
             }
         ),
         OpAny(
@@ -151,13 +163,19 @@ def main():
                 Op(
                     BaseOpEnum.json_parsing, 
                     kwargs={
-                        "file_system_manager": {...}
+                        "file_system_manager": {
+                            "path": ..., # Here can be your custom file system manager (as an example S3)
+                            "kwargs": {...},
+                        },
                     }
                 ),
                 Op(
                     BaseOpEnum.csv_parsing, 
                     kwargs={
-                        "file_system_manager": {...},
+                        "file_system_manager": {
+                            "path": ..., # Here can be your custom file system manager (as an example S3)
+                            "kwargs": {...},
+                        },,
                         "white_exceptions": [
                             TypeError,
                             BaseExceptionsEnum.csv_parsing_error,
@@ -193,8 +211,10 @@ The library is developer-friendly, making it easy to extend and integrate custom
 
 ```python
 from pipelet.processors.base import BaseProcessor
+from pipelet.decorators import logging_decorator
 
 class CustomProcessor(BaseProcessor):
+    @logging_decorator(custom_logger=...) # If you need this you can use either a custom logger or a default logger
     def process(self, input_data):
         # Custom data processing logic
         yield transformed_data

pipelet-0.0.3/pipelet/decarators/__init__.py

@@ -0,0 +1,3 @@
+from pipelet.decarators.logging_decorator import logging_decorator
+
+__all__ = ("logging_decorator",)

pipelet-0.0.3/pipelet/decarators/logging_decorator.py

@@ -0,0 +1,65 @@
+from functools import wraps
+from typing import Any, Callable, Generator, Optional, Union
+
+from pipelet.decarators.option_decorator import prepare_gen_processor
+from pipelet.log import logger_factory
+from pipelet.processors.base import BaseProcessor
+
+default_logger = logger_factory()
+
+
+def logging_decorator(
+    func: Callable[
+        ...,
+        Generator[Any, Any, Optional[Any]],
+    ],
+    custom_logger: Optional[Any] = None,
+) -> Callable[
+    ...,
+    Generator[Any, Any, Optional[Any]],
+]:
+    """
+    A decorator that adds logging functionality to a processor's `process` method.
+
+    Logs the processor's class name, arguments, and keyword arguments each time
+    the `process` method is invoked. Supports generator functions and normalizes
+    calls to ensure compatibility with bound and unbound methods.
+
+    Args:
+        func (Callable): The generator function (`process` method) to wrap.
+        custom_logger (Optional[Any]): Custom logger to use. Defaults to the global logger.
+
+    Returns:
+        Callable: A wrapped generator function with logging functionality.
+
+    Example:
+        ```
+        @logging_decorator
+        def process(self, *args, **kwargs):
+            yield ...
+        ```
+    """
+    logger_to_use = (
+        custom_logger or default_logger
+    )  # Используем кастомный логгер, если он передан
+
+    @wraps(func)
+    def wrapper(
+        obj: BaseProcessor[Any, Any, Any, Any],
+        *args,
+        **kwargs,
+    ) -> Generator[
+        Any,
+        Any,
+        Union[Any, None],
+    ]:
+        logger_to_use.info(
+            f"Processor: {obj.__class__.__name__}, args: {args}, kwargs: {kwargs}"
+        )
+        generator = prepare_gen_processor(
+            func, obj, *args, **kwargs
+        )  # Normalized call
+        while True:
+            yield next(generator)
+
+    return wrapper

pipelet-0.0.3/pipelet/decarators/option_decorator.py

@@ -0,0 +1,36 @@
+import types
+from typing import Any, Callable, Generator
+
+from pipelet.log import logger_factory
+from pipelet.processors.base import BaseProcessor
+
+logger = logger_factory()
+
+
+def prepare_gen_processor(
+    func: Callable[..., Generator[Any, Any, Any]],
+    obj: BaseProcessor[Any, Any, Any, Any],
+    *args,
+    **kwargs
+) -> Generator[Any, Any, Any]:
+    """
+    A utility function to prepare and execute generator-based processors.
+
+    This function normalizes the invocation of generator functions or methods.
+    If the provided callable `func` is a bound method, it is invoked directly
+    with `*args` and `**kwargs`. If `func` is an unbound function, it is
+    invoked with `obj` passed as the first argument along with `*args` and `**kwargs`.
+
+    Args:
+        func (Callable): The generator function or method to be executed.
+        obj (BaseProcessor): The processor object, passed explicitly for unbound functions.
+        *args: Positional arguments to pass to the generator.
+        **kwargs: Keyword arguments to pass to the generator.
+
+    Returns:
+        Generator: The generator object produced by executing `func`.
+
+    """
+    if isinstance(func, types.MethodType):
+        return func(*args, **kwargs)
+    return func(obj, *args, **kwargs)

{pipelet-0.0.2 → pipelet-0.0.3}/pipelet/etl/extract/http/http_extractor.py

@@ -102,7 +102,6 @@ class HttpDataExtractProcessor(
         Yields:
             PathLike | str: The file name or path of the downloaded file.
         """
-        logger.debug(f"Starting download from {input_data}")
         resp = httpx.get(
             input_data,
             params=self._params,
@@ -125,7 +124,6 @@ class HttpDataExtractProcessor(
             resp.content,
         )
 
-        logger.debug("Download was successful")
         if self._next is not None:
             yield from self._next.process(file_name)
         else:
@@ -218,7 +216,6 @@ class HttpxStreamDownloadProcessor(
         Yields:
             PathLike[str] | str: The file path of the downloaded file.
         """
-        logger.debug(f"Starting download from {input_data}")
         with httpx.stream(
             method=self._method,
             url=input_data,
@@ -238,7 +235,6 @@ class HttpxStreamDownloadProcessor(
                 if chunk:
                     self._file_system_manager.append_to_file(file_path, chunk)
 
-            logger.debug(f"Download was successful: {file_path}")
             if self._next is not None:
                 yield from self._next.process(file_path)
             else:

{pipelet-0.0.2 → pipelet-0.0.3}/pipelet/etl/transform/csv/reader.py

@@ -227,7 +227,6 @@ class CsvParser(
         Raises:
             CsvParsingError: If parsing fails.
         """
-        logger.debug(f"Start parsing: {input_data}")
         suffix = Path(input_data).suffix
         if suffix != ".csv":
             raise CsvParsingError(
@@ -292,7 +291,6 @@ class CsvParser(
                     yield records
         if self._auto_delete:
             self._file_system_manager.delete_file(input_data)
-        logger.debug(f"Successful parsing: {input_data}")
 
     def __str__(self) -> str:
         return (

{pipelet-0.0.2 → pipelet-0.0.3}/pipelet/etl/transform/json/reader.py

@@ -90,12 +90,10 @@ class JsonParser(
         """
         # Read content using the file system manager
         content = self._file_system_manager.read_file(input_data)
-        logger.debug("Read content from %s", input_data)
 
         # Parse the JSON content with custom arguments
         try:
             data = json.loads(content, **self._parse_kwargs)
-            logger.debug(f"JSON {input_data} has parsed successfully")
         except json.JSONDecodeError as e:
             logger.error("Failed to parse JSON: %s", e)
             raise

pipelet-0.0.3/pipelet/etl/transform/unzip/__init__.py

@@ -0,0 +1,3 @@
+from pipelet.etl.transform.unzip.base import UnzipProcessor
+
+__all__ = ("UnzipProcessor",)

{pipelet-0.0.2 → pipelet-0.0.3}/pipelet/etl/transform/unzip/base.py

@@ -15,7 +15,7 @@ from pipelet.processors.file_system import (
 logger = logger_factory()
 
 
-class BaseUnzipProcessor(
+class UnzipProcessor(
     BaseProcessor[
         Union[str, PathLike[str]], Union[str, PathLike[str]], None, None
     ],
@@ -45,7 +45,7 @@ class BaseUnzipProcessor(
         batch_size: Optional[int] = None,
     ) -> None:
         """
-        Initializes the BaseUnzipProcessor.
+        Initializes the UnzipProcessor.
 
         Args:
             file_system_manager (AbstractFileSystemManager): The file system manager for file operations.
@@ -84,7 +84,6 @@ class BaseUnzipProcessor(
             )
 
         file_path = self._file_system_manager.get_path(input_data)
-        logger.debug(f"Starting to unzip {input_data}")
 
         unzip_files: List[Union[str, PathLike[str]]] = []
         with zipfile.ZipFile(file_path, FileModeEnum.READ) as zip_ref:
@@ -107,8 +106,6 @@ class BaseUnzipProcessor(
 
                 unzip_files.append(extracted_file_path)
 
-        logger.debug("Unzipping was successful")
-
         if self._next is not None:
             for item in unzip_files:
                 gen = self._next.process(item)

{pipelet-0.0.2 → pipelet-0.0.3}/pipelet/operations/operations.py

@@ -24,7 +24,7 @@ Classes and Functions:
 - `BaseOpEnum`: Enum defining paths to processor classes for common operations.
 - `Op`: Represents a single operation configuration.
 - `OpAny`, `OpAll`: Define composite operations requiring any or all conditions.
-- `prepare_retry_processor`: Wraps a processor with retry logic.
+- `prepare_retry_option`: Wraps a option with retry logic.
 - `process_op`, `process_op_composite`: Handle individual and composite operation processing.
 - `get_processor_for_operation`: Dispatch function for retrieving the appropriate processor.
 
@@ -37,10 +37,12 @@ This framework enables modular, reusable, and extensible ETL workflows by abstra
 processing logic into well-defined processors and operations.
 """
 
+import types
 from dataclasses import dataclass, field
 from enum import Enum
 from typing import Any, Callable, Dict, Generic, List, Type, TypeVar, Union
 
+from pipelet.decarators.logging_decorator import logging_decorator
 from pipelet.factories import KEYWORD_FACTORY_MAP
 from pipelet.log import logger_factory
 from pipelet.processors import get_processor
@@ -59,11 +61,12 @@ class BaseOpEnum(str, Enum):
 
     downloading = "pipelet.etl.extract.http.HttpDataExtractProcessor"
     stream_downloading = "pipelet.etl.extract.http.HttpxStreamDownloadProcessor"
-    unzip = "pipelet.etl.transform.unzip.BaseUnzipProcessor"
+    unzip = "pipelet.etl.transform.unzip.UnzipProcessor"
     csv_parsing = "pipelet.etl.transform.csv.CsvParser"
     json_parsing = "pipelet.etl.transform.json.JsonParser"
     retrying = "pipelet.processors.retry_processor.RetryProcessor"
     splitting = "pipelet.processors.splitter_processor.SplitterProcessor"
+    logging = "pipelet.processors.logging_processor.LoggingProcessor"
 
 
 T = TypeVar("T", bound="BaseOpComposite")
@@ -75,7 +78,7 @@ class Op:
     Represents an operation configuration.
 
     Attributes:
-        operation (str): Path to the processor class in "module.ClassName" format.
+        operation (str): Path to the processor class in "module.Classmeta_option_decoratorName" format.
     """
 
     operation: str
@@ -115,7 +118,7 @@ class OpAll(BaseOpComposite[Union["OpAny", "OpAll"]]):
     """
 
 
-def prepare_retry_processor(
+def prepare_retry_option(
     processor: BaseProcessor[Any, Any, Any, Any], kwargs: Dict[str, Any]
 ) -> BaseProcessor[Any, Any, Any, Any]:
     """
@@ -149,6 +152,79 @@ def prepare_retry_processor(
     return processor
 
 
+def prepare_logging_option(
+    processor: BaseProcessor[Any, Any, Any, Any], kwargs: Dict[str, Any]
+) -> BaseProcessor[Any, Any, Any, Any]:
+    """
+    Prepares and applies logging functionality to the processor.
+
+    This function checks for a `logging` flag in the provided `kwargs`. If `logging` is enabled,
+    it wraps the `process` method of the processor with the `logging_decorator`.
+
+    Args:
+        processor (BaseProcessor): The processor instance to which logging functionality will be added.
+        kwargs (Dict[str, Any]): A dictionary of arguments, including the `logging` flag.
+
+    Returns:
+        BaseProcessor: The processor with logging functionality applied if enabled.
+    """
+    logging = kwargs.pop("logging", False)
+    logger = kwargs.pop("logger", None)
+    if logging:
+        processor.process = types.MethodType(
+            logging_decorator(processor.process, logger), processor
+        )
+
+    return processor
+
+
+OPTION_TO_HANDLER_MAP: Dict[
+    str,
+    Callable[
+        [BaseProcessor[Any, Any, Any, Any], Dict[str, Any]],
+        BaseProcessor[Any, Any, Any, Any],
+    ],
+] = {
+    "logging_args": prepare_logging_option,
+    "retry_args": prepare_retry_option,
+}
+
+
+def prepare_additional_options(
+    processor: BaseProcessor[Any, Any, Any, Any],
+    kwargs: Dict[str, Dict[str, Any]],
+) -> BaseProcessor[Any, Any, Any, Any]:
+    """
+    Prepares and applies additional options to the processor.
+
+    This function processes a dictionary of option names and their associated arguments,
+    applying each option to the processor using a corresponding handler function from
+    the `OPTION_TO_HANDLER_MAP`.
+
+    Supported options:
+        - `logging_args`: Configures logging functionality.
+        - `retry_args`: Configures retry logic.
+
+    Args:
+        processor (BaseProcessor): The processor instance to which additional options will be applied.
+        kwargs (Dict[str, Dict[str, Any]]): A dictionary where keys are option names
+                                            and values are their respective argument dictionaries.
+
+    Returns:
+        BaseProcessor: The processor with the additional options applied.
+
+    Raises:
+        KeyError: If an unsupported option name is provided in `kwargs`.
+    """
+    if kwargs:
+        for option_name, option_kwargs in kwargs.items():
+            option_handler = OPTION_TO_HANDLER_MAP.get(option_name, None)
+            if option_handler is None:
+                raise KeyError(f"Option {option_name} isn't supported")
+            processor = option_handler(processor, option_kwargs)
+    return processor
+
+
 def process_op(op: Op) -> BaseProcessor[Any, Any, Any, Any]:
     """
     Processes an Op and returns an instance of BaseProcessor.
@@ -164,15 +240,17 @@ def process_op(op: Op) -> BaseProcessor[Any, Any, Any, Any]:
     """
     type_processor = get_processor(op.operation)
     kwargs: Dict[str, Any] = {}
+    kwargs_to_additional_processors: Dict[str, Any] = {}
     for k, v in op.kwargs.items():
         factory = KEYWORD_FACTORY_MAP.get(k, None)
         if factory:
             kwargs[k] = factory(**v) if isinstance(v, dict) else factory(v)
         else:
-            kwargs[k] = v
-    retry_args: Dict[str, Any] = kwargs.pop("retry_args", {})
+            kwargs_to_additional_processors[k] = v
     processor = type_processor(**kwargs)
-    processor = prepare_retry_processor(processor, kwargs=retry_args)
+    processor = prepare_additional_options(
+        processor, kwargs_to_additional_processors
+    )
     return processor
 
 

{pipelet-0.0.2 → pipelet-0.0.3}/pipelet/processors/meta.py

@@ -21,6 +21,7 @@ Key Features:
 """
 
 from abc import ABCMeta
+from functools import wraps
 from typing import Any, Callable, Generator, List, Optional, Type, Union
 
 from pipelet.exceptions.base import (
@@ -68,6 +69,7 @@ class ProcessorMeta(ABCMeta):
             Callable: A wrapped generator function with added exception handling.
         """
 
+        @wraps(func)
         def wrapper(
             obj: AbstractProcessor,
             *args,

{pipelet-0.0.2 → pipelet-0.0.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "pipelet"
-version = "0.0.2"
+version = "0.0.3"
 description = ""
 authors = ["Daniel Naumowich <cskteam4@gmail.com>"]
 readme = "README.md"

pipelet-0.0.2/pipelet/etl/transform/unzip/__init__.py

@@ -1,3 +0,0 @@
-from pipelet.etl.transform.unzip.base import BaseUnzipProcessor
-
-__all__ = ("BaseUnzipProcessor",)