pipelet 0.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pipelet/__init__.py +0 -0
- pipelet/etl/__init__.py +0 -0
- pipelet/etl/extract/__init__.py +0 -0
- pipelet/etl/extract/http/__init__.py +9 -0
- pipelet/etl/extract/http/http_extractor.py +254 -0
- pipelet/etl/transform/__init__.py +0 -0
- pipelet/etl/transform/csv/__init__.py +3 -0
- pipelet/etl/transform/csv/reader.py +305 -0
- pipelet/etl/transform/json/__init__.py +3 -0
- pipelet/etl/transform/json/reader.py +118 -0
- pipelet/etl/transform/unzip/__init__.py +3 -0
- pipelet/etl/transform/unzip/base.py +132 -0
- pipelet/exceptions/__init__.py +31 -0
- pipelet/exceptions/base.py +55 -0
- pipelet/exceptions/csv_processor.py +24 -0
- pipelet/exceptions/download_processor.py +21 -0
- pipelet/exceptions/file_system.py +78 -0
- pipelet/exceptions/unzip_processor.py +24 -0
- pipelet/factories/__init__.py +9 -0
- pipelet/factories/exceptions_factory.py +69 -0
- pipelet/factories/file_system_factory.py +36 -0
- pipelet/log/__init__.py +6 -0
- pipelet/log/logger.py +19 -0
- pipelet/operations/__init__.py +17 -0
- pipelet/operations/operations.py +300 -0
- pipelet/operations/pipeline.py +49 -0
- pipelet/processors/__init__.py +57 -0
- pipelet/processors/abc.py +71 -0
- pipelet/processors/base.py +350 -0
- pipelet/processors/chain_processors/__init__.py +9 -0
- pipelet/processors/chain_processors/base.py +175 -0
- pipelet/processors/file_system/__init__.py +13 -0
- pipelet/processors/file_system/base.py +298 -0
- pipelet/processors/file_system/file_system_types.py +6 -0
- pipelet/processors/meta.py +115 -0
- pipelet/processors/mixins.py +0 -0
- pipelet/processors/processor_types.py +6 -0
- pipelet/processors/retry_processor.py +164 -0
- pipelet/processors/splitter_processor.py +54 -0
- pipelet/utils/__init__.py +11 -0
- pipelet/utils/common.py +97 -0
- pipelet-0.0.1.dist-info/LICENSE +9 -0
- pipelet-0.0.1.dist-info/METADATA +223 -0
- pipelet-0.0.1.dist-info/RECORD +45 -0
- pipelet-0.0.1.dist-info/WHEEL +4 -0
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
import json
|
|
2
|
+
from os import PathLike
|
|
3
|
+
from typing import (
|
|
4
|
+
Any,
|
|
5
|
+
Callable,
|
|
6
|
+
Dict,
|
|
7
|
+
Generator,
|
|
8
|
+
List,
|
|
9
|
+
Literal,
|
|
10
|
+
Optional,
|
|
11
|
+
Type,
|
|
12
|
+
Union,
|
|
13
|
+
)
|
|
14
|
+
|
|
15
|
+
from pipelet.log import logger_factory
|
|
16
|
+
from pipelet.processors.base import BaseProcessor
|
|
17
|
+
from pipelet.processors.file_system import AbstractFileSystemManager
|
|
18
|
+
|
|
19
|
+
logger = logger_factory()
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class JsonParser(
|
|
23
|
+
BaseProcessor[str | PathLike[str], Dict[str, Any], None, None],
|
|
24
|
+
):
|
|
25
|
+
"""
|
|
26
|
+
JsonParser
|
|
27
|
+
|
|
28
|
+
This processor reads a JSON file or string and parses its content into a Python dictionary.
|
|
29
|
+
|
|
30
|
+
Attributes:
|
|
31
|
+
file_system_manager (AbstractFileSystemManager): Manages file I/O operations.
|
|
32
|
+
white_exceptions (Optional[List[type[Exception]]]): List of exceptions that will be gracefully handled.
|
|
33
|
+
parse_kwargs (Dict[str, Any]): Keyword arguments passed to `json.loads` for customizing JSON parsing.
|
|
34
|
+
"""
|
|
35
|
+
|
|
36
|
+
def __init__(
|
|
37
|
+
self,
|
|
38
|
+
file_system_manager: AbstractFileSystemManager[
|
|
39
|
+
Union[str, PathLike[str]], Union[str, bytes]
|
|
40
|
+
],
|
|
41
|
+
white_exceptions: Optional[List[Type[Exception]]] = None,
|
|
42
|
+
parse_float: Optional[Callable[[str], Any]] = float,
|
|
43
|
+
parse_int: Optional[Callable[[str], Any]] = int,
|
|
44
|
+
parse_constant: Optional[
|
|
45
|
+
Literal["raise_error", "ignore", "custom_handle"]
|
|
46
|
+
] = None,
|
|
47
|
+
object_hook: Optional[Callable[[Dict[str, Any]], Any]] = None,
|
|
48
|
+
object_pairs_hook: Optional[
|
|
49
|
+
Callable[[List[tuple[str, Any]]], Any]
|
|
50
|
+
] = None,
|
|
51
|
+
) -> None:
|
|
52
|
+
"""
|
|
53
|
+
Initializes the JsonParser.
|
|
54
|
+
|
|
55
|
+
Args:
|
|
56
|
+
file_system_manager (AbstractFileSystemManager): Manages file operations such as reading and writing files.
|
|
57
|
+
white_exceptions (Optional[List[type[Exception]]]): List of exception types to be gracefully handled.
|
|
58
|
+
parse_float (type, optional): Custom type for parsing floating-point numbers. Defaults to `float`.
|
|
59
|
+
parse_int (type, optional): Custom type for parsing integers. Defaults to `int`.
|
|
60
|
+
parse_constant (Callable[[str], Any], optional): Custom handler for special constants like `NaN`, `Infinity`. Defaults to None.
|
|
61
|
+
object_hook (Callable[[Dict[str, Any]], Any], optional): Function to transform the JSON objects during decoding. Defaults to None.
|
|
62
|
+
object_pairs_hook (Callable[[List[Tuple[str, Any]]], Any], optional): Function to transform key-value pairs during decoding. Defaults to None.
|
|
63
|
+
"""
|
|
64
|
+
super().__init__(white_exceptions)
|
|
65
|
+
self._file_system_manager = file_system_manager
|
|
66
|
+
self._parse_float = parse_float
|
|
67
|
+
self._parse_int = parse_int
|
|
68
|
+
self._parse_constant = parse_constant
|
|
69
|
+
self._object_hook = object_hook
|
|
70
|
+
self._object_pairs_hook = object_pairs_hook
|
|
71
|
+
self._parse_kwargs = {
|
|
72
|
+
"parse_float": parse_float,
|
|
73
|
+
"parse_int": parse_int,
|
|
74
|
+
"parse_constant": parse_constant,
|
|
75
|
+
"object_hook": object_hook,
|
|
76
|
+
"object_pairs_hook": object_pairs_hook,
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
def process(
|
|
80
|
+
self, input_data: str | PathLike[str]
|
|
81
|
+
) -> Generator[Dict[str, Any], None, None]:
|
|
82
|
+
"""
|
|
83
|
+
Processes the input JSON file or string and parses it into a Python dictionary.
|
|
84
|
+
|
|
85
|
+
Args:
|
|
86
|
+
input_data (Union[str, PathLike[str]]): Path to the JSON file or a JSON string.
|
|
87
|
+
|
|
88
|
+
Yields:
|
|
89
|
+
Dict[str, Any]: The parsed JSON as a Python dictionary.
|
|
90
|
+
"""
|
|
91
|
+
# Read content using the file system manager
|
|
92
|
+
content = self._file_system_manager.read_file(input_data)
|
|
93
|
+
logger.debug("Read content from %s", input_data)
|
|
94
|
+
|
|
95
|
+
# Parse the JSON content with custom arguments
|
|
96
|
+
try:
|
|
97
|
+
data = json.loads(content, **self._parse_kwargs)
|
|
98
|
+
logger.debug(f"JSON {input_data} has parsed successfully")
|
|
99
|
+
except json.JSONDecodeError as e:
|
|
100
|
+
logger.error("Failed to parse JSON: %s", e)
|
|
101
|
+
raise
|
|
102
|
+
|
|
103
|
+
# Pass the data to the next processor if present, otherwise yield it
|
|
104
|
+
if self._next is not None:
|
|
105
|
+
gen = self._next.process(data)
|
|
106
|
+
yield from gen
|
|
107
|
+
else:
|
|
108
|
+
yield data
|
|
109
|
+
|
|
110
|
+
def __str__(self) -> str:
|
|
111
|
+
return (
|
|
112
|
+
f"{self.__class__.__name__}"
|
|
113
|
+
f"(parse_float={self._parse_float}, "
|
|
114
|
+
f"parse_int={self._parse_int}, "
|
|
115
|
+
f"parse_constant={self._parse_constant}, "
|
|
116
|
+
f"object_hook={self._object_hook}, "
|
|
117
|
+
f"object_pairs_hook={self._object_pairs_hook})"
|
|
118
|
+
)
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
import zipfile
|
|
2
|
+
from os import PathLike
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
from typing import Generator, List, Optional, Type, Union
|
|
5
|
+
|
|
6
|
+
from pipelet.exceptions.base import ProcessorStopIteration
|
|
7
|
+
from pipelet.exceptions.unzip_processor import UnzipError
|
|
8
|
+
from pipelet.log import logger_factory
|
|
9
|
+
from pipelet.processors.base import BaseProcessor
|
|
10
|
+
from pipelet.processors.file_system import (
|
|
11
|
+
AbstractFileSystemManager,
|
|
12
|
+
FileModeEnum,
|
|
13
|
+
)
|
|
14
|
+
|
|
15
|
+
logger = logger_factory()
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class BaseUnzipProcessor(
|
|
19
|
+
BaseProcessor[
|
|
20
|
+
Union[str, PathLike[str]], Union[str, PathLike[str]], None, None
|
|
21
|
+
],
|
|
22
|
+
):
|
|
23
|
+
"""
|
|
24
|
+
A processor that handles unzipping of `.zip` files.
|
|
25
|
+
|
|
26
|
+
This processor takes a `.zip` file as input, extracts its contents,
|
|
27
|
+
and yields the paths of the extracted files. Optionally, the processor
|
|
28
|
+
can delete the `.zip` file after extraction and can handle large files
|
|
29
|
+
in batches.
|
|
30
|
+
|
|
31
|
+
Attributes:
|
|
32
|
+
_file_system_manager (AbstractFileSystemManager): The file system manager for file operations.
|
|
33
|
+
_white_exceptions (List[Type[Exception]]): List of exceptions to bypass.
|
|
34
|
+
_auto_delete (bool): Flag to delete the zip file after processing.
|
|
35
|
+
_batch_size (Optional[int]): The batch size in bytes for extracting large files.
|
|
36
|
+
"""
|
|
37
|
+
|
|
38
|
+
def __init__(
|
|
39
|
+
self,
|
|
40
|
+
file_system_manager: AbstractFileSystemManager[
|
|
41
|
+
Union[str, PathLike[str]], Union[str, bytes]
|
|
42
|
+
],
|
|
43
|
+
white_exceptions: Optional[List[Type[Exception]]] = None,
|
|
44
|
+
auto_delete: bool = True,
|
|
45
|
+
batch_size: Optional[int] = None,
|
|
46
|
+
) -> None:
|
|
47
|
+
"""
|
|
48
|
+
Initializes the BaseUnzipProcessor.
|
|
49
|
+
|
|
50
|
+
Args:
|
|
51
|
+
file_system_manager (AbstractFileSystemManager): The file system manager for file operations.
|
|
52
|
+
white_exceptions (Optional[List[Type[Exception]]]): List of exceptions to bypass. Defaults to None.
|
|
53
|
+
auto_delete (bool): Flag to delete the zip file after processing. Defaults to True.
|
|
54
|
+
batch_size (Optional[int]): The batch size in megabytes for extracting large files. Defaults to None.
|
|
55
|
+
"""
|
|
56
|
+
self._file_system_manager = file_system_manager
|
|
57
|
+
self._white_exceptions = white_exceptions or []
|
|
58
|
+
self._auto_delete = auto_delete
|
|
59
|
+
self._batch_size = (
|
|
60
|
+
batch_size * 1024 * 1024 if batch_size is not None else None
|
|
61
|
+
)
|
|
62
|
+
|
|
63
|
+
def process(
|
|
64
|
+
self,
|
|
65
|
+
input_data: Union[str, PathLike[str]],
|
|
66
|
+
) -> Generator[Union[str, PathLike[str]], None, None]:
|
|
67
|
+
"""
|
|
68
|
+
Processes the given zip file and extracts its contents.
|
|
69
|
+
|
|
70
|
+
Args:
|
|
71
|
+
input_data (Union[str, PathLike[str]]): The path to the zip file to extract.
|
|
72
|
+
|
|
73
|
+
Yields:
|
|
74
|
+
Union[str, PathLike[str]]: The path to each extracted file.
|
|
75
|
+
|
|
76
|
+
Raises:
|
|
77
|
+
UnzipError: If the file is not a `.zip` file or cannot be processed.
|
|
78
|
+
"""
|
|
79
|
+
suffix = Path(input_data).suffix
|
|
80
|
+
if suffix != ".zip":
|
|
81
|
+
raise UnzipError(
|
|
82
|
+
input_data,
|
|
83
|
+
f"File has an extension '{suffix}', expected .zip",
|
|
84
|
+
)
|
|
85
|
+
|
|
86
|
+
file_path = self._file_system_manager.get_path(input_data)
|
|
87
|
+
logger.debug(f"Starting to unzip {input_data}")
|
|
88
|
+
|
|
89
|
+
unzip_files: List[Union[str, PathLike[str]]] = []
|
|
90
|
+
with zipfile.ZipFile(file_path, FileModeEnum.READ) as zip_ref:
|
|
91
|
+
for file_info in zip_ref.infolist():
|
|
92
|
+
extracted_file_path = self._file_system_manager.get_path(
|
|
93
|
+
file_info.filename
|
|
94
|
+
)
|
|
95
|
+
|
|
96
|
+
with zip_ref.open(
|
|
97
|
+
file_info, FileModeEnum.READ
|
|
98
|
+
) as source_file, open(
|
|
99
|
+
extracted_file_path,
|
|
100
|
+
FileModeEnum.WRITE_BINARY,
|
|
101
|
+
) as target_file:
|
|
102
|
+
if self._batch_size is not None:
|
|
103
|
+
while chunk := source_file.read(self._batch_size):
|
|
104
|
+
target_file.write(chunk)
|
|
105
|
+
else:
|
|
106
|
+
target_file.write(source_file.read())
|
|
107
|
+
|
|
108
|
+
unzip_files.append(extracted_file_path)
|
|
109
|
+
|
|
110
|
+
logger.debug("Unzipping was successful")
|
|
111
|
+
|
|
112
|
+
if self._next is not None:
|
|
113
|
+
for item in unzip_files:
|
|
114
|
+
gen = self._next.process(item)
|
|
115
|
+
try:
|
|
116
|
+
yield from gen
|
|
117
|
+
except ProcessorStopIteration:
|
|
118
|
+
# Continue processing other files in the list even if one fails
|
|
119
|
+
pass
|
|
120
|
+
else:
|
|
121
|
+
yield from iter(unzip_files)
|
|
122
|
+
|
|
123
|
+
# Optionally delete the zip file after extraction
|
|
124
|
+
if self._auto_delete:
|
|
125
|
+
self._file_system_manager.delete_file(input_data)
|
|
126
|
+
|
|
127
|
+
def __str__(self) -> str:
|
|
128
|
+
return (
|
|
129
|
+
f"{self.__class__.__name__}"
|
|
130
|
+
f"(auto_delete={self._auto_delete}, "
|
|
131
|
+
f"batch_size={self._batch_size})"
|
|
132
|
+
)
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
from enum import Enum
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class BaseExceptionsEnum(str, Enum):
|
|
5
|
+
csv_exception = "pipelet.exceptions.csv_processor.CsvProcessorException"
|
|
6
|
+
csv_parsing_error = "pipelet.exceptions.csv_processor.CsvParsingError"
|
|
7
|
+
unzip_exception = (
|
|
8
|
+
"pipelet.exceptions.unzip_processor.UnzipProcessorException"
|
|
9
|
+
)
|
|
10
|
+
unzip_error = "pipelet.exceptions.unzip_processor.UnzipError"
|
|
11
|
+
download_exception = (
|
|
12
|
+
"pipelet.exceptions.download_processor.DownloadProcessorException"
|
|
13
|
+
)
|
|
14
|
+
download_error = "pipelet.exceptions.download_processor.DownloadError"
|
|
15
|
+
|
|
16
|
+
file_system_exception = "pipelet.exceptions.base.BaseFileSystemException"
|
|
17
|
+
local_file_system_exception = (
|
|
18
|
+
"pipelet.exceptions.file_system.LocalFileSystemException"
|
|
19
|
+
)
|
|
20
|
+
local_file_system_create_file_error = (
|
|
21
|
+
"pipelet.exceptions.file_system.LocalFileCreationError"
|
|
22
|
+
)
|
|
23
|
+
local_file_system_read_file_error = (
|
|
24
|
+
"pipelet.exceptions.file_system.LocalFileReadError"
|
|
25
|
+
)
|
|
26
|
+
local_file_system_delete_file_error = (
|
|
27
|
+
"pipelet.exceptions.file_system.LocalFileDeleteError"
|
|
28
|
+
)
|
|
29
|
+
local_file_system_too_large_error = (
|
|
30
|
+
"pipelet.exceptions.file_system.LocalFileTooLargeError"
|
|
31
|
+
)
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
class BaseProcessorException(Exception):
|
|
2
|
+
"""Base exception for processor-related errors."""
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
class BaseFileSystemException(BaseProcessorException):
|
|
6
|
+
"""Base exception for file system-related errors."""
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class BaseCsvProcessorException(BaseProcessorException):
|
|
10
|
+
"""Base exception for CSV processor-related errors."""
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class BaseUnzipException(BaseProcessorException):
|
|
14
|
+
"""Base exception for unzip-related errors."""
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class BaseDownloadProcessorException(BaseProcessorException):
|
|
18
|
+
"""Base exception for download processor-related errors."""
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class ProcessorStopIteration(BaseProcessorException):
|
|
22
|
+
"""
|
|
23
|
+
Custom exception used to stop generators in the data processing system.
|
|
24
|
+
|
|
25
|
+
This exception is used as a replacement for the standard `StopIteration` exception.
|
|
26
|
+
It allows external logic to handle the termination of generators explicitly by catching
|
|
27
|
+
this exception. This provides more control over generator flow management compared
|
|
28
|
+
to the default behavior.
|
|
29
|
+
|
|
30
|
+
When this exception is raised in a generator, it indicates a normal termination
|
|
31
|
+
of the iteration. It should be used in scenarios where external code needs to
|
|
32
|
+
manage or log the termination of the generator.
|
|
33
|
+
|
|
34
|
+
Attributes:
|
|
35
|
+
message (str): A message describing the reason for the termination.
|
|
36
|
+
|
|
37
|
+
Example:
|
|
38
|
+
def my_generator():
|
|
39
|
+
yield 1
|
|
40
|
+
yield 2
|
|
41
|
+
raise ProcessorStopIteration("Reached end of processing")
|
|
42
|
+
|
|
43
|
+
Args:
|
|
44
|
+
message (str): A message describing the reason for the termination.
|
|
45
|
+
"""
|
|
46
|
+
|
|
47
|
+
def __init__(self, message: str):
|
|
48
|
+
"""
|
|
49
|
+
Initialize ProcessorStopIteration.
|
|
50
|
+
|
|
51
|
+
Args:
|
|
52
|
+
message (str): A message describing the reason for the termination.
|
|
53
|
+
"""
|
|
54
|
+
self.message = message
|
|
55
|
+
super().__init__(f"Processing stopped: {message}")
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
from os import PathLike
|
|
2
|
+
from typing import Union
|
|
3
|
+
|
|
4
|
+
from pipelet.exceptions.base import BaseCsvProcessorException
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class CsvProcessorException(BaseCsvProcessorException):
|
|
8
|
+
"""Base exception for CSV processing-related errors."""
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class CsvParsingError(CsvProcessorException):
|
|
12
|
+
"""Exception raised for errors occurring during CSV parsing."""
|
|
13
|
+
|
|
14
|
+
def __init__(self, filename: Union[PathLike[str], str], message: str):
|
|
15
|
+
"""
|
|
16
|
+
Initialize CsvParsingError.
|
|
17
|
+
|
|
18
|
+
Args:
|
|
19
|
+
filename (str): The name of the CSV file that caused the error.
|
|
20
|
+
message (str): A message describing the error.
|
|
21
|
+
"""
|
|
22
|
+
self.filename = filename
|
|
23
|
+
self.message = message
|
|
24
|
+
super().__init__(f"Failed to parse CSV file '{filename}': {message}")
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
from pipelet.exceptions.base import BaseDownloadProcessorException
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class DownloadProcessorException(BaseDownloadProcessorException):
|
|
5
|
+
"""Base exception for download processor-related errors."""
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class DownloadError(DownloadProcessorException):
|
|
9
|
+
"""Exception raised for errors occurring during file download."""
|
|
10
|
+
|
|
11
|
+
def __init__(self, url: str, message: str):
|
|
12
|
+
"""
|
|
13
|
+
Initialize DownloadError.
|
|
14
|
+
|
|
15
|
+
Args:
|
|
16
|
+
url (str): The URL of the file that caused the error.
|
|
17
|
+
message (str): A message describing the error.
|
|
18
|
+
"""
|
|
19
|
+
self.url = url
|
|
20
|
+
self.message = message
|
|
21
|
+
super().__init__(f"Failed to download file from '{url}': {message}")
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
from os import PathLike
|
|
2
|
+
from typing import Union
|
|
3
|
+
|
|
4
|
+
from pipelet.exceptions.base import BaseFileSystemException
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class LocalFileSystemException(BaseFileSystemException):
|
|
8
|
+
"""Base exception for local file system-related errors."""
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class LocalFileCreationError(LocalFileSystemException):
|
|
12
|
+
"""Exception raised for errors occurring during file creation."""
|
|
13
|
+
|
|
14
|
+
def __init__(self, filename: Union[PathLike[str], str], message: str):
|
|
15
|
+
"""
|
|
16
|
+
Initialize LocalFileCreationError.
|
|
17
|
+
|
|
18
|
+
Args:
|
|
19
|
+
filename (str): The name of the file that caused the error.
|
|
20
|
+
message (str): A message describing the error.
|
|
21
|
+
"""
|
|
22
|
+
self.filename = filename
|
|
23
|
+
self.message = message
|
|
24
|
+
super().__init__(f"Failed to create file '{filename}': {message}")
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class LocalFileReadError(LocalFileSystemException):
|
|
28
|
+
"""Exception raised for errors occurring during file reading."""
|
|
29
|
+
|
|
30
|
+
def __init__(self, filename: Union[PathLike[str], str], message: str):
|
|
31
|
+
"""
|
|
32
|
+
Initialize LocalFileReadError.
|
|
33
|
+
|
|
34
|
+
Args:
|
|
35
|
+
filename (str): The name of the file that caused the error.
|
|
36
|
+
message (str): A message describing the error.
|
|
37
|
+
"""
|
|
38
|
+
self.filename = filename
|
|
39
|
+
self.message = message
|
|
40
|
+
super().__init__(f"Failed to read file '{filename}': {message}")
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
class LocalFileDeleteError(LocalFileSystemException):
|
|
44
|
+
"""Exception raised for errors occurring during file deletion."""
|
|
45
|
+
|
|
46
|
+
def __init__(self, filename: Union[PathLike[str], str], message: str):
|
|
47
|
+
"""
|
|
48
|
+
Initialize LocalFileDeleteError.
|
|
49
|
+
|
|
50
|
+
Args:
|
|
51
|
+
filename (str): The name of the file that caused the error.
|
|
52
|
+
message (str): A message describing the error.
|
|
53
|
+
"""
|
|
54
|
+
self.filename = filename
|
|
55
|
+
self.message = message
|
|
56
|
+
super().__init__(f"Failed to delete file '{filename}': {message}")
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
class LocalFileTooLargeError(LocalFileSystemException):
|
|
60
|
+
"""Exception raised when a file is too large to be read."""
|
|
61
|
+
|
|
62
|
+
def __init__(
|
|
63
|
+
self, filename: Union[PathLike[str], str], size: int, max_size: int
|
|
64
|
+
):
|
|
65
|
+
"""
|
|
66
|
+
Initialize LocalFileTooLargeError.
|
|
67
|
+
|
|
68
|
+
Args:
|
|
69
|
+
filename (str): The name of the file that caused the error.
|
|
70
|
+
size (int): The size of the file.
|
|
71
|
+
max_size (int): The maximum allowed size of the file.
|
|
72
|
+
"""
|
|
73
|
+
self.filename = filename
|
|
74
|
+
self.size = size
|
|
75
|
+
self.max_size = max_size
|
|
76
|
+
super().__init__(
|
|
77
|
+
f"File '{filename}' is too large to read: {size} bytes (max allowed: {max_size} bytes)"
|
|
78
|
+
)
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
from os import PathLike
|
|
2
|
+
from typing import Union
|
|
3
|
+
|
|
4
|
+
from pipelet.exceptions.base import BaseUnzipException
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class UnzipProcessorException(BaseUnzipException):
|
|
8
|
+
"""Base exception for unzip processor-related errors."""
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class UnzipError(UnzipProcessorException):
|
|
12
|
+
"""Exception raised for errors occurring during the unzip process."""
|
|
13
|
+
|
|
14
|
+
def __init__(self, filename: Union[PathLike[str], str], message: str):
|
|
15
|
+
"""
|
|
16
|
+
Initialize UnzipError.
|
|
17
|
+
|
|
18
|
+
Args:
|
|
19
|
+
filename (str): The name of the file that caused the error.
|
|
20
|
+
message (str): A message describing the error.
|
|
21
|
+
"""
|
|
22
|
+
self.filename = filename
|
|
23
|
+
self.message = message
|
|
24
|
+
super().__init__(f"Failed to unzip file '{filename}': {message}")
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
from typing import Any, Callable, Dict
|
|
2
|
+
|
|
3
|
+
from pipelet.factories.exceptions_factory import custom_exceptions_factory
|
|
4
|
+
from pipelet.factories.file_system_factory import file_system_factory
|
|
5
|
+
|
|
6
|
+
KEYWORD_FACTORY_MAP: Dict[str, Callable[..., Any]] = {
|
|
7
|
+
"file_system_manager": file_system_factory,
|
|
8
|
+
"white_exceptions": custom_exceptions_factory,
|
|
9
|
+
}
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
import importlib
|
|
2
|
+
from os import PathLike
|
|
3
|
+
from typing import List, Type, Union
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def get_custom_exception(
|
|
7
|
+
path_to_exception: Union[PathLike, str, Type[Exception]],
|
|
8
|
+
) -> Type[Exception]:
|
|
9
|
+
"""
|
|
10
|
+
Retrieves a custom exception class by its string path or directly returns the exception class.
|
|
11
|
+
|
|
12
|
+
Args:
|
|
13
|
+
path_to_exception (Union[PathLike, str, Type[Exception]]):
|
|
14
|
+
Path to the exception class in the format "module.submodule.ClassName"
|
|
15
|
+
or the class itself.
|
|
16
|
+
|
|
17
|
+
Returns:
|
|
18
|
+
Type[Exception]: The exception class.
|
|
19
|
+
|
|
20
|
+
Raises:
|
|
21
|
+
ValueError: If the path is invalid or the class is not a subclass of Exception.
|
|
22
|
+
"""
|
|
23
|
+
# If we are passed a class directly, return it if it's valid.
|
|
24
|
+
if isinstance(path_to_exception, type) and issubclass(
|
|
25
|
+
path_to_exception, Exception
|
|
26
|
+
):
|
|
27
|
+
return path_to_exception
|
|
28
|
+
|
|
29
|
+
# Otherwise, handle the case where a string path is provided.
|
|
30
|
+
if isinstance(path_to_exception, str):
|
|
31
|
+
try:
|
|
32
|
+
exception_path, exception_name = path_to_exception.rsplit(".", 1)
|
|
33
|
+
exception_module = importlib.import_module(exception_path)
|
|
34
|
+
exception = getattr(exception_module, exception_name)
|
|
35
|
+
if not isinstance(exception, type) or not issubclass(
|
|
36
|
+
exception, Exception
|
|
37
|
+
):
|
|
38
|
+
raise ValueError(
|
|
39
|
+
f"{exception_name} is not a subclass of Exception."
|
|
40
|
+
)
|
|
41
|
+
except (ImportError, AttributeError, ValueError) as e:
|
|
42
|
+
raise ValueError(f"Invalid path: {path_to_exception}. Error: {e}")
|
|
43
|
+
else:
|
|
44
|
+
return exception
|
|
45
|
+
else:
|
|
46
|
+
raise ValueError(
|
|
47
|
+
f"Invalid type for path_to_exception: {type(path_to_exception)}"
|
|
48
|
+
)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def custom_exceptions_factory(
|
|
52
|
+
paths: List[Union[PathLike, str, Type[Exception]]],
|
|
53
|
+
) -> List[Type[Exception]]:
|
|
54
|
+
"""
|
|
55
|
+
Creates a list of custom exception classes from a list of string paths or directly from exception classes.
|
|
56
|
+
|
|
57
|
+
Args:
|
|
58
|
+
paths (List[Union[PathLike, str, Type[Exception]]]):
|
|
59
|
+
List of paths to exception classes in the format "module.submodule.ClassName"
|
|
60
|
+
or exception classes themselves.
|
|
61
|
+
|
|
62
|
+
Returns:
|
|
63
|
+
List[Type[Exception]]: List of exception classes.
|
|
64
|
+
"""
|
|
65
|
+
exceptions = []
|
|
66
|
+
for path in paths:
|
|
67
|
+
exception = get_custom_exception(path)
|
|
68
|
+
exceptions.append(exception)
|
|
69
|
+
return exceptions
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import importlib
|
|
2
|
+
from typing import Any, Dict
|
|
3
|
+
|
|
4
|
+
from pipelet.processors.file_system import AbstractFileSystemManager
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def file_system_factory(
|
|
8
|
+
path: str, kwargs: Dict[str, Any] | None = None
|
|
9
|
+
) -> AbstractFileSystemManager[Any, Any]:
|
|
10
|
+
"""
|
|
11
|
+
Factory function to create an instance of a file system manager.
|
|
12
|
+
|
|
13
|
+
Args:
|
|
14
|
+
path (str): Path to the file system manager class in the format "module.submodule.ClassName".
|
|
15
|
+
kwargs (Dict[str, Any]): Keyword arguments to pass to the file system manager constructor.
|
|
16
|
+
|
|
17
|
+
Returns:
|
|
18
|
+
AbstractFileSystemManager[Any, Any]: An instance of the file system manager class.
|
|
19
|
+
|
|
20
|
+
Raises:
|
|
21
|
+
ValueError: If the path is invalid or the class is not a subclass of AbstractFileSystemManager[Any, Any].
|
|
22
|
+
"""
|
|
23
|
+
try:
|
|
24
|
+
file_system_path, file_system_name = path.rsplit(".", 1)
|
|
25
|
+
file_system_module = importlib.import_module(file_system_path)
|
|
26
|
+
file_system_manager = getattr(file_system_module, file_system_name)
|
|
27
|
+
if not issubclass(file_system_manager, AbstractFileSystemManager):
|
|
28
|
+
raise ValueError(
|
|
29
|
+
f"{file_system_name} is not a subclass of AbstractFileSystemManager."
|
|
30
|
+
)
|
|
31
|
+
except (ImportError, AttributeError, ValueError) as e:
|
|
32
|
+
raise ValueError(f"Invalid path: {path}. Error: {e}")
|
|
33
|
+
else:
|
|
34
|
+
return (
|
|
35
|
+
file_system_manager(**kwargs) if kwargs else file_system_manager()
|
|
36
|
+
)
|
pipelet/log/__init__.py
ADDED
pipelet/log/logger.py
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import sys
|
|
2
|
+
from typing import TYPE_CHECKING
|
|
3
|
+
|
|
4
|
+
from loguru import logger
|
|
5
|
+
|
|
6
|
+
if TYPE_CHECKING:
|
|
7
|
+
from loguru import Logger
|
|
8
|
+
else:
|
|
9
|
+
Logger = None
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def logger_factory() -> Logger:
|
|
13
|
+
logger.remove()
|
|
14
|
+
logger.add(
|
|
15
|
+
sys.stdout,
|
|
16
|
+
level="DEBUG",
|
|
17
|
+
)
|
|
18
|
+
|
|
19
|
+
return logger
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
from pipelet.operations.operations import (
|
|
2
|
+
BaseOpEnum,
|
|
3
|
+
Op,
|
|
4
|
+
OpAll,
|
|
5
|
+
OpAny,
|
|
6
|
+
get_processor_for_operation,
|
|
7
|
+
)
|
|
8
|
+
from pipelet.operations.pipeline import PipelineConverter
|
|
9
|
+
|
|
10
|
+
__all__ = (
|
|
11
|
+
"BaseOpEnum",
|
|
12
|
+
"Op",
|
|
13
|
+
"OpAny",
|
|
14
|
+
"OpAll",
|
|
15
|
+
"get_processor_for_operation",
|
|
16
|
+
"PipelineConverter",
|
|
17
|
+
)
|