pipelet 0.0.3__tar.gz → 0.0.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pipelet-0.0.3 → pipelet-0.0.4}/PKG-INFO +6 -2
- {pipelet-0.0.3 → pipelet-0.0.4}/README.md +5 -1
- pipelet-0.0.4/pipelet/etl/transform/unzip/__init__.py +7 -0
- pipelet-0.0.4/pipelet/etl/transform/unzip/tar_processor.py +135 -0
- {pipelet-0.0.3 → pipelet-0.0.4}/pipelet/operations/operations.py +5 -1
- {pipelet-0.0.3 → pipelet-0.0.4}/pyproject.toml +1 -1
- pipelet-0.0.3/pipelet/etl/transform/unzip/__init__.py +0 -3
- {pipelet-0.0.3 → pipelet-0.0.4}/LICENSE +0 -0
- {pipelet-0.0.3 → pipelet-0.0.4}/pipelet/__init__.py +0 -0
- {pipelet-0.0.3 → pipelet-0.0.4}/pipelet/decarators/__init__.py +0 -0
- {pipelet-0.0.3 → pipelet-0.0.4}/pipelet/decarators/logging_decorator.py +0 -0
- {pipelet-0.0.3 → pipelet-0.0.4}/pipelet/decarators/option_decorator.py +0 -0
- {pipelet-0.0.3 → pipelet-0.0.4}/pipelet/etl/__init__.py +0 -0
- {pipelet-0.0.3 → pipelet-0.0.4}/pipelet/etl/extract/__init__.py +0 -0
- {pipelet-0.0.3 → pipelet-0.0.4}/pipelet/etl/extract/http/__init__.py +0 -0
- {pipelet-0.0.3 → pipelet-0.0.4}/pipelet/etl/extract/http/http_extractor.py +0 -0
- {pipelet-0.0.3 → pipelet-0.0.4}/pipelet/etl/transform/__init__.py +0 -0
- {pipelet-0.0.3 → pipelet-0.0.4}/pipelet/etl/transform/csv/__init__.py +0 -0
- {pipelet-0.0.3 → pipelet-0.0.4}/pipelet/etl/transform/csv/reader.py +0 -0
- {pipelet-0.0.3 → pipelet-0.0.4}/pipelet/etl/transform/json/__init__.py +0 -0
- {pipelet-0.0.3 → pipelet-0.0.4}/pipelet/etl/transform/json/reader.py +0 -0
- {pipelet-0.0.3 → pipelet-0.0.4}/pipelet/etl/transform/unzip/base.py +0 -0
- {pipelet-0.0.3 → pipelet-0.0.4}/pipelet/exceptions/__init__.py +0 -0
- {pipelet-0.0.3 → pipelet-0.0.4}/pipelet/exceptions/base.py +0 -0
- {pipelet-0.0.3 → pipelet-0.0.4}/pipelet/exceptions/csv_processor.py +0 -0
- {pipelet-0.0.3 → pipelet-0.0.4}/pipelet/exceptions/download_processor.py +0 -0
- {pipelet-0.0.3 → pipelet-0.0.4}/pipelet/exceptions/file_system.py +0 -0
- {pipelet-0.0.3 → pipelet-0.0.4}/pipelet/exceptions/unzip_processor.py +0 -0
- {pipelet-0.0.3 → pipelet-0.0.4}/pipelet/factories/__init__.py +0 -0
- {pipelet-0.0.3 → pipelet-0.0.4}/pipelet/factories/exceptions_factory.py +0 -0
- {pipelet-0.0.3 → pipelet-0.0.4}/pipelet/factories/file_system_factory.py +0 -0
- {pipelet-0.0.3 → pipelet-0.0.4}/pipelet/log/__init__.py +0 -0
- {pipelet-0.0.3 → pipelet-0.0.4}/pipelet/log/logger.py +0 -0
- {pipelet-0.0.3 → pipelet-0.0.4}/pipelet/operations/__init__.py +0 -0
- {pipelet-0.0.3 → pipelet-0.0.4}/pipelet/operations/pipeline.py +0 -0
- {pipelet-0.0.3 → pipelet-0.0.4}/pipelet/processors/__init__.py +0 -0
- {pipelet-0.0.3 → pipelet-0.0.4}/pipelet/processors/abc.py +0 -0
- {pipelet-0.0.3 → pipelet-0.0.4}/pipelet/processors/base.py +0 -0
- {pipelet-0.0.3 → pipelet-0.0.4}/pipelet/processors/chain_processors/__init__.py +0 -0
- {pipelet-0.0.3 → pipelet-0.0.4}/pipelet/processors/chain_processors/base.py +0 -0
- {pipelet-0.0.3 → pipelet-0.0.4}/pipelet/processors/file_system/__init__.py +0 -0
- {pipelet-0.0.3 → pipelet-0.0.4}/pipelet/processors/file_system/base.py +0 -0
- {pipelet-0.0.3 → pipelet-0.0.4}/pipelet/processors/file_system/file_system_types.py +0 -0
- {pipelet-0.0.3 → pipelet-0.0.4}/pipelet/processors/meta.py +0 -0
- {pipelet-0.0.3 → pipelet-0.0.4}/pipelet/processors/mixins.py +0 -0
- {pipelet-0.0.3 → pipelet-0.0.4}/pipelet/processors/processor_types.py +0 -0
- {pipelet-0.0.3 → pipelet-0.0.4}/pipelet/processors/retry_processor.py +0 -0
- {pipelet-0.0.3 → pipelet-0.0.4}/pipelet/processors/splitter_processor.py +0 -0
- {pipelet-0.0.3 → pipelet-0.0.4}/pipelet/utils/__init__.py +0 -0
- {pipelet-0.0.3 → pipelet-0.0.4}/pipelet/utils/common.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: pipelet
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.4
|
|
4
4
|
Summary:
|
|
5
5
|
Author: Daniel Naumowich
|
|
6
6
|
Author-email: cskteam4@gmail.com
|
|
@@ -72,7 +72,11 @@ A processor for parsing JSON data from strings or files into Python dictionaries
|
|
|
72
72
|
### 8. **UnzipProcessor**
|
|
73
73
|
A processor for extracting files from `.zip` archives. It supports extracting data into files, managing chunk sizes, and deleting the original archive after processing.
|
|
74
74
|
|
|
75
|
-
|
|
75
|
+
|
|
76
|
+
### 9. **TarExtractProcessor**
|
|
77
|
+
A processor for extracting files from `".tar", ".gz", ".bz2", ".xz", ".tgz", ".tbz2"` archives. It supports extracting data into files, managing chunk sizes, and deleting the original archive after processing.
|
|
78
|
+
|
|
79
|
+
### 10. **RetryProcessor**
|
|
76
80
|
|
|
77
81
|
A processor designed to handle transient errors by retrying operations based on a customizable retry policy.
|
|
78
82
|
|
|
@@ -52,7 +52,11 @@ A processor for parsing JSON data from strings or files into Python dictionaries
|
|
|
52
52
|
### 8. **UnzipProcessor**
|
|
53
53
|
A processor for extracting files from `.zip` archives. It supports extracting data into files, managing chunk sizes, and deleting the original archive after processing.
|
|
54
54
|
|
|
55
|
-
|
|
55
|
+
|
|
56
|
+
### 9. **TarExtractProcessor**
|
|
57
|
+
A processor for extracting files from `".tar", ".gz", ".bz2", ".xz", ".tgz", ".tbz2"` archives. It supports extracting data into files, managing chunk sizes, and deleting the original archive after processing.
|
|
58
|
+
|
|
59
|
+
### 10. **RetryProcessor**
|
|
56
60
|
|
|
57
61
|
A processor designed to handle transient errors by retrying operations based on a customizable retry policy.
|
|
58
62
|
|
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
import tarfile
|
|
2
|
+
from os import PathLike
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
from typing import Generator, List, Optional, Type, Union
|
|
5
|
+
|
|
6
|
+
from pipelet.exceptions.base import ProcessorStopIteration
|
|
7
|
+
from pipelet.exceptions.unzip_processor import UnzipError
|
|
8
|
+
from pipelet.log import logger_factory
|
|
9
|
+
from pipelet.processors.base import BaseProcessor
|
|
10
|
+
from pipelet.processors.file_system import (
|
|
11
|
+
AbstractFileSystemManager,
|
|
12
|
+
FileModeEnum,
|
|
13
|
+
)
|
|
14
|
+
|
|
15
|
+
logger = logger_factory()
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class TarExtractProcessor(
|
|
19
|
+
BaseProcessor[
|
|
20
|
+
Union[str, PathLike[str]], Union[str, PathLike[str]], None, None
|
|
21
|
+
],
|
|
22
|
+
):
|
|
23
|
+
"""
|
|
24
|
+
A processor that handles extracting `.tar` archives (including compressed formats).
|
|
25
|
+
|
|
26
|
+
This processor takes a `.tar` file (or compressed `.tar` variants like `.tar.gz`, `.tgz`),
|
|
27
|
+
extracts its contents, and yields the paths of the extracted files and directories.
|
|
28
|
+
|
|
29
|
+
Attributes:
|
|
30
|
+
_file_system_manager (AbstractFileSystemManager): File system manager for file operations.
|
|
31
|
+
_white_exceptions (List[Type[Exception]]): List of exceptions to bypass.
|
|
32
|
+
_auto_delete (bool): Flag to delete the tar file after processing.
|
|
33
|
+
_batch_size (Optional[int]): The batch size in bytes for extracting large files.
|
|
34
|
+
"""
|
|
35
|
+
|
|
36
|
+
def __init__(
|
|
37
|
+
self,
|
|
38
|
+
file_system_manager: AbstractFileSystemManager[
|
|
39
|
+
Union[str, PathLike[str]], Union[str, bytes]
|
|
40
|
+
],
|
|
41
|
+
white_exceptions: Optional[List[Type[Exception]]] = None,
|
|
42
|
+
auto_delete: bool = True,
|
|
43
|
+
batch_size: Optional[int] = None,
|
|
44
|
+
) -> None:
|
|
45
|
+
self._file_system_manager = file_system_manager
|
|
46
|
+
self._white_exceptions = white_exceptions or []
|
|
47
|
+
self._auto_delete = auto_delete
|
|
48
|
+
self._batch_size = (
|
|
49
|
+
batch_size * 1024 * 1024 if batch_size is not None else None
|
|
50
|
+
)
|
|
51
|
+
|
|
52
|
+
def process(
|
|
53
|
+
self,
|
|
54
|
+
input_data: Union[str, PathLike[str]],
|
|
55
|
+
) -> Generator[Union[str, PathLike[str]], None, None]:
|
|
56
|
+
"""
|
|
57
|
+
Processes the given tar file and extracts its contents.
|
|
58
|
+
|
|
59
|
+
Args:
|
|
60
|
+
input_data (Union[str, PathLike[str]]): The path to the tar file to extract.
|
|
61
|
+
|
|
62
|
+
Yields:
|
|
63
|
+
Union[str, PathLike[str]]: The path to each extracted file or directory.
|
|
64
|
+
|
|
65
|
+
Raises:
|
|
66
|
+
UnzipError: If the file is not a `.tar` file or cannot be processed.
|
|
67
|
+
"""
|
|
68
|
+
suffix = Path(input_data).suffix
|
|
69
|
+
if suffix not in [".tar", ".gz", ".bz2", ".xz", ".tgz", ".tbz2"]:
|
|
70
|
+
raise UnzipError(
|
|
71
|
+
input_data,
|
|
72
|
+
f"File has an extension '{suffix}', expected a supported .tar format",
|
|
73
|
+
)
|
|
74
|
+
|
|
75
|
+
file_path = self._file_system_manager.get_path(input_data)
|
|
76
|
+
extracted_paths: List[Union[str, PathLike[str]]] = []
|
|
77
|
+
|
|
78
|
+
try:
|
|
79
|
+
with tarfile.open(file_path, mode="r:*") as tar:
|
|
80
|
+
for member in tar.getmembers():
|
|
81
|
+
extracted_path = self._file_system_manager.get_path(
|
|
82
|
+
member.name
|
|
83
|
+
)
|
|
84
|
+
if member.isdir():
|
|
85
|
+
# Create the directory if it does not exist
|
|
86
|
+
Path(extracted_path).mkdir(parents=True, exist_ok=True)
|
|
87
|
+
logger.info(f"Created directory: {member.name}")
|
|
88
|
+
extracted_paths.append(extracted_path)
|
|
89
|
+
elif member.isfile():
|
|
90
|
+
# Extract file and handle large files with batching
|
|
91
|
+
source_file = tar.extractfile(member)
|
|
92
|
+
if source_file is not None:
|
|
93
|
+
with source_file, open(
|
|
94
|
+
extracted_path, FileModeEnum.WRITE_BINARY
|
|
95
|
+
) as target_file:
|
|
96
|
+
if self._batch_size is not None:
|
|
97
|
+
while chunk := source_file.read(
|
|
98
|
+
self._batch_size
|
|
99
|
+
):
|
|
100
|
+
target_file.write(chunk)
|
|
101
|
+
else:
|
|
102
|
+
target_file.write(source_file.read())
|
|
103
|
+
extracted_paths.append(extracted_path)
|
|
104
|
+
else:
|
|
105
|
+
logger.warning(
|
|
106
|
+
f"Could not extract file: {member.name}"
|
|
107
|
+
)
|
|
108
|
+
else:
|
|
109
|
+
logger.info(
|
|
110
|
+
f"Skipping unsupported member type: {member.name}"
|
|
111
|
+
)
|
|
112
|
+
|
|
113
|
+
except Exception as e:
|
|
114
|
+
raise UnzipError(file_path, f"Failed to extract tar file: {e}")
|
|
115
|
+
|
|
116
|
+
# Pass extracted paths to the next processor, if defined
|
|
117
|
+
if self._next is not None:
|
|
118
|
+
for item in extracted_paths:
|
|
119
|
+
try:
|
|
120
|
+
yield from self._next.process(item)
|
|
121
|
+
except ProcessorStopIteration:
|
|
122
|
+
pass
|
|
123
|
+
else:
|
|
124
|
+
yield from iter(extracted_paths)
|
|
125
|
+
|
|
126
|
+
# Optionally delete the tar file after processing
|
|
127
|
+
if self._auto_delete:
|
|
128
|
+
self._file_system_manager.delete_file(input_data)
|
|
129
|
+
|
|
130
|
+
def __str__(self) -> str:
|
|
131
|
+
return (
|
|
132
|
+
f"{self.__class__.__name__}"
|
|
133
|
+
f"(auto_delete={self._auto_delete}, "
|
|
134
|
+
f"batch_size={self._batch_size})"
|
|
135
|
+
)
|
|
@@ -62,11 +62,13 @@ class BaseOpEnum(str, Enum):
|
|
|
62
62
|
downloading = "pipelet.etl.extract.http.HttpDataExtractProcessor"
|
|
63
63
|
stream_downloading = "pipelet.etl.extract.http.HttpxStreamDownloadProcessor"
|
|
64
64
|
unzip = "pipelet.etl.transform.unzip.UnzipProcessor"
|
|
65
|
+
tar_extract = (
|
|
66
|
+
"pipelet.etl.transform.unzip.tar_processor.TarExtractProcessor"
|
|
67
|
+
)
|
|
65
68
|
csv_parsing = "pipelet.etl.transform.csv.CsvParser"
|
|
66
69
|
json_parsing = "pipelet.etl.transform.json.JsonParser"
|
|
67
70
|
retrying = "pipelet.processors.retry_processor.RetryProcessor"
|
|
68
71
|
splitting = "pipelet.processors.splitter_processor.SplitterProcessor"
|
|
69
|
-
logging = "pipelet.processors.logging_processor.LoggingProcessor"
|
|
70
72
|
|
|
71
73
|
|
|
72
74
|
T = TypeVar("T", bound="BaseOpComposite")
|
|
@@ -245,6 +247,8 @@ def process_op(op: Op) -> BaseProcessor[Any, Any, Any, Any]:
|
|
|
245
247
|
factory = KEYWORD_FACTORY_MAP.get(k, None)
|
|
246
248
|
if factory:
|
|
247
249
|
kwargs[k] = factory(**v) if isinstance(v, dict) else factory(v)
|
|
250
|
+
elif OPTION_TO_HANDLER_MAP.get(k, None) is None:
|
|
251
|
+
kwargs[k] = v
|
|
248
252
|
else:
|
|
249
253
|
kwargs_to_additional_processors[k] = v
|
|
250
254
|
processor = type_processor(**kwargs)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|