pipelet 0.0.3__tar.gz → 0.0.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. {pipelet-0.0.3 → pipelet-0.0.4}/PKG-INFO +6 -2
  2. {pipelet-0.0.3 → pipelet-0.0.4}/README.md +5 -1
  3. pipelet-0.0.4/pipelet/etl/transform/unzip/__init__.py +7 -0
  4. pipelet-0.0.4/pipelet/etl/transform/unzip/tar_processor.py +135 -0
  5. {pipelet-0.0.3 → pipelet-0.0.4}/pipelet/operations/operations.py +5 -1
  6. {pipelet-0.0.3 → pipelet-0.0.4}/pyproject.toml +1 -1
  7. pipelet-0.0.3/pipelet/etl/transform/unzip/__init__.py +0 -3
  8. {pipelet-0.0.3 → pipelet-0.0.4}/LICENSE +0 -0
  9. {pipelet-0.0.3 → pipelet-0.0.4}/pipelet/__init__.py +0 -0
  10. {pipelet-0.0.3 → pipelet-0.0.4}/pipelet/decarators/__init__.py +0 -0
  11. {pipelet-0.0.3 → pipelet-0.0.4}/pipelet/decarators/logging_decorator.py +0 -0
  12. {pipelet-0.0.3 → pipelet-0.0.4}/pipelet/decarators/option_decorator.py +0 -0
  13. {pipelet-0.0.3 → pipelet-0.0.4}/pipelet/etl/__init__.py +0 -0
  14. {pipelet-0.0.3 → pipelet-0.0.4}/pipelet/etl/extract/__init__.py +0 -0
  15. {pipelet-0.0.3 → pipelet-0.0.4}/pipelet/etl/extract/http/__init__.py +0 -0
  16. {pipelet-0.0.3 → pipelet-0.0.4}/pipelet/etl/extract/http/http_extractor.py +0 -0
  17. {pipelet-0.0.3 → pipelet-0.0.4}/pipelet/etl/transform/__init__.py +0 -0
  18. {pipelet-0.0.3 → pipelet-0.0.4}/pipelet/etl/transform/csv/__init__.py +0 -0
  19. {pipelet-0.0.3 → pipelet-0.0.4}/pipelet/etl/transform/csv/reader.py +0 -0
  20. {pipelet-0.0.3 → pipelet-0.0.4}/pipelet/etl/transform/json/__init__.py +0 -0
  21. {pipelet-0.0.3 → pipelet-0.0.4}/pipelet/etl/transform/json/reader.py +0 -0
  22. {pipelet-0.0.3 → pipelet-0.0.4}/pipelet/etl/transform/unzip/base.py +0 -0
  23. {pipelet-0.0.3 → pipelet-0.0.4}/pipelet/exceptions/__init__.py +0 -0
  24. {pipelet-0.0.3 → pipelet-0.0.4}/pipelet/exceptions/base.py +0 -0
  25. {pipelet-0.0.3 → pipelet-0.0.4}/pipelet/exceptions/csv_processor.py +0 -0
  26. {pipelet-0.0.3 → pipelet-0.0.4}/pipelet/exceptions/download_processor.py +0 -0
  27. {pipelet-0.0.3 → pipelet-0.0.4}/pipelet/exceptions/file_system.py +0 -0
  28. {pipelet-0.0.3 → pipelet-0.0.4}/pipelet/exceptions/unzip_processor.py +0 -0
  29. {pipelet-0.0.3 → pipelet-0.0.4}/pipelet/factories/__init__.py +0 -0
  30. {pipelet-0.0.3 → pipelet-0.0.4}/pipelet/factories/exceptions_factory.py +0 -0
  31. {pipelet-0.0.3 → pipelet-0.0.4}/pipelet/factories/file_system_factory.py +0 -0
  32. {pipelet-0.0.3 → pipelet-0.0.4}/pipelet/log/__init__.py +0 -0
  33. {pipelet-0.0.3 → pipelet-0.0.4}/pipelet/log/logger.py +0 -0
  34. {pipelet-0.0.3 → pipelet-0.0.4}/pipelet/operations/__init__.py +0 -0
  35. {pipelet-0.0.3 → pipelet-0.0.4}/pipelet/operations/pipeline.py +0 -0
  36. {pipelet-0.0.3 → pipelet-0.0.4}/pipelet/processors/__init__.py +0 -0
  37. {pipelet-0.0.3 → pipelet-0.0.4}/pipelet/processors/abc.py +0 -0
  38. {pipelet-0.0.3 → pipelet-0.0.4}/pipelet/processors/base.py +0 -0
  39. {pipelet-0.0.3 → pipelet-0.0.4}/pipelet/processors/chain_processors/__init__.py +0 -0
  40. {pipelet-0.0.3 → pipelet-0.0.4}/pipelet/processors/chain_processors/base.py +0 -0
  41. {pipelet-0.0.3 → pipelet-0.0.4}/pipelet/processors/file_system/__init__.py +0 -0
  42. {pipelet-0.0.3 → pipelet-0.0.4}/pipelet/processors/file_system/base.py +0 -0
  43. {pipelet-0.0.3 → pipelet-0.0.4}/pipelet/processors/file_system/file_system_types.py +0 -0
  44. {pipelet-0.0.3 → pipelet-0.0.4}/pipelet/processors/meta.py +0 -0
  45. {pipelet-0.0.3 → pipelet-0.0.4}/pipelet/processors/mixins.py +0 -0
  46. {pipelet-0.0.3 → pipelet-0.0.4}/pipelet/processors/processor_types.py +0 -0
  47. {pipelet-0.0.3 → pipelet-0.0.4}/pipelet/processors/retry_processor.py +0 -0
  48. {pipelet-0.0.3 → pipelet-0.0.4}/pipelet/processors/splitter_processor.py +0 -0
  49. {pipelet-0.0.3 → pipelet-0.0.4}/pipelet/utils/__init__.py +0 -0
  50. {pipelet-0.0.3 → pipelet-0.0.4}/pipelet/utils/common.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: pipelet
3
- Version: 0.0.3
3
+ Version: 0.0.4
4
4
  Summary:
5
5
  Author: Daniel Naumowich
6
6
  Author-email: cskteam4@gmail.com
@@ -72,7 +72,11 @@ A processor for parsing JSON data from strings or files into Python dictionaries
72
72
  ### 8. **UnzipProcessor**
73
73
  A processor for extracting files from `.zip` archives. It supports extracting data into files, managing chunk sizes, and deleting the original archive after processing.
74
74
 
75
- ### 9. **RetryProcessor**
75
+
76
+ ### 9. **TarExtractProcessor**
77
+ A processor for extracting files from `".tar", ".gz", ".bz2", ".xz", ".tgz", ".tbz2"` archives. It supports extracting data into files, managing chunk sizes, and deleting the original archive after processing.
78
+
79
+ ### 10. **RetryProcessor**
76
80
 
77
81
  A processor designed to handle transient errors by retrying operations based on a customizable retry policy.
78
82
 
@@ -52,7 +52,11 @@ A processor for parsing JSON data from strings or files into Python dictionaries
52
52
  ### 8. **UnzipProcessor**
53
53
  A processor for extracting files from `.zip` archives. It supports extracting data into files, managing chunk sizes, and deleting the original archive after processing.
54
54
 
55
- ### 9. **RetryProcessor**
55
+
56
+ ### 9. **TarExtractProcessor**
57
+ A processor for extracting files from `".tar", ".gz", ".bz2", ".xz", ".tgz", ".tbz2"` archives. It supports extracting data into files, managing chunk sizes, and deleting the original archive after processing.
58
+
59
+ ### 10. **RetryProcessor**
56
60
 
57
61
  A processor designed to handle transient errors by retrying operations based on a customizable retry policy.
58
62
 
@@ -0,0 +1,7 @@
1
+ from pipelet.etl.transform.unzip.base import UnzipProcessor
2
+ from pipelet.etl.transform.unzip.tar_processor import TarExtractProcessor
3
+
4
+ __all__ = (
5
+ "UnzipProcessor",
6
+ "TarExtractProcessor",
7
+ )
@@ -0,0 +1,135 @@
1
+ import tarfile
2
+ from os import PathLike
3
+ from pathlib import Path
4
+ from typing import Generator, List, Optional, Type, Union
5
+
6
+ from pipelet.exceptions.base import ProcessorStopIteration
7
+ from pipelet.exceptions.unzip_processor import UnzipError
8
+ from pipelet.log import logger_factory
9
+ from pipelet.processors.base import BaseProcessor
10
+ from pipelet.processors.file_system import (
11
+ AbstractFileSystemManager,
12
+ FileModeEnum,
13
+ )
14
+
15
+ logger = logger_factory()
16
+
17
+
18
+ class TarExtractProcessor(
19
+ BaseProcessor[
20
+ Union[str, PathLike[str]], Union[str, PathLike[str]], None, None
21
+ ],
22
+ ):
23
+ """
24
+ A processor that handles extracting `.tar` archives (including compressed formats).
25
+
26
+ This processor takes a `.tar` file (or compressed `.tar` variants like `.tar.gz`, `.tgz`),
27
+ extracts its contents, and yields the paths of the extracted files and directories.
28
+
29
+ Attributes:
30
+ _file_system_manager (AbstractFileSystemManager): File system manager for file operations.
31
+ _white_exceptions (List[Type[Exception]]): List of exceptions to bypass.
32
+ _auto_delete (bool): Flag to delete the tar file after processing.
33
+ _batch_size (Optional[int]): The batch size in bytes for extracting large files.
34
+ """
35
+
36
+ def __init__(
37
+ self,
38
+ file_system_manager: AbstractFileSystemManager[
39
+ Union[str, PathLike[str]], Union[str, bytes]
40
+ ],
41
+ white_exceptions: Optional[List[Type[Exception]]] = None,
42
+ auto_delete: bool = True,
43
+ batch_size: Optional[int] = None,
44
+ ) -> None:
45
+ self._file_system_manager = file_system_manager
46
+ self._white_exceptions = white_exceptions or []
47
+ self._auto_delete = auto_delete
48
+ self._batch_size = (
49
+ batch_size * 1024 * 1024 if batch_size is not None else None
50
+ )
51
+
52
+ def process(
53
+ self,
54
+ input_data: Union[str, PathLike[str]],
55
+ ) -> Generator[Union[str, PathLike[str]], None, None]:
56
+ """
57
+ Processes the given tar file and extracts its contents.
58
+
59
+ Args:
60
+ input_data (Union[str, PathLike[str]]): The path to the tar file to extract.
61
+
62
+ Yields:
63
+ Union[str, PathLike[str]]: The path to each extracted file or directory.
64
+
65
+ Raises:
66
+ UnzipError: If the file is not a `.tar` file or cannot be processed.
67
+ """
68
+ suffix = Path(input_data).suffix
69
+ if suffix not in [".tar", ".gz", ".bz2", ".xz", ".tgz", ".tbz2"]:
70
+ raise UnzipError(
71
+ input_data,
72
+ f"File has an extension '{suffix}', expected a supported .tar format",
73
+ )
74
+
75
+ file_path = self._file_system_manager.get_path(input_data)
76
+ extracted_paths: List[Union[str, PathLike[str]]] = []
77
+
78
+ try:
79
+ with tarfile.open(file_path, mode="r:*") as tar:
80
+ for member in tar.getmembers():
81
+ extracted_path = self._file_system_manager.get_path(
82
+ member.name
83
+ )
84
+ if member.isdir():
85
+ # Create the directory if it does not exist
86
+ Path(extracted_path).mkdir(parents=True, exist_ok=True)
87
+ logger.info(f"Created directory: {member.name}")
88
+ extracted_paths.append(extracted_path)
89
+ elif member.isfile():
90
+ # Extract file and handle large files with batching
91
+ source_file = tar.extractfile(member)
92
+ if source_file is not None:
93
+ with source_file, open(
94
+ extracted_path, FileModeEnum.WRITE_BINARY
95
+ ) as target_file:
96
+ if self._batch_size is not None:
97
+ while chunk := source_file.read(
98
+ self._batch_size
99
+ ):
100
+ target_file.write(chunk)
101
+ else:
102
+ target_file.write(source_file.read())
103
+ extracted_paths.append(extracted_path)
104
+ else:
105
+ logger.warning(
106
+ f"Could not extract file: {member.name}"
107
+ )
108
+ else:
109
+ logger.info(
110
+ f"Skipping unsupported member type: {member.name}"
111
+ )
112
+
113
+ except Exception as e:
114
+ raise UnzipError(file_path, f"Failed to extract tar file: {e}")
115
+
116
+ # Pass extracted paths to the next processor, if defined
117
+ if self._next is not None:
118
+ for item in extracted_paths:
119
+ try:
120
+ yield from self._next.process(item)
121
+ except ProcessorStopIteration:
122
+ pass
123
+ else:
124
+ yield from iter(extracted_paths)
125
+
126
+ # Optionally delete the tar file after processing
127
+ if self._auto_delete:
128
+ self._file_system_manager.delete_file(input_data)
129
+
130
+ def __str__(self) -> str:
131
+ return (
132
+ f"{self.__class__.__name__}"
133
+ f"(auto_delete={self._auto_delete}, "
134
+ f"batch_size={self._batch_size})"
135
+ )
@@ -62,11 +62,13 @@ class BaseOpEnum(str, Enum):
62
62
  downloading = "pipelet.etl.extract.http.HttpDataExtractProcessor"
63
63
  stream_downloading = "pipelet.etl.extract.http.HttpxStreamDownloadProcessor"
64
64
  unzip = "pipelet.etl.transform.unzip.UnzipProcessor"
65
+ tar_extract = (
66
+ "pipelet.etl.transform.unzip.tar_processor.TarExtractProcessor"
67
+ )
65
68
  csv_parsing = "pipelet.etl.transform.csv.CsvParser"
66
69
  json_parsing = "pipelet.etl.transform.json.JsonParser"
67
70
  retrying = "pipelet.processors.retry_processor.RetryProcessor"
68
71
  splitting = "pipelet.processors.splitter_processor.SplitterProcessor"
69
- logging = "pipelet.processors.logging_processor.LoggingProcessor"
70
72
 
71
73
 
72
74
  T = TypeVar("T", bound="BaseOpComposite")
@@ -245,6 +247,8 @@ def process_op(op: Op) -> BaseProcessor[Any, Any, Any, Any]:
245
247
  factory = KEYWORD_FACTORY_MAP.get(k, None)
246
248
  if factory:
247
249
  kwargs[k] = factory(**v) if isinstance(v, dict) else factory(v)
250
+ elif OPTION_TO_HANDLER_MAP.get(k, None) is None:
251
+ kwargs[k] = v
248
252
  else:
249
253
  kwargs_to_additional_processors[k] = v
250
254
  processor = type_processor(**kwargs)
@@ -1,6 +1,6 @@
1
1
  [tool.poetry]
2
2
  name = "pipelet"
3
- version = "0.0.3"
3
+ version = "0.0.4"
4
4
  description = ""
5
5
  authors = ["Daniel Naumowich <cskteam4@gmail.com>"]
6
6
  readme = "README.md"
@@ -1,3 +0,0 @@
1
- from pipelet.etl.transform.unzip.base import UnzipProcessor
2
-
3
- __all__ = ("UnzipProcessor",)
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes