containerio 2.3.2__tar.gz → 2.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- containerio-2.3.2/README.md → containerio-2.4.0/PKG-INFO +26 -0
- containerio-2.3.2/PKG-INFO → containerio-2.4.0/README.md +12 -14
- containerio-2.4.0/pyproject.toml +74 -0
- containerio-2.3.2/pyproject.toml → containerio-2.4.0/pyproject.toml.orig +1 -1
- {containerio-2.3.2 → containerio-2.4.0}/src/containerio/handler.py +297 -140
- {containerio-2.3.2 → containerio-2.4.0}/src/containerio/tools/cli.py +101 -1
- {containerio-2.3.2 → containerio-2.4.0}/src/containerio/__init__.py +0 -0
- {containerio-2.3.2 → containerio-2.4.0}/src/containerio/azure_config.py +0 -0
- {containerio-2.3.2 → containerio-2.4.0}/src/containerio/session.py +0 -0
|
@@ -1,3 +1,17 @@
|
|
|
1
|
+
Metadata-Version: 2.3
|
|
2
|
+
Name: containerio
|
|
3
|
+
Version: 2.4.0
|
|
4
|
+
Summary: Azure Blob Storage handler with unified local/cloud interface
|
|
5
|
+
Author: Alliance SwissPass
|
|
6
|
+
Requires-Dist: polars>=1.30.0
|
|
7
|
+
Requires-Dist: azure-storage-blob>=12.26.0
|
|
8
|
+
Requires-Dist: azure-identity>=1.25.0
|
|
9
|
+
Requires-Dist: python-dotenv>=1.0.0
|
|
10
|
+
Requires-Dist: rich>=13.0.0
|
|
11
|
+
Requires-Python: >=3.11
|
|
12
|
+
Project-URL: Repository, https://codefloe.com/Alliance-SwissPass/py-containerio
|
|
13
|
+
Description-Content-Type: text/markdown
|
|
14
|
+
|
|
1
15
|
# py-containerio
|
|
2
16
|
|
|
3
17
|
Shared Azure Blob Storage handler for internal and collaboration projects of [Alliance SwissPass](https://www.allianceswisspass.ch/).
|
|
@@ -288,9 +302,21 @@ uv run io ls --env staging --container raw-data
|
|
|
288
302
|
# Download a blob to the local data/ directory
|
|
289
303
|
uv run io download path/to/file.csv --env staging --container raw-data
|
|
290
304
|
|
|
305
|
+
# Recursively download all blobs under a prefix to a local directory
|
|
306
|
+
uv run io download-dir archive/ --env staging --container raw-data
|
|
307
|
+
uv run io download-dir archive/ --local-dir ./downloads --env staging --container raw-data
|
|
308
|
+
|
|
309
|
+
# Recursively download the blobs of entire container (no prefix, from root)
|
|
310
|
+
uv run io download-dir --env staging --container raw-data
|
|
311
|
+
|
|
291
312
|
# Upload a local file
|
|
292
313
|
uv run io upload path/to/blob.csv ./local/file.csv --env staging --container curated
|
|
293
314
|
|
|
315
|
+
# Upload a local directory recursively to the container root directory
|
|
316
|
+
uv run io upload-dir ./local/data --env staging --container curated
|
|
317
|
+
# Upload a local directory recursively to a prefix in the container
|
|
318
|
+
uv run io upload-dir ./local/data --blob-prefix archive/2025 --env staging --container curated
|
|
319
|
+
|
|
294
320
|
# Delete a blob or folder
|
|
295
321
|
uv run io rm path/to/blob.csv --env staging --container curated
|
|
296
322
|
uv run io rm old-folder/ --env staging --container curated
|
|
@@ -1,17 +1,3 @@
|
|
|
1
|
-
Metadata-Version: 2.3
|
|
2
|
-
Name: containerio
|
|
3
|
-
Version: 2.3.2
|
|
4
|
-
Summary: Azure Blob Storage handler with unified local/cloud interface
|
|
5
|
-
Author: Alliance SwissPass
|
|
6
|
-
Requires-Dist: polars>=1.30.0
|
|
7
|
-
Requires-Dist: azure-storage-blob>=12.26.0
|
|
8
|
-
Requires-Dist: azure-identity>=1.25.0
|
|
9
|
-
Requires-Dist: python-dotenv>=1.0.0
|
|
10
|
-
Requires-Dist: rich>=13.0.0
|
|
11
|
-
Requires-Python: >=3.11
|
|
12
|
-
Project-URL: Repository, https://codefloe.com/Alliance-SwissPass/py-containerio
|
|
13
|
-
Description-Content-Type: text/markdown
|
|
14
|
-
|
|
15
1
|
# py-containerio
|
|
16
2
|
|
|
17
3
|
Shared Azure Blob Storage handler for internal and collaboration projects of [Alliance SwissPass](https://www.allianceswisspass.ch/).
|
|
@@ -302,9 +288,21 @@ uv run io ls --env staging --container raw-data
|
|
|
302
288
|
# Download a blob to the local data/ directory
|
|
303
289
|
uv run io download path/to/file.csv --env staging --container raw-data
|
|
304
290
|
|
|
291
|
+
# Recursively download all blobs under a prefix to a local directory
|
|
292
|
+
uv run io download-dir archive/ --env staging --container raw-data
|
|
293
|
+
uv run io download-dir archive/ --local-dir ./downloads --env staging --container raw-data
|
|
294
|
+
|
|
295
|
+
# Recursively download the blobs of entire container (no prefix, from root)
|
|
296
|
+
uv run io download-dir --env staging --container raw-data
|
|
297
|
+
|
|
305
298
|
# Upload a local file
|
|
306
299
|
uv run io upload path/to/blob.csv ./local/file.csv --env staging --container curated
|
|
307
300
|
|
|
301
|
+
# Upload a local directory recursively to the container root directory
|
|
302
|
+
uv run io upload-dir ./local/data --env staging --container curated
|
|
303
|
+
# Upload a local directory recursively to a prefix in the container
|
|
304
|
+
uv run io upload-dir ./local/data --blob-prefix archive/2025 --env staging --container curated
|
|
305
|
+
|
|
308
306
|
# Delete a blob or folder
|
|
309
307
|
uv run io rm path/to/blob.csv --env staging --container curated
|
|
310
308
|
uv run io rm old-folder/ --env staging --container curated
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "containerio"
|
|
3
|
+
version = "2.4.0"
|
|
4
|
+
description = "Azure Blob Storage handler with unified local/cloud interface"
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
requires-python = ">=3.11"
|
|
7
|
+
dependencies = [
|
|
8
|
+
"polars>=1.30.0",
|
|
9
|
+
"azure-storage-blob>=12.26.0",
|
|
10
|
+
"azure-identity>=1.25.0",
|
|
11
|
+
"python-dotenv>=1.0.0",
|
|
12
|
+
"rich>=13.0.0",
|
|
13
|
+
]
|
|
14
|
+
|
|
15
|
+
[[project.authors]]
|
|
16
|
+
name = "Alliance SwissPass"
|
|
17
|
+
|
|
18
|
+
[project.urls]
|
|
19
|
+
Repository = "https://codefloe.com/Alliance-SwissPass/py-containerio"
|
|
20
|
+
|
|
21
|
+
[project.scripts]
|
|
22
|
+
io = "containerio.tools.cli:main"
|
|
23
|
+
|
|
24
|
+
[dependency-groups]
|
|
25
|
+
dev = [
|
|
26
|
+
"uv>=0.1.0",
|
|
27
|
+
"twine>=6.0.0",
|
|
28
|
+
]
|
|
29
|
+
lint = ["ruff>=0.11.0"]
|
|
30
|
+
test = ["pytest>=8.3.5"]
|
|
31
|
+
|
|
32
|
+
[tool.pytest.ini_options]
|
|
33
|
+
testpaths = ["tests"]
|
|
34
|
+
python_files = "test_*.py"
|
|
35
|
+
python_functions = "test_*"
|
|
36
|
+
pythonpath = ["src"]
|
|
37
|
+
|
|
38
|
+
[tool.uv]
|
|
39
|
+
package = true
|
|
40
|
+
|
|
41
|
+
[tool.ruff]
|
|
42
|
+
line-length = 88
|
|
43
|
+
fix = true
|
|
44
|
+
|
|
45
|
+
[tool.ruff.lint]
|
|
46
|
+
select = [
|
|
47
|
+
"E",
|
|
48
|
+
"W",
|
|
49
|
+
"F",
|
|
50
|
+
"I",
|
|
51
|
+
"N",
|
|
52
|
+
"B",
|
|
53
|
+
"D",
|
|
54
|
+
]
|
|
55
|
+
ignore = [
|
|
56
|
+
"D100",
|
|
57
|
+
"D101",
|
|
58
|
+
"D102",
|
|
59
|
+
"D103",
|
|
60
|
+
"D104",
|
|
61
|
+
"D105",
|
|
62
|
+
"D106",
|
|
63
|
+
"D107",
|
|
64
|
+
]
|
|
65
|
+
|
|
66
|
+
[tool.ruff.lint.pydocstyle]
|
|
67
|
+
convention = "google"
|
|
68
|
+
|
|
69
|
+
[tool.ruff.lint.pycodestyle]
|
|
70
|
+
max-doc-length = 72
|
|
71
|
+
|
|
72
|
+
[build-system]
|
|
73
|
+
requires = ["uv_build>=0.11.6,<0.12"]
|
|
74
|
+
build-backend = "uv_build"
|
|
@@ -49,8 +49,9 @@ class StorageHandler:
|
|
|
49
49
|
"""Handles interactions with both local filesystem and Blob Storage.
|
|
50
50
|
|
|
51
51
|
This class provides a unified interface for reading, writing, and
|
|
52
|
-
managing data regardless of whether it's stored locally or in
|
|
53
|
-
The storage backend is determined by the `config`
|
|
52
|
+
managing data regardless of whether it's stored locally or in
|
|
53
|
+
Azure. The storage backend is determined by the `config`
|
|
54
|
+
argument:
|
|
54
55
|
- `config=None`: local filesystem (./data/ directory)
|
|
55
56
|
- `config=<AzureStorageConfig>`: cloud storage using that
|
|
56
57
|
configuration
|
|
@@ -62,6 +63,143 @@ class StorageHandler:
|
|
|
62
63
|
token (with user delegation key) or client secret credentials.
|
|
63
64
|
"""
|
|
64
65
|
|
|
66
|
+
def read_json_blob(self, blob_name: str, **kwargs: Dict) -> pl.DataFrame:
|
|
67
|
+
"""Reads a JSON file from storage into a Polars DataFrame.
|
|
68
|
+
|
|
69
|
+
Args:
|
|
70
|
+
blob_name (str): The path/name of the file to read.
|
|
71
|
+
**kwargs: Additional keyword arguments passed to
|
|
72
|
+
`polars.read_json`.
|
|
73
|
+
|
|
74
|
+
Returns:
|
|
75
|
+
pl.DataFrame: The content of the JSON file as a Polars
|
|
76
|
+
DataFrame.
|
|
77
|
+
"""
|
|
78
|
+
data = self.read_blob_as_bytes(blob_name=blob_name)
|
|
79
|
+
return pl.read_json(source=BytesIO(data), **kwargs)
|
|
80
|
+
|
|
81
|
+
def read_csv_blob(self, blob_name: str, **kwargs: Dict) -> pl.DataFrame:
|
|
82
|
+
"""Reads a CSV file from storage into a Polars DataFrame.
|
|
83
|
+
|
|
84
|
+
Args:
|
|
85
|
+
blob_name (str): The path/name of the file to read.
|
|
86
|
+
**kwargs: Additional keyword arguments passed to
|
|
87
|
+
`polars.read_csv`.
|
|
88
|
+
|
|
89
|
+
Returns:
|
|
90
|
+
pl.DataFrame: The content of the CSV file as a Polars
|
|
91
|
+
DataFrame.
|
|
92
|
+
"""
|
|
93
|
+
data = self.read_blob_as_bytes(blob_name=blob_name)
|
|
94
|
+
return pl.read_csv(source=BytesIO(data), **kwargs)
|
|
95
|
+
|
|
96
|
+
def read_excel_blob(self, blob_name: str, **kwargs: Dict) -> pl.DataFrame:
|
|
97
|
+
"""Reads an Excel file from storage into a Polars DataFrame.
|
|
98
|
+
|
|
99
|
+
Args:
|
|
100
|
+
blob_name (str): The path/name of the file to read.
|
|
101
|
+
**kwargs: Additional keyword arguments passed to
|
|
102
|
+
`polars.read_excel`.
|
|
103
|
+
|
|
104
|
+
Returns:
|
|
105
|
+
pl.DataFrame: The content of the Excel file as a Polars
|
|
106
|
+
DataFrame.
|
|
107
|
+
"""
|
|
108
|
+
data = self.read_blob_as_bytes(blob_name=blob_name)
|
|
109
|
+
df = pl.read_excel(source=BytesIO(data), **kwargs)
|
|
110
|
+
|
|
111
|
+
if not isinstance(df, pl.DataFrame):
|
|
112
|
+
raise TypeError(
|
|
113
|
+
"read_excel_blob expects a single sheet returning a DataFrame, "
|
|
114
|
+
"but multiple sheets were read. Pass a sheet_id or sheet_name "
|
|
115
|
+
"keyword argument that selects exactly one sheet."
|
|
116
|
+
)
|
|
117
|
+
|
|
118
|
+
return df
|
|
119
|
+
|
|
120
|
+
def read_parquet_blob(self, blob_name: str) -> Tuple[pl.DataFrame, Dict]:
|
|
121
|
+
"""Reads a Parquet file from storage into a Polars DataFrame.
|
|
122
|
+
|
|
123
|
+
Args:
|
|
124
|
+
blob_name (str): The path/name of the file to read.
|
|
125
|
+
|
|
126
|
+
Returns:
|
|
127
|
+
Tuple[pl.DataFrame, Dict]: A tuple containing the DataFrame
|
|
128
|
+
and the Parquet file's metadata.
|
|
129
|
+
"""
|
|
130
|
+
data = self.read_blob_as_bytes(blob_name=blob_name)
|
|
131
|
+
df = pl.read_parquet(source=BytesIO(data))
|
|
132
|
+
metadata = pl.read_parquet_metadata(source=BytesIO(data))
|
|
133
|
+
return df, metadata
|
|
134
|
+
|
|
135
|
+
def write_json_blob(self, blob_name: str, df: pl.DataFrame, **kwargs: Dict) -> None:
|
|
136
|
+
"""Writes a Polars DataFrame to a JSON file in storage.
|
|
137
|
+
|
|
138
|
+
Args:
|
|
139
|
+
blob_name (str): The path/name of the file to write to.
|
|
140
|
+
df (pl.DataFrame): The DataFrame to write.
|
|
141
|
+
**kwargs: Additional keyword arguments passed to
|
|
142
|
+
`DataFrame.write_json`.
|
|
143
|
+
|
|
144
|
+
Returns:
|
|
145
|
+
None
|
|
146
|
+
"""
|
|
147
|
+
buffer = BytesIO()
|
|
148
|
+
df.write_json(file=buffer, **kwargs)
|
|
149
|
+
self.write_blob_as_bytes(blob_name=blob_name, data=buffer.getvalue())
|
|
150
|
+
|
|
151
|
+
def write_csv_blob(self, blob_name: str, df: pl.DataFrame, **kwargs: Dict) -> None:
|
|
152
|
+
"""Writes a Polars DataFrame to a CSV file in storage.
|
|
153
|
+
|
|
154
|
+
Args:
|
|
155
|
+
blob_name (str): The path/name of the file to write to.
|
|
156
|
+
df (pl.DataFrame): The DataFrame to write.
|
|
157
|
+
**kwargs: Additional keyword arguments passed to
|
|
158
|
+
`DataFrame.write_csv`.
|
|
159
|
+
|
|
160
|
+
Returns:
|
|
161
|
+
None
|
|
162
|
+
"""
|
|
163
|
+
buffer = BytesIO()
|
|
164
|
+
df.write_csv(file=buffer, **kwargs)
|
|
165
|
+
self.write_blob_as_bytes(blob_name=blob_name, data=buffer.getvalue())
|
|
166
|
+
|
|
167
|
+
def write_excel_blob(
|
|
168
|
+
self, blob_name: str, df: pl.DataFrame, **kwargs: Dict
|
|
169
|
+
) -> None:
|
|
170
|
+
"""Writes a Polars DataFrame to an Excel file in storage.
|
|
171
|
+
|
|
172
|
+
Args:
|
|
173
|
+
blob_name (str): The path/name of the file to write to.
|
|
174
|
+
df (pl.DataFrame): The DataFrame to write.
|
|
175
|
+
**kwargs: Additional keyword arguments passed to
|
|
176
|
+
`DataFrame.write_excel`.
|
|
177
|
+
|
|
178
|
+
Returns:
|
|
179
|
+
None
|
|
180
|
+
"""
|
|
181
|
+
buffer = BytesIO()
|
|
182
|
+
df.write_excel(workbook=buffer, **kwargs)
|
|
183
|
+
self.write_blob_as_bytes(blob_name=blob_name, data=buffer.getvalue())
|
|
184
|
+
|
|
185
|
+
def write_parquet_blob(
|
|
186
|
+
self, blob_name: str, df: pl.DataFrame, **kwargs: Dict
|
|
187
|
+
) -> None:
|
|
188
|
+
"""Writes a Polars DataFrame to a Parquet file in storage.
|
|
189
|
+
|
|
190
|
+
Args:
|
|
191
|
+
blob_name (str): The path/name of the file to write to.
|
|
192
|
+
df (pl.DataFrame): The DataFrame to write.
|
|
193
|
+
**kwargs: Additional keyword arguments passed to
|
|
194
|
+
`DataFrame.write_parquet`.
|
|
195
|
+
|
|
196
|
+
Returns:
|
|
197
|
+
None
|
|
198
|
+
"""
|
|
199
|
+
buffer = BytesIO()
|
|
200
|
+
df.write_parquet(file=buffer, **kwargs)
|
|
201
|
+
self.write_blob_as_bytes(blob_name=blob_name, data=buffer.getvalue())
|
|
202
|
+
|
|
65
203
|
DEFAULT_SCOPES = ["https://storage.azure.com/.default"]
|
|
66
204
|
|
|
67
205
|
DEFAULT_TOKEN_CACHE = os.path.expanduser("~/.cache/polars_azure/token_cache.bin")
|
|
@@ -125,7 +263,7 @@ class StorageHandler:
|
|
|
125
263
|
self,
|
|
126
264
|
auth_type: Optional[str] = None,
|
|
127
265
|
) -> AzureAuthType:
|
|
128
|
-
"""Resolve the Azure auth type from a string.
|
|
266
|
+
"""Resolve the Azure auth type from a string or session.
|
|
129
267
|
|
|
130
268
|
If no auth_type is provided, checks the session
|
|
131
269
|
(written by ``io auth login``) for a saved auth_method.
|
|
@@ -672,36 +810,6 @@ class StorageHandler:
|
|
|
672
810
|
console = Console()
|
|
673
811
|
console.print(tree)
|
|
674
812
|
|
|
675
|
-
def read_json_blob(self, blob_name: str, **kwargs: Dict) -> pl.DataFrame:
|
|
676
|
-
"""Reads a JSON file from storage into a Polars DataFrame.
|
|
677
|
-
|
|
678
|
-
Args:
|
|
679
|
-
blob_name (str): The path/name of the file to read.
|
|
680
|
-
**kwargs: Additional keyword arguments passed to
|
|
681
|
-
`polars.read_json`.
|
|
682
|
-
|
|
683
|
-
Returns:
|
|
684
|
-
pl.DataFrame: The content of the JSON file as a Polars
|
|
685
|
-
DataFrame.
|
|
686
|
-
"""
|
|
687
|
-
data = self.read_blob_as_bytes(blob_name=blob_name)
|
|
688
|
-
return pl.read_json(source=BytesIO(data), **kwargs)
|
|
689
|
-
|
|
690
|
-
def read_csv_blob(self, blob_name: str, **kwargs: Dict) -> pl.DataFrame:
|
|
691
|
-
"""Reads a CSV file from storage into a Polars DataFrame.
|
|
692
|
-
|
|
693
|
-
Args:
|
|
694
|
-
blob_name (str): The path/name of the file to read.
|
|
695
|
-
**kwargs: Additional keyword arguments passed to
|
|
696
|
-
`polars.read_csv`.
|
|
697
|
-
|
|
698
|
-
Returns:
|
|
699
|
-
pl.DataFrame: The content of the CSV file as a Polars
|
|
700
|
-
DataFrame.
|
|
701
|
-
"""
|
|
702
|
-
data = self.read_blob_as_bytes(blob_name=blob_name)
|
|
703
|
-
return pl.read_csv(source=BytesIO(data), **kwargs)
|
|
704
|
-
|
|
705
813
|
def scan_csv_blob(self, blob_name: str, **kwargs: Dict) -> pl.LazyFrame:
|
|
706
814
|
"""Scans a CSV file or multiple files from storage.
|
|
707
815
|
|
|
@@ -728,45 +836,6 @@ class StorageHandler:
|
|
|
728
836
|
auth_kwargs = self._get_azure_auth_kwargs()
|
|
729
837
|
return pl.scan_csv(source=abfss_path, **auth_kwargs, **kwargs)
|
|
730
838
|
|
|
731
|
-
def read_excel_blob(self, blob_name: str, **kwargs: Dict) -> pl.DataFrame:
|
|
732
|
-
"""Reads an Excel file from storage into a Polars DataFrame.
|
|
733
|
-
|
|
734
|
-
Args:
|
|
735
|
-
blob_name (str): The path/name of the file to read.
|
|
736
|
-
**kwargs: Additional keyword arguments passed to
|
|
737
|
-
`polars.read_excel`.
|
|
738
|
-
|
|
739
|
-
Returns:
|
|
740
|
-
pl.DataFrame: The content of the Excel file as a Polars
|
|
741
|
-
DataFrame.
|
|
742
|
-
"""
|
|
743
|
-
data = self.read_blob_as_bytes(blob_name=blob_name)
|
|
744
|
-
df = pl.read_excel(source=BytesIO(data), **kwargs)
|
|
745
|
-
|
|
746
|
-
if not isinstance(df, pl.DataFrame):
|
|
747
|
-
raise TypeError(
|
|
748
|
-
"read_excel_blob expects a single sheet returning a DataFrame, "
|
|
749
|
-
"but multiple sheets were read. Pass a sheet_id or sheet_name "
|
|
750
|
-
"keyword argument that selects exactly one sheet."
|
|
751
|
-
)
|
|
752
|
-
|
|
753
|
-
return df
|
|
754
|
-
|
|
755
|
-
def read_parquet_blob(self, blob_name: str) -> Tuple[pl.DataFrame, Dict]:
|
|
756
|
-
"""Reads a Parquet file from storage into a Polars DataFrame.
|
|
757
|
-
|
|
758
|
-
Args:
|
|
759
|
-
blob_name (str): The path/name of the file to read.
|
|
760
|
-
|
|
761
|
-
Returns:
|
|
762
|
-
Tuple[pl.DataFrame, Dict]: A tuple containing the DataFrame
|
|
763
|
-
and the Parquet file's metadata.
|
|
764
|
-
"""
|
|
765
|
-
data = self.read_blob_as_bytes(blob_name=blob_name)
|
|
766
|
-
df = pl.read_parquet(source=BytesIO(data))
|
|
767
|
-
metadata = pl.read_parquet_metadata(source=BytesIO(data))
|
|
768
|
-
return df, metadata
|
|
769
|
-
|
|
770
839
|
def scan_parquet_blob(
|
|
771
840
|
self, blob_name: str, **kwargs: Dict
|
|
772
841
|
) -> Tuple[pl.LazyFrame, Dict]:
|
|
@@ -844,6 +913,87 @@ class StorageHandler:
|
|
|
844
913
|
|
|
845
914
|
return download_path
|
|
846
915
|
|
|
916
|
+
def download_dir(
|
|
917
|
+
self,
|
|
918
|
+
dir_prefix: str,
|
|
919
|
+
local_dir: Optional[Union[str, Path]] = None,
|
|
920
|
+
progress_hook: Optional[Callable] = None,
|
|
921
|
+
) -> List[Path]:
|
|
922
|
+
"""Download all blobs under a prefix to a local directory.
|
|
923
|
+
|
|
924
|
+
Recursively lists blobs matching ``dir_prefix`` and downloads
|
|
925
|
+
each one, preserving the relative directory structure beneath
|
|
926
|
+
the prefix.
|
|
927
|
+
|
|
928
|
+
Args:
|
|
929
|
+
dir_prefix (str): The storage path/prefix to download
|
|
930
|
+
recursively. A trailing ``/`` is optional.
|
|
931
|
+
local_dir (Optional[Union[str, Path]]): The local
|
|
932
|
+
destination directory. Defaults to ``./data``
|
|
933
|
+
(mirroring :meth:`download_blob`). Created if it
|
|
934
|
+
does not exist.
|
|
935
|
+
progress_hook (Optional[Callable]): Optional callback
|
|
936
|
+
invoked after each file finishes download with
|
|
937
|
+
``(file_path, bytes_downloaded, total_bytes)``.
|
|
938
|
+
|
|
939
|
+
Returns:
|
|
940
|
+
List[Path]: Paths of all downloaded files.
|
|
941
|
+
|
|
942
|
+
Raises:
|
|
943
|
+
FileNotFoundError: If the prefix does not exist or
|
|
944
|
+
contains no files.
|
|
945
|
+
"""
|
|
946
|
+
if self.local:
|
|
947
|
+
return []
|
|
948
|
+
|
|
949
|
+
local_base = Path(local_dir) if local_dir else Path("./data")
|
|
950
|
+
local_base.mkdir(parents=True, exist_ok=True)
|
|
951
|
+
|
|
952
|
+
prefix = dir_prefix.rstrip("/")
|
|
953
|
+
prefix_filter = prefix + "/" if prefix else ""
|
|
954
|
+
|
|
955
|
+
blobs: List[BlobInfo] = []
|
|
956
|
+
|
|
957
|
+
for blob in self.container_client.list_blobs(
|
|
958
|
+
include=["metadata"], name_starts_with=prefix_filter
|
|
959
|
+
):
|
|
960
|
+
if self._is_hns_directory(blob.metadata):
|
|
961
|
+
continue
|
|
962
|
+
blobs.append(
|
|
963
|
+
BlobInfo(
|
|
964
|
+
name=blob.name,
|
|
965
|
+
size=blob.size,
|
|
966
|
+
last_modified=blob.last_modified,
|
|
967
|
+
)
|
|
968
|
+
)
|
|
969
|
+
|
|
970
|
+
if not blobs:
|
|
971
|
+
raise FileNotFoundError(f"No blobs found under prefix: {dir_prefix}")
|
|
972
|
+
|
|
973
|
+
downloaded_files: List[Path] = []
|
|
974
|
+
|
|
975
|
+
for blob in blobs:
|
|
976
|
+
rel_name = blob.name
|
|
977
|
+
if rel_name.startswith(prefix_filter) and prefix_filter:
|
|
978
|
+
rel_name = rel_name[len(prefix_filter) :]
|
|
979
|
+
|
|
980
|
+
dest_path = local_base / rel_name
|
|
981
|
+
dest_path.parent.mkdir(parents=True, exist_ok=True)
|
|
982
|
+
|
|
983
|
+
blob_client = self._get_blob_client(blob_name=blob.name)
|
|
984
|
+
stream = blob_client.download_blob()
|
|
985
|
+
downloaded = 0
|
|
986
|
+
with open(dest_path, "wb") as out_file:
|
|
987
|
+
for chunk in stream.chunks():
|
|
988
|
+
out_file.write(chunk)
|
|
989
|
+
downloaded += len(chunk)
|
|
990
|
+
if progress_hook:
|
|
991
|
+
progress_hook(dest_path, downloaded, blob.size)
|
|
992
|
+
|
|
993
|
+
downloaded_files.append(dest_path)
|
|
994
|
+
|
|
995
|
+
return downloaded_files
|
|
996
|
+
|
|
847
997
|
def write_blob_as_bytes(
|
|
848
998
|
self, blob_name: str, data: bytes
|
|
849
999
|
) -> Optional[Dict[str, Any]]:
|
|
@@ -884,74 +1034,6 @@ class StorageHandler:
|
|
|
884
1034
|
except Exception as e:
|
|
885
1035
|
raise RuntimeError(f"Failed to write blob '{blob_name}': {e}") from e
|
|
886
1036
|
|
|
887
|
-
def write_json_blob(self, blob_name: str, df: pl.DataFrame, **kwargs: Dict) -> None:
|
|
888
|
-
"""Writes a Polars DataFrame to a JSON file in storage.
|
|
889
|
-
|
|
890
|
-
Args:
|
|
891
|
-
blob_name (str): The path/name of the file to write to.
|
|
892
|
-
df (pl.DataFrame): The DataFrame to write.
|
|
893
|
-
**kwargs: Additional keyword arguments passed to
|
|
894
|
-
`DataFrame.write_json`.
|
|
895
|
-
|
|
896
|
-
Returns:
|
|
897
|
-
None
|
|
898
|
-
"""
|
|
899
|
-
buffer = BytesIO()
|
|
900
|
-
df.write_json(file=buffer, **kwargs)
|
|
901
|
-
self.write_blob_as_bytes(blob_name=blob_name, data=buffer.getvalue())
|
|
902
|
-
|
|
903
|
-
def write_csv_blob(self, blob_name: str, df: pl.DataFrame, **kwargs: Dict) -> None:
|
|
904
|
-
"""Writes a Polars DataFrame to a CSV file in storage.
|
|
905
|
-
|
|
906
|
-
Args:
|
|
907
|
-
blob_name (str): The path/name of the file to write to.
|
|
908
|
-
df (pl.DataFrame): The DataFrame to write.
|
|
909
|
-
**kwargs: Additional keyword arguments passed to
|
|
910
|
-
`DataFrame.write_csv`.
|
|
911
|
-
|
|
912
|
-
Returns:
|
|
913
|
-
None
|
|
914
|
-
"""
|
|
915
|
-
buffer = BytesIO()
|
|
916
|
-
df.write_csv(file=buffer, **kwargs)
|
|
917
|
-
self.write_blob_as_bytes(blob_name=blob_name, data=buffer.getvalue())
|
|
918
|
-
|
|
919
|
-
def write_excel_blob(
|
|
920
|
-
self, blob_name: str, df: pl.DataFrame, **kwargs: Dict
|
|
921
|
-
) -> None:
|
|
922
|
-
"""Writes a Polars DataFrame to an Excel file in storage.
|
|
923
|
-
|
|
924
|
-
Args:
|
|
925
|
-
blob_name (str): The path/name of the file to write to.
|
|
926
|
-
df (pl.DataFrame): The DataFrame to write.
|
|
927
|
-
**kwargs: Additional keyword arguments passed to
|
|
928
|
-
`DataFrame.write_excel`.
|
|
929
|
-
|
|
930
|
-
Returns:
|
|
931
|
-
None
|
|
932
|
-
"""
|
|
933
|
-
buffer = BytesIO()
|
|
934
|
-
df.write_excel(workbook=buffer, **kwargs)
|
|
935
|
-
self.write_blob_as_bytes(blob_name=blob_name, data=buffer.getvalue())
|
|
936
|
-
|
|
937
|
-
def write_parquet_blob(
|
|
938
|
-
self, blob_name: str, df: pl.DataFrame, **kwargs: Dict
|
|
939
|
-
) -> None:
|
|
940
|
-
"""Writes a Polars DataFrame to a Parquet file in storage.
|
|
941
|
-
|
|
942
|
-
Args:
|
|
943
|
-
blob_name (str): The path/name of the file to write to.
|
|
944
|
-
df (pl.DataFrame): The DataFrame to write.
|
|
945
|
-
**kwargs: Additional keyword arguments passed to
|
|
946
|
-
`DataFrame.write_parquet`.
|
|
947
|
-
|
|
948
|
-
Returns:
|
|
949
|
-
None
|
|
950
|
-
"""
|
|
951
|
-
buffer = BytesIO()
|
|
952
|
-
df.write_parquet(file=buffer, **kwargs)
|
|
953
|
-
self.write_blob_as_bytes(blob_name=blob_name, data=buffer.getvalue())
|
|
954
|
-
|
|
955
1037
|
def sink_parquet_blob(
|
|
956
1038
|
self, blob_name: str, lf: pl.LazyFrame, **kwargs: Dict
|
|
957
1039
|
) -> None:
|
|
@@ -1011,6 +1093,81 @@ class StorageHandler:
|
|
|
1011
1093
|
progress_hook=progress_hook,
|
|
1012
1094
|
)
|
|
1013
1095
|
|
|
1096
|
+
def upload_dir(
|
|
1097
|
+
self,
|
|
1098
|
+
local_dir: Union[str, Path],
|
|
1099
|
+
blob_prefix: str = "",
|
|
1100
|
+
progress_hook: Optional[Callable] = None,
|
|
1101
|
+
) -> List[str]:
|
|
1102
|
+
"""Upload all files from a local directory to storage.
|
|
1103
|
+
|
|
1104
|
+
Recursively walks ``local_dir`` and uploads every file,
|
|
1105
|
+
preserving the relative directory structure beneath
|
|
1106
|
+
``blob_prefix`` in storage.
|
|
1107
|
+
|
|
1108
|
+
Args:
|
|
1109
|
+
local_dir (Union[str, Path]): The local directory to
|
|
1110
|
+
upload recursively.
|
|
1111
|
+
blob_prefix (str): The destination prefix in storage.
|
|
1112
|
+
Defaults to empty (root of container).
|
|
1113
|
+
progress_hook (Optional[Callable]): Optional callback
|
|
1114
|
+
invoked during each file upload with
|
|
1115
|
+
``(blob_name, bytes_uploaded, total_bytes)``.
|
|
1116
|
+
|
|
1117
|
+
Returns:
|
|
1118
|
+
List[str]: Names of all uploaded blobs.
|
|
1119
|
+
|
|
1120
|
+
Raises:
|
|
1121
|
+
FileNotFoundError: If ``local_dir`` does not exist or
|
|
1122
|
+
contains no files.
|
|
1123
|
+
"""
|
|
1124
|
+
local_path = Path(local_dir)
|
|
1125
|
+
|
|
1126
|
+
if not local_path.is_dir():
|
|
1127
|
+
raise FileNotFoundError(f"Directory not found: {local_path}")
|
|
1128
|
+
|
|
1129
|
+
files = sorted(f for f in local_path.rglob("*") if f.is_file())
|
|
1130
|
+
|
|
1131
|
+
if not files:
|
|
1132
|
+
raise FileNotFoundError(f"No files found in: {local_path}")
|
|
1133
|
+
|
|
1134
|
+
prefix = blob_prefix.rstrip("/")
|
|
1135
|
+
uploaded_blobs: List[str] = []
|
|
1136
|
+
|
|
1137
|
+
for file_path in files:
|
|
1138
|
+
rel = file_path.relative_to(local_path)
|
|
1139
|
+
blob_name = f"{prefix}/{rel}".lstrip("/") if prefix else str(rel)
|
|
1140
|
+
|
|
1141
|
+
total_size = file_path.stat().st_size
|
|
1142
|
+
|
|
1143
|
+
def make_hook(name: str, total: int) -> Callable:
|
|
1144
|
+
def hook(current: int, _total: int) -> None:
|
|
1145
|
+
if progress_hook:
|
|
1146
|
+
progress_hook(name, current, total)
|
|
1147
|
+
|
|
1148
|
+
return hook
|
|
1149
|
+
|
|
1150
|
+
with open(file_path, "rb") as in_file:
|
|
1151
|
+
if self.local:
|
|
1152
|
+
dest = self.base_path / blob_name
|
|
1153
|
+
dest.parent.mkdir(parents=True, exist_ok=True)
|
|
1154
|
+
dest.write_bytes(in_file.read())
|
|
1155
|
+
if progress_hook:
|
|
1156
|
+
progress_hook(blob_name, total_size, total_size)
|
|
1157
|
+
else:
|
|
1158
|
+
blob_client = self._get_blob_client(blob_name=blob_name)
|
|
1159
|
+
blob_client.upload_blob(
|
|
1160
|
+
data=in_file,
|
|
1161
|
+
blob_type="BlockBlob",
|
|
1162
|
+
overwrite=True,
|
|
1163
|
+
length=total_size,
|
|
1164
|
+
progress_hook=make_hook(blob_name, total_size),
|
|
1165
|
+
)
|
|
1166
|
+
|
|
1167
|
+
uploaded_blobs.append(blob_name)
|
|
1168
|
+
|
|
1169
|
+
return uploaded_blobs
|
|
1170
|
+
|
|
1014
1171
|
def delete_blob(self, blob_name: str) -> List:
|
|
1015
1172
|
"""Delete a blob or prefix from storage.
|
|
1016
1173
|
|
|
@@ -686,7 +686,10 @@ def _cmd_download(args: argparse.Namespace) -> None:
|
|
|
686
686
|
handler = _get_handler(args=args)
|
|
687
687
|
|
|
688
688
|
if handler.local:
|
|
689
|
-
print(
|
|
689
|
+
print(
|
|
690
|
+
"Download is only available for blob storage and not "
|
|
691
|
+
"local storage. Skipping."
|
|
692
|
+
)
|
|
690
693
|
return
|
|
691
694
|
|
|
692
695
|
try:
|
|
@@ -706,6 +709,39 @@ def _cmd_download(args: argparse.Namespace) -> None:
|
|
|
706
709
|
_fail(f"File '{args.blob_name}' does not exist.")
|
|
707
710
|
|
|
708
711
|
|
|
712
|
+
def _cmd_download_dir(args: argparse.Namespace) -> None:
|
|
713
|
+
handler = _get_handler(args=args)
|
|
714
|
+
|
|
715
|
+
if handler.local:
|
|
716
|
+
print(
|
|
717
|
+
"Download is only available for blob storage and not "
|
|
718
|
+
"local storage. Skipping."
|
|
719
|
+
)
|
|
720
|
+
return
|
|
721
|
+
|
|
722
|
+
try:
|
|
723
|
+
with _progress_bar() as progress:
|
|
724
|
+
task = progress.add_task(
|
|
725
|
+
description=args.dir_prefix,
|
|
726
|
+
total=None,
|
|
727
|
+
)
|
|
728
|
+
downloaded = handler.download_dir(
|
|
729
|
+
dir_prefix=args.dir_prefix,
|
|
730
|
+
local_dir=args.local_dir,
|
|
731
|
+
progress_hook=lambda path, current, total: progress.update(
|
|
732
|
+
task_id=task,
|
|
733
|
+
description=str(path.name),
|
|
734
|
+
total=total,
|
|
735
|
+
completed=current,
|
|
736
|
+
),
|
|
737
|
+
)
|
|
738
|
+
print(f"Downloaded {len(downloaded)} file(s) from '{args.dir_prefix}':")
|
|
739
|
+
for path in downloaded:
|
|
740
|
+
print(f" {path}")
|
|
741
|
+
except FileNotFoundError:
|
|
742
|
+
_fail(f"Directory '{args.dir_prefix}' does not exist or is empty.")
|
|
743
|
+
|
|
744
|
+
|
|
709
745
|
def _cmd_upload(args: argparse.Namespace) -> None:
|
|
710
746
|
handler = _get_handler(args=args)
|
|
711
747
|
try:
|
|
@@ -726,6 +762,33 @@ def _cmd_upload(args: argparse.Namespace) -> None:
|
|
|
726
762
|
_fail(f"File '{args.file_path}' does not exist.")
|
|
727
763
|
|
|
728
764
|
|
|
765
|
+
def _cmd_upload_dir(args: argparse.Namespace) -> None:
|
|
766
|
+
handler = _get_handler(args=args)
|
|
767
|
+
|
|
768
|
+
try:
|
|
769
|
+
with _progress_bar() as progress:
|
|
770
|
+
task = progress.add_task(
|
|
771
|
+
description=args.local_dir,
|
|
772
|
+
total=None,
|
|
773
|
+
)
|
|
774
|
+
uploaded = handler.upload_dir(
|
|
775
|
+
local_dir=args.local_dir,
|
|
776
|
+
blob_prefix=args.blob_prefix,
|
|
777
|
+
progress_hook=lambda name, current, total: progress.update(
|
|
778
|
+
task_id=task,
|
|
779
|
+
description=name,
|
|
780
|
+
total=total,
|
|
781
|
+
completed=current,
|
|
782
|
+
),
|
|
783
|
+
)
|
|
784
|
+
prefix_label = f"{args.blob_prefix}/" if args.blob_prefix else "(root)"
|
|
785
|
+
print(f"Uploaded {len(uploaded)} file(s) to '{prefix_label}':")
|
|
786
|
+
for blob_name in uploaded:
|
|
787
|
+
print(f" {blob_name}")
|
|
788
|
+
except FileNotFoundError:
|
|
789
|
+
_fail(f"Directory '{args.local_dir}' does not exist or is empty.")
|
|
790
|
+
|
|
791
|
+
|
|
729
792
|
def _cmd_delete(args: argparse.Namespace) -> None:
|
|
730
793
|
handler = _get_handler(args=args)
|
|
731
794
|
|
|
@@ -849,6 +912,26 @@ def main() -> None:
|
|
|
849
912
|
)
|
|
850
913
|
parser_download.set_defaults(func=_cmd_download)
|
|
851
914
|
|
|
915
|
+
# --- download-dir ---
|
|
916
|
+
parser_download_dir = subparsers.add_parser(
|
|
917
|
+
"download-dir",
|
|
918
|
+
parents=[parent],
|
|
919
|
+
help="Download all blobs under a prefix to a local directory",
|
|
920
|
+
)
|
|
921
|
+
parser_download_dir.add_argument(
|
|
922
|
+
"dir_prefix",
|
|
923
|
+
nargs="?",
|
|
924
|
+
default="",
|
|
925
|
+
help="Storage path/prefix to download recursively "
|
|
926
|
+
"(default: empty = entire container)",
|
|
927
|
+
)
|
|
928
|
+
parser_download_dir.add_argument(
|
|
929
|
+
"--local-dir",
|
|
930
|
+
default=None,
|
|
931
|
+
help="Local destination directory (default: ./data)",
|
|
932
|
+
)
|
|
933
|
+
parser_download_dir.set_defaults(func=_cmd_download_dir)
|
|
934
|
+
|
|
852
935
|
# --- upload ---
|
|
853
936
|
parser_upload = subparsers.add_parser(
|
|
854
937
|
"upload",
|
|
@@ -865,6 +948,23 @@ def main() -> None:
|
|
|
865
948
|
)
|
|
866
949
|
parser_upload.set_defaults(func=_cmd_upload)
|
|
867
950
|
|
|
951
|
+
# --- upload-dir ---
|
|
952
|
+
parser_upload_dir = subparsers.add_parser(
|
|
953
|
+
"upload-dir",
|
|
954
|
+
parents=[parent],
|
|
955
|
+
help="Upload a local directory recursively to a container",
|
|
956
|
+
)
|
|
957
|
+
parser_upload_dir.add_argument(
|
|
958
|
+
"local_dir",
|
|
959
|
+
help="Path to the local directory to upload",
|
|
960
|
+
)
|
|
961
|
+
parser_upload_dir.add_argument(
|
|
962
|
+
"--blob-prefix",
|
|
963
|
+
default="",
|
|
964
|
+
help="Destination prefix in the container (default: root)",
|
|
965
|
+
)
|
|
966
|
+
parser_upload_dir.set_defaults(func=_cmd_upload_dir)
|
|
967
|
+
|
|
868
968
|
# --- rm ---
|
|
869
969
|
parser_delete = subparsers.add_parser(
|
|
870
970
|
"rm",
|
|
File without changes
|
|
File without changes
|
|
File without changes
|