KegStorage 0.20261119.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,10 @@
1
+ from keg_storage.version import VERSION as __version__ # noqa
2
+ from importlib.util import find_spec as _find_spec
3
+
4
+ if _find_spec('flask'):
5
+ from keg_storage.plugin import ( # noqa: F401
6
+ LinkViewMixin,
7
+ Storage,
8
+ StorageOperations,
9
+ )
10
+ from keg_storage.backends import * # noqa
@@ -0,0 +1,47 @@
1
+ from importlib.util import find_spec
2
+
3
+ from .base import (
4
+ FileMode,
5
+ FileNotFoundInStorageError,
6
+ InternalLinksStorageBackend,
7
+ InternalLinkTokenData,
8
+ ProgressCallback,
9
+ RemoteFile,
10
+ ShareLinkOperation,
11
+ StorageBackend,
12
+ )
13
+ from .filesystem import (
14
+ LocalFSError,
15
+ LocalFSStorage,
16
+ )
17
+
18
+
19
+ __all__ = [
20
+ 'FileMode',
21
+ 'FileNotFoundInStorageError',
22
+ 'ProgressCallback',
23
+ 'RemoteFile',
24
+ 'StorageBackend',
25
+ 'base',
26
+ 'filesystem',
27
+ 'LocalFSStorage',
28
+ 'LocalFSError',
29
+ 'InternalLinksStorageBackend',
30
+ 'InternalLinkTokenData',
31
+ 'ShareLinkOperation',
32
+ ]
33
+
34
+ if find_spec('boto3') is not None:
35
+ from .s3 import S3Storage # noqa
36
+
37
+ __all__.extend(['s3', 'S3Storage'])
38
+
39
+ if find_spec('paramiko') is not None:
40
+ from .sftp import SFTPStorage # noqa
41
+
42
+ __all__.extend(['sftp', 'SFTPStorage'])
43
+
44
+ if find_spec('azure') is not None:
45
+ from .azure import AzureStorage # noqa
46
+
47
+ __all__.extend(['azure', 'AzureStorage'])
@@ -0,0 +1,355 @@
1
+ import base64
2
+ from datetime import datetime
3
+ import os
4
+ from typing import ClassVar
5
+ import urllib.parse
6
+ import warnings
7
+
8
+ import arrow
9
+ from azure.storage.blob import (
10
+ BlobBlock,
11
+ BlobClient,
12
+ BlobPrefix,
13
+ BlobSasPermissions,
14
+ BlobServiceClient,
15
+ ContainerClient,
16
+ generate_blob_sas,
17
+ generate_container_sas,
18
+ )
19
+
20
+ from keg_storage.backends import base
21
+
22
+
23
+ DEFAULT_CHUNK_SIZE = 5 * 1024 * 1024
24
+
25
+
26
+ class AzureFile(base.RemoteFile):
27
+ """
28
+ Base class for Azure file interface. Since read and write operations are very different and
29
+ integrating the two would introduce a lot of complexity there are distinct subclasses for files
30
+ opened for reading and writing.
31
+ """
32
+
33
+ def __init__(self, mode: base.FileMode, blob_client: BlobClient, chunk_size=DEFAULT_CHUNK_SIZE):
34
+ """
35
+ :param mode: file mode
36
+ :param blob_client: blob client instance to use for API calls
37
+ :param chunk_size: read/write buffer size
38
+ """
39
+ super().__init__(mode)
40
+ self.chunk_size = chunk_size
41
+ self.client = blob_client
42
+
43
+ # Local buffer to reduce number of requests
44
+ self.buffer = bytearray()
45
+
46
+
47
+ class AzureWriter(AzureFile):
48
+ """
49
+ We are using Azure Block Blobs for all operations. The process for writing them is substantially
50
+ similar to that of S3 with a couple of differences.
51
+
52
+ 1. We generate the IDs for the blocks
53
+ 2. There is no separate call to instantiate the upload. The first call to put_block will create
54
+ the blob.
55
+
56
+ """
57
+
58
+ max_block_size: ClassVar[int] = 100 * 1024 * 1024
59
+
60
+ def __init__(
61
+ self,
62
+ mode: base.FileMode,
63
+ blob_client: BlobClient,
64
+ chunk_size=DEFAULT_CHUNK_SIZE,
65
+ ):
66
+ if chunk_size is not None:
67
+ # chunk_size cannot be larger than max_block_size due to API restrictions
68
+ chunk_size = min(chunk_size, self.max_block_size)
69
+ super().__init__(
70
+ mode=mode,
71
+ blob_client=blob_client,
72
+ chunk_size=chunk_size,
73
+ )
74
+ self.blocks: list[BlobBlock] = []
75
+
76
+ def _gen_block_id(self) -> str:
77
+ """
78
+ Generate a unique ID for the block. This is meant to be opaque but it is generated from:
79
+ 1. The index of the block as an 64 bit unsigned big endian integer
80
+ 2. 40 bytes of random data
81
+ The two parts are concatenated and base64 encoded giving us 64 bytes which is the maximum
82
+ Azure allows.
83
+ """
84
+ index_part = len(self.blocks).to_bytes(8, byteorder='big', signed=False)
85
+ random_part = os.urandom(40)
86
+ return base64.b64encode(index_part + random_part).decode()
87
+
88
+ def _flush(self):
89
+ if len(self.buffer) == 0:
90
+ # If there is no buffered data, we don't need to do anything
91
+ return
92
+
93
+ # Upload at most chunk_size bytes to a new block
94
+ block_id = self._gen_block_id()
95
+ self.client.stage_block(block_id=block_id, data=bytes(self.buffer[: self.chunk_size]))
96
+
97
+ # Store the block_id to later concatenate when we close this file
98
+ self.blocks.append(BlobBlock(block_id=block_id))
99
+
100
+ # Cycle the buffer
101
+ self.buffer = self.buffer[self.chunk_size :]
102
+
103
+ def _finalize(self):
104
+ self.client.commit_block_list(block_list=self.blocks)
105
+ self.blocks = []
106
+
107
+ def write(self, data: bytes) -> None:
108
+ self.buffer.extend(data)
109
+ while len(self.buffer) >= self.chunk_size:
110
+ # Write may be bigger than the chunk size and _flush() only uploads a single chunk so
111
+ # repeated calls may be necessary
112
+ self._flush()
113
+
114
+ def close(self):
115
+ self._flush()
116
+ if self.blocks:
117
+ # If we haven't created any blocks, we don't need to finalize
118
+ self._finalize()
119
+
120
+
121
+ class AzureReader(AzureFile):
122
+ """
123
+ The Azure reader uses byte ranged API calls to fill a local buffer to avoid lots of API overhead
124
+ for small read sizes.
125
+ """
126
+
127
+ def __init__(
128
+ self,
129
+ mode: base.FileMode,
130
+ blob_client: BlobClient,
131
+ chunk_size=DEFAULT_CHUNK_SIZE,
132
+ ):
133
+ super().__init__(
134
+ mode=mode,
135
+ blob_client=blob_client,
136
+ chunk_size=chunk_size,
137
+ )
138
+ self.stream = self.client.download_blob()
139
+ self.chunks = self.stream.chunks()
140
+
141
+ def _read_from_buffer(self, max_size):
142
+ """
143
+ Read up to max_size bytes from the local buffer.
144
+ """
145
+ read_size = min(len(self.buffer), max_size)
146
+ output = self.buffer[:read_size]
147
+ self.buffer = self.buffer[read_size:]
148
+ return output
149
+
150
+ def read(self, size: int) -> bytes:
151
+ output_buf = b''
152
+
153
+ while len(output_buf) < size:
154
+ if len(self.buffer) == 0:
155
+ try:
156
+ # Load the next chunk into the local buffer
157
+ next_chunk = next(self.chunks)
158
+ self.buffer.extend(next_chunk)
159
+ except StopIteration:
160
+ # All chunks have been consumed
161
+ break
162
+
163
+ read_remainder = size - len(output_buf)
164
+ output_buf += self._read_from_buffer(read_remainder)
165
+
166
+ return output_buf
167
+
168
+
169
+ class AzureStorage(base.StorageBackend):
170
+ account_url: str | None
171
+ container_url: str | None
172
+ blob_url: str | None
173
+
174
+ def __init__(
175
+ self,
176
+ account: str | None = None,
177
+ key: str | None = None,
178
+ bucket: str | None = None,
179
+ sas_container_url: str | None = None,
180
+ sas_blob_url: str | None = None,
181
+ chunk_size=DEFAULT_CHUNK_SIZE,
182
+ name: str = 'azure',
183
+ ):
184
+ super().__init__(name)
185
+ self.chunk_size = chunk_size
186
+
187
+ self.account = account
188
+ self.key = key
189
+ self.bucket = bucket
190
+
191
+ self.account_url = None
192
+ self.container_url = None
193
+ self.blob_url = None
194
+
195
+ if account and key and bucket:
196
+ self.account_url = f'https://{self.account}.blob.core.windows.net'
197
+ elif sas_container_url:
198
+ self.container_url = sas_container_url
199
+ elif sas_blob_url:
200
+ self.blob_url = sas_blob_url
201
+ else:
202
+ raise ValueError(
203
+ 'Must provide a sas_container_url, a sas_blob_url, '
204
+ 'or a combination of account, key, and bucket',
205
+ )
206
+
207
+ def _create_container_client(self) -> ContainerClient:
208
+ """Create a ContainerClient.
209
+
210
+ First see if a ``container_url`` was configured. Otherwise fall back to credentials.
211
+ """
212
+
213
+ if self.container_url:
214
+ return ContainerClient.from_container_url(self.container_url)
215
+
216
+ service_client = BlobServiceClient(account_url=self.account_url, credential=self.key)
217
+ return service_client.get_container_client(self.bucket)
218
+
219
+ def _create_blob_client(self, path: str) -> BlobClient:
220
+ """Create a BlobClient for the given path.
221
+
222
+ First see if a ``blob_url`` was configured. Otherwise fall back to ``container_url`` or
223
+ credentials.
224
+ """
225
+
226
+ if self.blob_url:
227
+ blob_client = BlobClient.from_blob_url(self.blob_url)
228
+ if blob_client.blob_name != path:
229
+ raise ValueError('Invalid path for the configured SAS blob URL')
230
+ return blob_client
231
+
232
+ container_client = self._create_container_client()
233
+ return container_client.get_blob_client(path)
234
+
235
+ def _clean_path(self, path: str):
236
+ return path.lstrip('/')
237
+
238
+ def list(self, path: str) -> list[base.ListEntry]:
239
+ if self.blob_url:
240
+ raise ValueError('Cannot perform list operation when configured with SAS blob URL')
241
+ client = self._create_container_client()
242
+
243
+ if not path.endswith('/'):
244
+ path = path + '/'
245
+ path = self._clean_path(path)
246
+ list_iter = client.walk_blobs(path)
247
+
248
+ def construct_entry(blob):
249
+ if isinstance(blob, BlobPrefix):
250
+ return base.ListEntry(name=blob.name, last_modified=None, size=0)
251
+ return base.ListEntry(
252
+ name=blob.name,
253
+ last_modified=arrow.get(blob.last_modified),
254
+ size=blob.size,
255
+ )
256
+
257
+ return [construct_entry(blob) for blob in list_iter]
258
+
259
+ def open(self, path: str, mode: base.FileMode | str, extra_args=None) -> AzureFile:
260
+ mode = base.FileMode.as_mode(mode)
261
+
262
+ path = self._clean_path(path)
263
+ blob_client = self._create_blob_client(path)
264
+
265
+ if (mode & base.FileMode.read) and (mode & base.FileMode.write):
266
+ raise NotImplementedError('Read+write mode not supported by the Azure backend')
267
+ elif mode & base.FileMode.write:
268
+ return AzureWriter(mode=mode, blob_client=blob_client, chunk_size=self.chunk_size)
269
+ elif mode & base.FileMode.read:
270
+ return AzureReader(mode=mode, blob_client=blob_client, chunk_size=self.chunk_size)
271
+ else:
272
+ raise ValueError('Unsupported mode. Accepted modes are FileMode.read or FileMode.write')
273
+
274
+ def delete(self, path: str):
275
+ path = self._clean_path(path)
276
+ blob_client = self._create_blob_client(path)
277
+ blob_client.delete_blob()
278
+
279
+ def create_upload_url(self, path: str, expire: arrow.Arrow | datetime):
280
+ """
281
+ Create an SAS URL that can be used to upload a blob without any additional authentication.
282
+ This url can be used in following way to authenticate a client and upload to the
283
+ pre-specified path:
284
+
285
+ client = BlobClient.from_blob_url(url)
286
+ client.upload_blob(data)
287
+ """
288
+ warnings.warn('create_upload_url is deprecated. Use link_to instead', DeprecationWarning)
289
+ return self.link_to(path=path, operation=base.ShareLinkOperation.upload, expire=expire)
290
+
291
+ def create_download_url(self, path: str, expire: arrow.Arrow | datetime):
292
+ """
293
+ Create an SAS URL that can be used to download a blob without any additional authentication.
294
+ This url may be accessed directly to download the blob:
295
+
296
+ requests.get(url)
297
+ """
298
+ warnings.warn('create_download_url is deprecated. Use link_to instead', DeprecationWarning)
299
+ return self.link_to(path=path, operation=base.ShareLinkOperation.download, expire=expire)
300
+
301
+ def link_to(
302
+ self,
303
+ path: str,
304
+ operation: base.ShareLinkOperation | str,
305
+ expire: arrow.Arrow | datetime,
306
+ output_path: str | None = None,
307
+ content_type: str | None = None,
308
+ ) -> str:
309
+ if not self.account_url or not self.key:
310
+ raise ValueError('Cannot create a SAS URL without account credentials')
311
+ path = self._clean_path(path)
312
+ expire = expire.datetime if isinstance(expire, arrow.Arrow) else expire
313
+
314
+ operation = base.ShareLinkOperation.as_operation(operation)
315
+ perms = BlobSasPermissions(
316
+ read=operation & base.ShareLinkOperation.download,
317
+ add=False, # only useful for append blobs that are not supported
318
+ create=operation & base.ShareLinkOperation.upload,
319
+ write=operation & base.ShareLinkOperation.upload,
320
+ delete=operation & base.ShareLinkOperation.remove,
321
+ tag=False,
322
+ )
323
+
324
+ token = generate_blob_sas(
325
+ account_name=self.account,
326
+ container_name=self.bucket,
327
+ blob_name=path,
328
+ account_key=self.key,
329
+ permission=perms,
330
+ expiry=expire,
331
+ content_disposition=f'attachment;filename={output_path}' if output_path else None,
332
+ content_type=content_type,
333
+ )
334
+ escaped_path = urllib.parse.quote(path, safe='')
335
+ url = urllib.parse.urljoin(self.account_url, f'{self.bucket}/{escaped_path}')
336
+ return f'{url}?{token}'
337
+
338
+ def create_container_url(
339
+ self,
340
+ expire: arrow.Arrow | datetime,
341
+ ip: str | None = None,
342
+ ):
343
+ if not self.account_url:
344
+ raise ValueError('Cannot create a SAS URL without account credentials')
345
+ expire = expire.datetime if isinstance(expire, arrow.Arrow) else expire
346
+ token = generate_container_sas(
347
+ account_name=self.account,
348
+ container_name=self.bucket,
349
+ account_key=self.key,
350
+ permission='rwdl',
351
+ expiry=expire,
352
+ ip=ip,
353
+ )
354
+ url = urllib.parse.urljoin(self.account_url, self.bucket)
355
+ return f'{url}?{token}'