kiarina-lib-google-cloud-storage 1.5.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- kiarina/lib/google/cloud_storage/__init__.py +43 -0
- kiarina/lib/google/cloud_storage/_get_blob.py +24 -0
- kiarina/lib/google/cloud_storage/_get_bucket.py +16 -0
- kiarina/lib/google/cloud_storage/_get_storage_client.py +12 -0
- kiarina/lib/google/cloud_storage/py.typed +0 -0
- kiarina/lib/google/cloud_storage/settings.py +15 -0
- kiarina_lib_google_cloud_storage-1.5.0.dist-info/METADATA +719 -0
- kiarina_lib_google_cloud_storage-1.5.0.dist-info/RECORD +9 -0
- kiarina_lib_google_cloud_storage-1.5.0.dist-info/WHEEL +4 -0
@@ -0,0 +1,43 @@
|
|
1
|
+
# __init__.py
|
2
|
+
import logging
|
3
|
+
from importlib import import_module
|
4
|
+
from importlib.metadata import version
|
5
|
+
from typing import TYPE_CHECKING
|
6
|
+
|
7
|
+
if TYPE_CHECKING:
|
8
|
+
from ._get_blob import get_blob
|
9
|
+
from ._get_bucket import get_bucket
|
10
|
+
from ._get_storage_client import get_storage_client
|
11
|
+
from .settings import GoogleCloudStorageSettings, settings_manager
|
12
|
+
|
13
|
+
__version__ = version("kiarina-lib-google-cloud-storage")
|
14
|
+
|
15
|
+
__all__ = [
|
16
|
+
# ._helpers
|
17
|
+
"get_blob",
|
18
|
+
"get_bucket",
|
19
|
+
"get_storage_client",
|
20
|
+
# .settings
|
21
|
+
"GoogleCloudStorageSettings",
|
22
|
+
"settings_manager",
|
23
|
+
]
|
24
|
+
|
25
|
+
logging.getLogger(__name__).addHandler(logging.NullHandler())
|
26
|
+
|
27
|
+
|
28
|
+
def __getattr__(name: str) -> object:
|
29
|
+
if name not in __all__:
|
30
|
+
raise AttributeError(f"module '{__name__}' has no attribute '{name}'")
|
31
|
+
|
32
|
+
module_map = {
|
33
|
+
# ._helpers
|
34
|
+
"get_blob": "._get_blob",
|
35
|
+
"get_bucket": "._get_bucket",
|
36
|
+
"get_storage_client": "._get_storage_client",
|
37
|
+
# .settings
|
38
|
+
"GoogleCloudStorageSettings": ".settings",
|
39
|
+
"settings_manager": ".settings",
|
40
|
+
}
|
41
|
+
|
42
|
+
globals()[name] = getattr(import_module(module_map[name], __name__), name)
|
43
|
+
return globals()[name]
|
@@ -0,0 +1,24 @@
|
|
1
|
+
from typing import Any
|
2
|
+
|
3
|
+
from google.cloud import storage # type: ignore[import-untyped]
|
4
|
+
|
5
|
+
from ._get_bucket import get_bucket
|
6
|
+
from .settings import settings_manager
|
7
|
+
|
8
|
+
|
9
|
+
def get_blob(
|
10
|
+
config_key: str | None = None, blob_name: str | None = None, **kwargs: Any
|
11
|
+
) -> storage.Blob:
|
12
|
+
settings = settings_manager.get_settings_by_key(config_key)
|
13
|
+
|
14
|
+
if blob_name is None and settings.blob_name is None:
|
15
|
+
raise ValueError("blob_name is not set in the settings and not provided")
|
16
|
+
|
17
|
+
if blob_name is None:
|
18
|
+
blob_name = settings.blob_name
|
19
|
+
|
20
|
+
if settings.blob_name_prefix:
|
21
|
+
blob_name = f"{settings.blob_name_prefix}/{blob_name}"
|
22
|
+
|
23
|
+
bucket = get_bucket(config_key, **kwargs)
|
24
|
+
return bucket.blob(blob_name)
|
@@ -0,0 +1,16 @@
|
|
1
|
+
from typing import Any
|
2
|
+
|
3
|
+
from google.cloud import storage # type: ignore[import-untyped]
|
4
|
+
|
5
|
+
from ._get_storage_client import get_storage_client
|
6
|
+
from .settings import settings_manager
|
7
|
+
|
8
|
+
|
9
|
+
def get_bucket(config_key: str | None = None, **kwargs: Any) -> storage.Bucket:
|
10
|
+
settings = settings_manager.get_settings_by_key(config_key)
|
11
|
+
|
12
|
+
if settings.bucket_name is None:
|
13
|
+
raise ValueError("bucket_name is not set in the settings")
|
14
|
+
|
15
|
+
client = get_storage_client(config_key, **kwargs)
|
16
|
+
return client.bucket(settings.bucket_name)
|
@@ -0,0 +1,12 @@
|
|
1
|
+
from typing import Any
|
2
|
+
|
3
|
+
from google.cloud import storage # type: ignore[import-untyped]
|
4
|
+
from kiarina.lib.google.auth import get_credentials
|
5
|
+
|
6
|
+
from .settings import settings_manager
|
7
|
+
|
8
|
+
|
9
|
+
def get_storage_client(config_key: str | None = None, **kwargs: Any) -> storage.Client:
|
10
|
+
settings = settings_manager.get_settings_by_key(config_key)
|
11
|
+
credentials = get_credentials(settings.google_auth_config_key)
|
12
|
+
return storage.Client(credentials=credentials, **kwargs)
|
File without changes
|
@@ -0,0 +1,15 @@
|
|
1
|
+
from pydantic_settings import BaseSettings
|
2
|
+
from pydantic_settings_manager import SettingsManager
|
3
|
+
|
4
|
+
|
5
|
+
class GoogleCloudStorageSettings(BaseSettings):
|
6
|
+
google_auth_config_key: str | None = None
|
7
|
+
|
8
|
+
bucket_name: str | None = None
|
9
|
+
|
10
|
+
blob_name_prefix: str | None = None
|
11
|
+
|
12
|
+
blob_name: str | None = None
|
13
|
+
|
14
|
+
|
15
|
+
settings_manager = SettingsManager(GoogleCloudStorageSettings, multi=True)
|
@@ -0,0 +1,719 @@
|
|
1
|
+
Metadata-Version: 2.4
|
2
|
+
Name: kiarina-lib-google-cloud-storage
|
3
|
+
Version: 1.5.0
|
4
|
+
Summary: Google Cloud Storage client library for kiarina namespace
|
5
|
+
Project-URL: Homepage, https://github.com/kiarina/kiarina-python
|
6
|
+
Project-URL: Repository, https://github.com/kiarina/kiarina-python
|
7
|
+
Project-URL: Issues, https://github.com/kiarina/kiarina-python/issues
|
8
|
+
Project-URL: Changelog, https://github.com/kiarina/kiarina-python/blob/main/packages/kiarina-lib-google-cloud-storage/CHANGELOG.md
|
9
|
+
Project-URL: Documentation, https://github.com/kiarina/kiarina-python/tree/main/packages/kiarina-lib-google-cloud-storage#readme
|
10
|
+
Author-email: kiarina <kiarinadawa@gmail.com>
|
11
|
+
Maintainer-email: kiarina <kiarinadawa@gmail.com>
|
12
|
+
License-Expression: MIT
|
13
|
+
Keywords: client,cloud,gcp,gcs,google,google-cloud,pydantic,settings,storage
|
14
|
+
Classifier: Development Status :: 4 - Beta
|
15
|
+
Classifier: Intended Audience :: Developers
|
16
|
+
Classifier: Operating System :: OS Independent
|
17
|
+
Classifier: Programming Language :: Python :: 3
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
19
|
+
Classifier: Programming Language :: Python :: 3.13
|
20
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
21
|
+
Classifier: Typing :: Typed
|
22
|
+
Requires-Python: >=3.12
|
23
|
+
Requires-Dist: google-cloud-storage>=2.19.0
|
24
|
+
Requires-Dist: kiarina-lib-google-auth>=1.4.0
|
25
|
+
Requires-Dist: pydantic-settings-manager>=2.1.0
|
26
|
+
Requires-Dist: pydantic-settings>=2.10.1
|
27
|
+
Description-Content-Type: text/markdown
|
28
|
+
|
29
|
+
# kiarina-lib-google-cloud-storage
|
30
|
+
|
31
|
+
A Python client library for Google Cloud Storage that separates infrastructure configuration from application logic.
|
32
|
+
|
33
|
+
## Design Philosophy
|
34
|
+
|
35
|
+
This library follows the principle of **separating infrastructure concerns from application logic**.
|
36
|
+
|
37
|
+
### The Problem
|
38
|
+
|
39
|
+
When using Google Cloud Storage directly, application code becomes tightly coupled with infrastructure details:
|
40
|
+
|
41
|
+
```python
|
42
|
+
# ❌ Infrastructure details leak into application code
|
43
|
+
from google.cloud import storage
|
44
|
+
from google.oauth2 import service_account
|
45
|
+
|
46
|
+
# Hard-coded credentials and bucket configuration
|
47
|
+
credentials = service_account.Credentials.from_service_account_file(
|
48
|
+
'/path/to/service-account-key.json'
|
49
|
+
)
|
50
|
+
client = storage.Client(credentials=credentials)
|
51
|
+
bucket = client.bucket("prod-us-west1-app-data") # Hard-coded bucket name
|
52
|
+
blob = bucket.blob("v2/users/data.json") # Hard-coded path structure
|
53
|
+
```
|
54
|
+
|
55
|
+
**Problems with this approach:**
|
56
|
+
- Environment-specific details are scattered throughout the codebase
|
57
|
+
- Difficult to test (requires mocking or actual GCS access)
|
58
|
+
- Hard to support multiple environments (dev, staging, production)
|
59
|
+
- Challenging to implement multi-tenancy
|
60
|
+
- Credentials management is error-prone
|
61
|
+
|
62
|
+
### The Solution
|
63
|
+
|
64
|
+
This library externalizes all infrastructure configuration:
|
65
|
+
|
66
|
+
```python
|
67
|
+
# ✅ Application code only knows logical names
|
68
|
+
from kiarina.lib.google.cloud_storage import get_blob
|
69
|
+
|
70
|
+
# All infrastructure details are managed externally
|
71
|
+
blob = get_blob(blob_name="user_data.json")
|
72
|
+
blob.upload_from_string(json.dumps(data))
|
73
|
+
```
|
74
|
+
|
75
|
+
**Benefits:**
|
76
|
+
- **Environment-agnostic**: Same code works in dev, staging, and production
|
77
|
+
- **Testable**: Easy to inject test configurations
|
78
|
+
- **Multi-tenant ready**: Different configurations for different tenants
|
79
|
+
- **Secure**: Credentials managed through kiarina-lib-google-auth
|
80
|
+
- **Maintainable**: Infrastructure changes don't require code changes
|
81
|
+
|
82
|
+
## Features
|
83
|
+
|
84
|
+
- **Configuration Management**: Use `pydantic-settings-manager` for flexible configuration
|
85
|
+
- **Type Safety**: Full type hints and Pydantic validation
|
86
|
+
- **Integration with kiarina-lib-google-auth**: Seamless authentication
|
87
|
+
- **Multiple Configurations**: Support for multiple named configurations
|
88
|
+
- **Environment Variable Support**: Configure via environment variables
|
89
|
+
- **Blob Name Prefix**: Organize blobs with configurable prefixes
|
90
|
+
- **Native API Access**: Returns native `google-cloud-storage` objects for full API access
|
91
|
+
|
92
|
+
## Installation
|
93
|
+
|
94
|
+
```bash
|
95
|
+
pip install kiarina-lib-google-cloud-storage
|
96
|
+
```
|
97
|
+
|
98
|
+
## Quick Start
|
99
|
+
|
100
|
+
### Basic Usage
|
101
|
+
|
102
|
+
```python
|
103
|
+
from kiarina.lib.google.cloud_storage import get_blob, settings_manager
|
104
|
+
|
105
|
+
# Configure once (typically in your app initialization)
|
106
|
+
settings_manager.user_config = {
|
107
|
+
"default": {
|
108
|
+
"google_auth_config_key": "default",
|
109
|
+
"bucket_name": "my-app-data",
|
110
|
+
"blob_name_prefix": "production/v1"
|
111
|
+
}
|
112
|
+
}
|
113
|
+
|
114
|
+
# Application code - clean and simple
|
115
|
+
blob = get_blob(blob_name="user_data.json")
|
116
|
+
# Actual path: gs://my-app-data/production/v1/user_data.json
|
117
|
+
|
118
|
+
# Use native google-cloud-storage API
|
119
|
+
blob.upload_from_string("Hello, World!")
|
120
|
+
content = blob.download_as_text()
|
121
|
+
```
|
122
|
+
|
123
|
+
### Configuration via Environment Variables
|
124
|
+
|
125
|
+
```bash
|
126
|
+
# Set once in your deployment environment
|
127
|
+
export KIARINA_LIB_GOOGLE_CLOUD_STORAGE_BUCKET_NAME="my-app-data"
|
128
|
+
export KIARINA_LIB_GOOGLE_CLOUD_STORAGE_BLOB_NAME_PREFIX="production/v1"
|
129
|
+
export KIARINA_LIB_GOOGLE_CLOUD_STORAGE_GOOGLE_AUTH_CONFIG_KEY="default"
|
130
|
+
```
|
131
|
+
|
132
|
+
```python
|
133
|
+
# Application code - no configuration needed
|
134
|
+
from kiarina.lib.google.cloud_storage import get_blob
|
135
|
+
|
136
|
+
blob = get_blob(blob_name="user_data.json")
|
137
|
+
blob.upload_from_string("Hello, World!")
|
138
|
+
```
|
139
|
+
|
140
|
+
## Real-World Use Cases
|
141
|
+
|
142
|
+
### Use Case 1: Multi-Environment Deployment
|
143
|
+
|
144
|
+
Deploy the same application code to different environments with different configurations.
|
145
|
+
|
146
|
+
```yaml
|
147
|
+
# config/production.yaml
|
148
|
+
google_cloud_storage:
|
149
|
+
default:
|
150
|
+
google_auth_config_key: "production"
|
151
|
+
bucket_name: "prod-us-west1-app-data"
|
152
|
+
blob_name_prefix: "v2/production"
|
153
|
+
|
154
|
+
# config/staging.yaml
|
155
|
+
google_cloud_storage:
|
156
|
+
default:
|
157
|
+
google_auth_config_key: "staging"
|
158
|
+
bucket_name: "staging-app-data"
|
159
|
+
blob_name_prefix: "v2/staging"
|
160
|
+
|
161
|
+
# config/development.yaml
|
162
|
+
google_cloud_storage:
|
163
|
+
default:
|
164
|
+
google_auth_config_key: "development"
|
165
|
+
bucket_name: "dev-local-data"
|
166
|
+
blob_name_prefix: "v2/dev"
|
167
|
+
```
|
168
|
+
|
169
|
+
```python
|
170
|
+
# Application code (same for all environments)
|
171
|
+
from kiarina.lib.google.cloud_storage import get_blob
|
172
|
+
|
173
|
+
def save_user_profile(user_id: str, profile: dict):
|
174
|
+
"""Save user profile - works in any environment"""
|
175
|
+
blob = get_blob(blob_name=f"users/{user_id}/profile.json")
|
176
|
+
blob.upload_from_string(json.dumps(profile))
|
177
|
+
|
178
|
+
def load_user_profile(user_id: str) -> dict:
|
179
|
+
"""Load user profile - works in any environment"""
|
180
|
+
blob = get_blob(blob_name=f"users/{user_id}/profile.json")
|
181
|
+
return json.loads(blob.download_as_text())
|
182
|
+
```
|
183
|
+
|
184
|
+
**Result:**
|
185
|
+
- Production: `gs://prod-us-west1-app-data/v2/production/users/{user_id}/profile.json`
|
186
|
+
- Staging: `gs://staging-app-data/v2/staging/users/{user_id}/profile.json`
|
187
|
+
- Development: `gs://dev-local-data/v2/dev/users/{user_id}/profile.json`
|
188
|
+
|
189
|
+
### Use Case 2: Multi-Tenant Application
|
190
|
+
|
191
|
+
Support multiple tenants with isolated storage, without changing application code.
|
192
|
+
|
193
|
+
```python
|
194
|
+
from kiarina.lib.google.cloud_storage import settings_manager, get_blob
|
195
|
+
|
196
|
+
# Configure tenant-specific storage
|
197
|
+
settings_manager.user_config = {
|
198
|
+
"tenant_acme": {
|
199
|
+
"google_auth_config_key": "tenant_acme",
|
200
|
+
"bucket_name": "acme-corp-data",
|
201
|
+
"blob_name_prefix": "app-data"
|
202
|
+
},
|
203
|
+
"tenant_globex": {
|
204
|
+
"google_auth_config_key": "tenant_globex",
|
205
|
+
"bucket_name": "globex-data",
|
206
|
+
"blob_name_prefix": "app-data"
|
207
|
+
}
|
208
|
+
}
|
209
|
+
|
210
|
+
# Application code - tenant-agnostic
|
211
|
+
def save_document(tenant_id: str, doc_id: str, content: bytes):
|
212
|
+
"""Save document for any tenant"""
|
213
|
+
config_key = f"tenant_{tenant_id}"
|
214
|
+
blob = get_blob(config_key=config_key, blob_name=f"documents/{doc_id}.pdf")
|
215
|
+
blob.upload_from_string(content)
|
216
|
+
|
217
|
+
def list_documents(tenant_id: str) -> list[str]:
|
218
|
+
"""List documents for any tenant"""
|
219
|
+
from kiarina.lib.google.cloud_storage import get_bucket
|
220
|
+
|
221
|
+
config_key = f"tenant_{tenant_id}"
|
222
|
+
bucket = get_bucket(config_key=config_key)
|
223
|
+
|
224
|
+
# Get prefix from settings
|
225
|
+
settings = settings_manager.get_settings_by_key(config_key)
|
226
|
+
prefix = f"{settings.blob_name_prefix}/documents/" if settings.blob_name_prefix else "documents/"
|
227
|
+
|
228
|
+
blobs = bucket.list_blobs(prefix=prefix)
|
229
|
+
return [blob.name for blob in blobs]
|
230
|
+
```
|
231
|
+
|
232
|
+
**Result:**
|
233
|
+
- Tenant ACME: `gs://acme-corp-data/app-data/documents/{doc_id}.pdf`
|
234
|
+
- Tenant Globex: `gs://globex-data/app-data/documents/{doc_id}.pdf`
|
235
|
+
|
236
|
+
### Use Case 3: Testing
|
237
|
+
|
238
|
+
Write tests without touching real Google Cloud Storage.
|
239
|
+
|
240
|
+
```python
|
241
|
+
# tests/conftest.py
|
242
|
+
import pytest
|
243
|
+
from kiarina.lib.google.cloud_storage import settings_manager
|
244
|
+
|
245
|
+
@pytest.fixture
|
246
|
+
def mock_storage_config():
|
247
|
+
"""Configure test storage"""
|
248
|
+
settings_manager.user_config = {
|
249
|
+
"test": {
|
250
|
+
"google_auth_config_key": "test",
|
251
|
+
"bucket_name": "test-bucket",
|
252
|
+
"blob_name_prefix": f"test-run-{datetime.now().isoformat()}"
|
253
|
+
}
|
254
|
+
}
|
255
|
+
yield
|
256
|
+
# Cleanup test data if needed
|
257
|
+
|
258
|
+
# tests/test_user_service.py
|
259
|
+
def test_save_user_profile(mock_storage_config):
|
260
|
+
"""Test user profile saving"""
|
261
|
+
from myapp.services import save_user_profile
|
262
|
+
|
263
|
+
# Application code uses test configuration automatically
|
264
|
+
save_user_profile("user123", {"name": "Alice"})
|
265
|
+
|
266
|
+
# Verify using the same configuration
|
267
|
+
from kiarina.lib.google.cloud_storage import get_blob
|
268
|
+
blob = get_blob(blob_name="users/user123/profile.json")
|
269
|
+
assert blob.exists()
|
270
|
+
```
|
271
|
+
|
272
|
+
### Use Case 4: Debugging and Troubleshooting
|
273
|
+
|
274
|
+
Understand where your data is actually stored.
|
275
|
+
|
276
|
+
```python
|
277
|
+
from kiarina.lib.google.cloud_storage import settings_manager
|
278
|
+
|
279
|
+
def debug_storage_config(config_key: str | None = None):
|
280
|
+
"""Show actual storage paths for debugging"""
|
281
|
+
settings = settings_manager.get_settings_by_key(config_key)
|
282
|
+
|
283
|
+
print(f"Configuration: {config_key or 'default'}")
|
284
|
+
print(f" Bucket: {settings.bucket_name}")
|
285
|
+
print(f" Prefix: {settings.blob_name_prefix or '(none)'}")
|
286
|
+
print(f" Auth: {settings.google_auth_config_key}")
|
287
|
+
|
288
|
+
# Example paths
|
289
|
+
example_blob = "users/123/profile.json"
|
290
|
+
if settings.blob_name_prefix:
|
291
|
+
full_path = f"{settings.blob_name_prefix}/{example_blob}"
|
292
|
+
else:
|
293
|
+
full_path = example_blob
|
294
|
+
|
295
|
+
print(f" Example: gs://{settings.bucket_name}/{full_path}")
|
296
|
+
|
297
|
+
# Usage
|
298
|
+
debug_storage_config("production")
|
299
|
+
# Output:
|
300
|
+
# Configuration: production
|
301
|
+
# Bucket: prod-us-west1-app-data
|
302
|
+
# Prefix: v2/production
|
303
|
+
# Auth: production
|
304
|
+
# Example: gs://prod-us-west1-app-data/v2/production/users/123/profile.json
|
305
|
+
```
|
306
|
+
|
307
|
+
## Configuration
|
308
|
+
|
309
|
+
This library uses [pydantic-settings-manager](https://github.com/kiarina/pydantic-settings-manager) for flexible configuration management.
|
310
|
+
|
311
|
+
### GoogleCloudStorageSettings
|
312
|
+
|
313
|
+
| Field | Type | Required | Description |
|
314
|
+
|-------|------|----------|-------------|
|
315
|
+
| `google_auth_config_key` | `str \| None` | No | Configuration key for kiarina-lib-google-auth |
|
316
|
+
| `bucket_name` | `str \| None` | Yes* | Google Cloud Storage bucket name |
|
317
|
+
| `blob_name_prefix` | `str \| None` | No | Prefix for blob names (e.g., "production/v1") |
|
318
|
+
| `blob_name` | `str \| None` | No | Default blob name (rarely used) |
|
319
|
+
|
320
|
+
*Required when using `get_bucket()` or `get_blob()`
|
321
|
+
|
322
|
+
### Configuration Methods
|
323
|
+
|
324
|
+
#### 1. Programmatic Configuration
|
325
|
+
|
326
|
+
```python
|
327
|
+
from kiarina.lib.google.cloud_storage import settings_manager
|
328
|
+
|
329
|
+
settings_manager.user_config = {
|
330
|
+
"default": {
|
331
|
+
"google_auth_config_key": "default",
|
332
|
+
"bucket_name": "my-bucket",
|
333
|
+
"blob_name_prefix": "app-data"
|
334
|
+
}
|
335
|
+
}
|
336
|
+
```
|
337
|
+
|
338
|
+
#### 2. Environment Variables
|
339
|
+
|
340
|
+
All settings can be configured via environment variables with the `KIARINA_LIB_GOOGLE_CLOUD_STORAGE_` prefix:
|
341
|
+
|
342
|
+
```bash
|
343
|
+
export KIARINA_LIB_GOOGLE_CLOUD_STORAGE_GOOGLE_AUTH_CONFIG_KEY="default"
|
344
|
+
export KIARINA_LIB_GOOGLE_CLOUD_STORAGE_BUCKET_NAME="my-bucket"
|
345
|
+
export KIARINA_LIB_GOOGLE_CLOUD_STORAGE_BLOB_NAME_PREFIX="app-data"
|
346
|
+
```
|
347
|
+
|
348
|
+
#### 3. Configuration Files (with YAML)
|
349
|
+
|
350
|
+
```python
|
351
|
+
import yaml
|
352
|
+
from kiarina.lib.google.cloud_storage import settings_manager
|
353
|
+
|
354
|
+
# Load from YAML file
|
355
|
+
with open("config.yaml") as f:
|
356
|
+
config = yaml.safe_load(f)
|
357
|
+
|
358
|
+
settings_manager.user_config = config["google_cloud_storage"]
|
359
|
+
```
|
360
|
+
|
361
|
+
### Integration with kiarina-lib-google-auth
|
362
|
+
|
363
|
+
This library integrates seamlessly with [kiarina-lib-google-auth](../kiarina-lib-google-auth/) for authentication:
|
364
|
+
|
365
|
+
```python
|
366
|
+
from kiarina.lib.google.auth import settings_manager as auth_settings_manager
|
367
|
+
from kiarina.lib.google.cloud_storage import settings_manager as storage_settings_manager
|
368
|
+
|
369
|
+
# Configure authentication
|
370
|
+
auth_settings_manager.user_config = {
|
371
|
+
"default": {
|
372
|
+
"type": "service_account",
|
373
|
+
"service_account_file": "~/service-account-key.json"
|
374
|
+
}
|
375
|
+
}
|
376
|
+
|
377
|
+
# Configure storage (references auth config)
|
378
|
+
storage_settings_manager.user_config = {
|
379
|
+
"default": {
|
380
|
+
"google_auth_config_key": "default", # References auth config above
|
381
|
+
"bucket_name": "my-bucket"
|
382
|
+
}
|
383
|
+
}
|
384
|
+
|
385
|
+
# Use authenticated storage
|
386
|
+
from kiarina.lib.google.cloud_storage import get_bucket
|
387
|
+
bucket = get_bucket() # Automatically authenticated
|
388
|
+
```
|
389
|
+
|
390
|
+
## API Reference
|
391
|
+
|
392
|
+
### get_storage_client()
|
393
|
+
|
394
|
+
Get a Google Cloud Storage client with credentials from kiarina-lib-google-auth.
|
395
|
+
|
396
|
+
```python
|
397
|
+
def get_storage_client(
|
398
|
+
config_key: str | None = None,
|
399
|
+
**kwargs: Any
|
400
|
+
) -> storage.Client
|
401
|
+
```
|
402
|
+
|
403
|
+
**Parameters:**
|
404
|
+
- `config_key`: Configuration key to use (default: None uses active key)
|
405
|
+
- `**kwargs`: Additional arguments passed to `storage.Client()`
|
406
|
+
|
407
|
+
**Returns:**
|
408
|
+
- `storage.Client`: Authenticated Google Cloud Storage client
|
409
|
+
|
410
|
+
**Example:**
|
411
|
+
```python
|
412
|
+
client = get_storage_client()
|
413
|
+
client = get_storage_client(config_key="production")
|
414
|
+
client = get_storage_client(project="my-project") # Override project
|
415
|
+
```
|
416
|
+
|
417
|
+
### get_bucket()
|
418
|
+
|
419
|
+
Get a Google Cloud Storage bucket.
|
420
|
+
|
421
|
+
```python
|
422
|
+
def get_bucket(
|
423
|
+
config_key: str | None = None,
|
424
|
+
**kwargs: Any
|
425
|
+
) -> storage.Bucket
|
426
|
+
```
|
427
|
+
|
428
|
+
**Parameters:**
|
429
|
+
- `config_key`: Configuration key to use (default: None uses active key)
|
430
|
+
- `**kwargs`: Additional arguments passed to `get_storage_client()`
|
431
|
+
|
432
|
+
**Returns:**
|
433
|
+
- `storage.Bucket`: Google Cloud Storage bucket
|
434
|
+
|
435
|
+
**Raises:**
|
436
|
+
- `ValueError`: If `bucket_name` is not set in settings
|
437
|
+
|
438
|
+
**Example:**
|
439
|
+
```python
|
440
|
+
bucket = get_bucket()
|
441
|
+
bucket = get_bucket(config_key="production")
|
442
|
+
|
443
|
+
# Use native google-cloud-storage API
|
444
|
+
for blob in bucket.list_blobs(prefix="users/"):
|
445
|
+
print(blob.name)
|
446
|
+
```
|
447
|
+
|
448
|
+
### get_blob()
|
449
|
+
|
450
|
+
Get a Google Cloud Storage blob.
|
451
|
+
|
452
|
+
```python
|
453
|
+
def get_blob(
|
454
|
+
config_key: str | None = None,
|
455
|
+
blob_name: str | None = None,
|
456
|
+
**kwargs: Any
|
457
|
+
) -> storage.Blob
|
458
|
+
```
|
459
|
+
|
460
|
+
**Parameters:**
|
461
|
+
- `config_key`: Configuration key to use (default: None uses active key)
|
462
|
+
- `blob_name`: Blob name (default: None uses settings.blob_name)
|
463
|
+
- `**kwargs`: Additional arguments passed to `get_bucket()`
|
464
|
+
|
465
|
+
**Returns:**
|
466
|
+
- `storage.Blob`: Google Cloud Storage blob
|
467
|
+
|
468
|
+
**Raises:**
|
469
|
+
- `ValueError`: If `blob_name` is not provided and not set in settings
|
470
|
+
|
471
|
+
**Example:**
|
472
|
+
```python
|
473
|
+
# Basic usage
|
474
|
+
blob = get_blob(blob_name="data.json")
|
475
|
+
|
476
|
+
# With blob_name_prefix in settings
|
477
|
+
# If blob_name_prefix="production/v1" and blob_name="data.json"
|
478
|
+
# Actual blob name will be "production/v1/data.json"
|
479
|
+
blob = get_blob(blob_name="data.json")
|
480
|
+
|
481
|
+
# Use native google-cloud-storage API
|
482
|
+
blob.upload_from_string("content")
|
483
|
+
content = blob.download_as_text()
|
484
|
+
blob.delete()
|
485
|
+
```
|
486
|
+
|
487
|
+
### settings_manager
|
488
|
+
|
489
|
+
Global settings manager instance for Google Cloud Storage configuration.
|
490
|
+
|
491
|
+
```python
|
492
|
+
settings_manager: SettingsManager[GoogleCloudStorageSettings]
|
493
|
+
```
|
494
|
+
|
495
|
+
**Properties:**
|
496
|
+
- `settings`: Get the current active settings
|
497
|
+
- `user_config`: Get/set user configuration
|
498
|
+
- `active_key`: Get/set active configuration key
|
499
|
+
|
500
|
+
**Methods:**
|
501
|
+
- `get_settings_by_key(key: str)`: Get settings by specific key
|
502
|
+
- `clear()`: Clear cached settings
|
503
|
+
|
504
|
+
## Common Operations
|
505
|
+
|
506
|
+
### Upload and Download
|
507
|
+
|
508
|
+
```python
|
509
|
+
from kiarina.lib.google.cloud_storage import get_blob
|
510
|
+
|
511
|
+
# Upload from string
|
512
|
+
blob = get_blob(blob_name="data.json")
|
513
|
+
blob.upload_from_string(json.dumps({"key": "value"}))
|
514
|
+
|
515
|
+
# Upload from file
|
516
|
+
blob = get_blob(blob_name="document.pdf")
|
517
|
+
blob.upload_from_filename("/path/to/document.pdf")
|
518
|
+
|
519
|
+
# Download as string
|
520
|
+
blob = get_blob(blob_name="data.json")
|
521
|
+
content = blob.download_as_text()
|
522
|
+
data = json.loads(content)
|
523
|
+
|
524
|
+
# Download to file
|
525
|
+
blob = get_blob(blob_name="document.pdf")
|
526
|
+
blob.download_to_filename("/path/to/downloaded.pdf")
|
527
|
+
```
|
528
|
+
|
529
|
+
### List Blobs
|
530
|
+
|
531
|
+
```python
|
532
|
+
from kiarina.lib.google.cloud_storage import get_bucket, settings_manager
|
533
|
+
|
534
|
+
bucket = get_bucket()
|
535
|
+
|
536
|
+
# Get prefix from settings
|
537
|
+
settings = settings_manager.settings
|
538
|
+
prefix = f"{settings.blob_name_prefix}/users/" if settings.blob_name_prefix else "users/"
|
539
|
+
|
540
|
+
# List blobs
|
541
|
+
for blob in bucket.list_blobs(prefix=prefix):
|
542
|
+
print(f"{blob.name}: {blob.size} bytes")
|
543
|
+
```
|
544
|
+
|
545
|
+
### Check Existence
|
546
|
+
|
547
|
+
```python
|
548
|
+
from kiarina.lib.google.cloud_storage import get_blob
|
549
|
+
|
550
|
+
blob = get_blob(blob_name="data.json")
|
551
|
+
if blob.exists():
|
552
|
+
print("Blob exists")
|
553
|
+
print(f"Size: {blob.size} bytes")
|
554
|
+
print(f"Updated: {blob.updated}")
|
555
|
+
else:
|
556
|
+
print("Blob does not exist")
|
557
|
+
```
|
558
|
+
|
559
|
+
### Delete Blob
|
560
|
+
|
561
|
+
```python
|
562
|
+
from kiarina.lib.google.cloud_storage import get_blob
|
563
|
+
|
564
|
+
blob = get_blob(blob_name="old_data.json")
|
565
|
+
if blob.exists():
|
566
|
+
blob.delete()
|
567
|
+
print("Blob deleted")
|
568
|
+
```
|
569
|
+
|
570
|
+
### Copy Blob
|
571
|
+
|
572
|
+
```python
|
573
|
+
from kiarina.lib.google.cloud_storage import get_bucket
|
574
|
+
|
575
|
+
bucket = get_bucket()
|
576
|
+
source_blob = bucket.blob("source.json")
|
577
|
+
destination_blob = bucket.blob("backup/source.json")
|
578
|
+
|
579
|
+
bucket.copy_blob(source_blob, bucket, destination_blob.name)
|
580
|
+
print("Blob copied")
|
581
|
+
```
|
582
|
+
|
583
|
+
## Error Handling
|
584
|
+
|
585
|
+
```python
|
586
|
+
from google.cloud import storage
|
587
|
+
from google.api_core import exceptions
|
588
|
+
from kiarina.lib.google.cloud_storage import get_blob
|
589
|
+
|
590
|
+
try:
|
591
|
+
blob = get_blob(blob_name="data.json")
|
592
|
+
content = blob.download_as_text()
|
593
|
+
except exceptions.NotFound:
|
594
|
+
print("Blob not found")
|
595
|
+
except exceptions.Forbidden:
|
596
|
+
print("Permission denied")
|
597
|
+
except exceptions.GoogleAPIError as e:
|
598
|
+
print(f"Google API error: {e}")
|
599
|
+
except ValueError as e:
|
600
|
+
print(f"Configuration error: {e}")
|
601
|
+
```
|
602
|
+
|
603
|
+
## Best Practices
|
604
|
+
|
605
|
+
### 1. Configure Once, Use Everywhere
|
606
|
+
|
607
|
+
```python
|
608
|
+
# app/config.py
|
609
|
+
from kiarina.lib.google.cloud_storage import settings_manager
|
610
|
+
import yaml
|
611
|
+
|
612
|
+
def init_storage_config():
|
613
|
+
"""Initialize storage configuration at app startup"""
|
614
|
+
with open("config/storage.yaml") as f:
|
615
|
+
config = yaml.safe_load(f)
|
616
|
+
settings_manager.user_config = config
|
617
|
+
|
618
|
+
# app/main.py
|
619
|
+
from app.config import init_storage_config
|
620
|
+
|
621
|
+
def main():
|
622
|
+
init_storage_config()
|
623
|
+
# Now all modules can use get_blob() without configuration
|
624
|
+
```
|
625
|
+
|
626
|
+
### 2. Use Logical Names
|
627
|
+
|
628
|
+
```python
|
629
|
+
# ✅ Good - logical, environment-agnostic names
|
630
|
+
blob = get_blob(blob_name="users/123/profile.json")
|
631
|
+
blob = get_blob(blob_name="reports/2024/january.pdf")
|
632
|
+
|
633
|
+
# ❌ Bad - environment-specific details in code
|
634
|
+
blob = get_blob(blob_name="prod-v2-users-123-profile.json")
|
635
|
+
```
|
636
|
+
|
637
|
+
### 3. Organize with Prefixes
|
638
|
+
|
639
|
+
```python
|
640
|
+
# Configure hierarchical structure
|
641
|
+
settings_manager.user_config = {
|
642
|
+
"default": {
|
643
|
+
"bucket_name": "my-app-data",
|
644
|
+
"blob_name_prefix": "production/v2" # Version and environment
|
645
|
+
}
|
646
|
+
}
|
647
|
+
|
648
|
+
# Application uses clean paths
|
649
|
+
blob = get_blob(blob_name="users/123/profile.json")
|
650
|
+
# Actual: gs://my-app-data/production/v2/users/123/profile.json
|
651
|
+
```
|
652
|
+
|
653
|
+
### 4. Validate Configuration
|
654
|
+
|
655
|
+
```python
|
656
|
+
def validate_storage_config(config_key: str | None = None) -> bool:
|
657
|
+
"""Validate storage configuration at startup"""
|
658
|
+
try:
|
659
|
+
from kiarina.lib.google.cloud_storage import get_bucket
|
660
|
+
bucket = get_bucket(config_key)
|
661
|
+
bucket.exists()
|
662
|
+
print(f"✓ Storage configuration valid: {bucket.name}")
|
663
|
+
return True
|
664
|
+
except Exception as e:
|
665
|
+
print(f"✗ Storage configuration invalid: {e}")
|
666
|
+
return False
|
667
|
+
|
668
|
+
# At app startup
|
669
|
+
if not validate_storage_config():
|
670
|
+
raise RuntimeError("Invalid storage configuration")
|
671
|
+
```
|
672
|
+
|
673
|
+
## Development
|
674
|
+
|
675
|
+
### Prerequisites
|
676
|
+
|
677
|
+
- Python 3.12+
|
678
|
+
|
679
|
+
### Setup
|
680
|
+
|
681
|
+
```bash
|
682
|
+
# Clone the repository
|
683
|
+
git clone https://github.com/kiarina/kiarina-python.git
|
684
|
+
cd kiarina-python
|
685
|
+
|
686
|
+
# Setup development environment
|
687
|
+
mise run setup
|
688
|
+
```
|
689
|
+
|
690
|
+
### Running Tests
|
691
|
+
|
692
|
+
```bash
|
693
|
+
# Run format, lint, type checks and tests
|
694
|
+
mise run package kiarina-lib-google-cloud-storage
|
695
|
+
|
696
|
+
# Coverage report
|
697
|
+
mise run package:test kiarina-lib-google-cloud-storage --coverage
|
698
|
+
```
|
699
|
+
|
700
|
+
## Dependencies
|
701
|
+
|
702
|
+
- [google-cloud-storage](https://github.com/googleapis/python-storage) - Google Cloud Storage client library
|
703
|
+
- [kiarina-lib-google-auth](../kiarina-lib-google-auth/) - Google Cloud authentication library
|
704
|
+
- [pydantic-settings](https://docs.pydantic.dev/latest/concepts/pydantic_settings/) - Settings management
|
705
|
+
- [pydantic-settings-manager](https://github.com/kiarina/pydantic-settings-manager) - Advanced settings management
|
706
|
+
|
707
|
+
## License
|
708
|
+
|
709
|
+
This project is licensed under the MIT License - see the [LICENSE](../../LICENSE) file for details.
|
710
|
+
|
711
|
+
## Contributing
|
712
|
+
|
713
|
+
This is a personal project, but contributions are welcome! Please feel free to submit issues or pull requests.
|
714
|
+
|
715
|
+
## Related Projects
|
716
|
+
|
717
|
+
- [kiarina-python](https://github.com/kiarina/kiarina-python) - The main monorepo containing this package
|
718
|
+
- [kiarina-lib-google-auth](../kiarina-lib-google-auth/) - Google Cloud authentication library
|
719
|
+
- [pydantic-settings-manager](https://github.com/kiarina/pydantic-settings-manager) - Configuration management library used by this package
|
@@ -0,0 +1,9 @@
|
|
1
|
+
kiarina/lib/google/cloud_storage/__init__.py,sha256=q2AdOOOHUj_D-eot8_1-atrIaCKsCg0YatUOdq3kVVE,1184
|
2
|
+
kiarina/lib/google/cloud_storage/_get_blob.py,sha256=baZ7msH9xPme3jw0mI9x2NzikapdeSyBqH_6iDdgohI,728
|
3
|
+
kiarina/lib/google/cloud_storage/_get_bucket.py,sha256=Y7gX-igAkLoLhxa3cDazTAcNX_X_uUjJRUs93lcRg98,534
|
4
|
+
kiarina/lib/google/cloud_storage/_get_storage_client.py,sha256=XTsjQdVN4_tqO1KX-pC4-st7As7mMxeB1XhagzuJLaY,463
|
5
|
+
kiarina/lib/google/cloud_storage/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
6
|
+
kiarina/lib/google/cloud_storage/settings.py,sha256=QSY37za-Vi3S7fQLX7AzC5m65QxjxtuYYT9wq08Ee0o,381
|
7
|
+
kiarina_lib_google_cloud_storage-1.5.0.dist-info/METADATA,sha256=SCFb0PFxO75UWAhAtGMGVTMb3KvU0N7QPjc_2MdRUuY,20994
|
8
|
+
kiarina_lib_google_cloud_storage-1.5.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
9
|
+
kiarina_lib_google_cloud_storage-1.5.0.dist-info/RECORD,,
|