tiny-object-storage 0.1.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tiny_object_storage/__init__.py +8 -0
- tiny_object_storage/api/__init__.py +0 -0
- tiny_object_storage/api/buckets.py +114 -0
- tiny_object_storage/api/health.py +25 -0
- tiny_object_storage/api/objects.py +196 -0
- tiny_object_storage/cli.py +128 -0
- tiny_object_storage/config.py +37 -0
- tiny_object_storage/core/__init__.py +0 -0
- tiny_object_storage/core/errors.py +33 -0
- tiny_object_storage/core/utils.py +209 -0
- tiny_object_storage/logging_config.py +73 -0
- tiny_object_storage/main.py +199 -0
- tiny_object_storage/models/__init__.py +0 -0
- tiny_object_storage/models/bucket.py +10 -0
- tiny_object_storage/models/object_info.py +18 -0
- tiny_object_storage/models/responses.py +48 -0
- tiny_object_storage/services/__init__.py +0 -0
- tiny_object_storage/services/bucket_service.py +50 -0
- tiny_object_storage/services/object_service.py +79 -0
- tiny_object_storage/storage/__init__.py +0 -0
- tiny_object_storage/storage/base.py +67 -0
- tiny_object_storage/storage/filesystem.py +318 -0
- tiny_object_storage-0.1.1.dist-info/METADATA +228 -0
- tiny_object_storage-0.1.1.dist-info/RECORD +28 -0
- tiny_object_storage-0.1.1.dist-info/WHEEL +5 -0
- tiny_object_storage-0.1.1.dist-info/entry_points.txt +2 -0
- tiny_object_storage-0.1.1.dist-info/licenses/LICENSE +21 -0
- tiny_object_storage-0.1.1.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
from typing import Any, BinaryIO
|
|
2
|
+
|
|
3
|
+
from tiny_object_storage.logging_config import get_logger
|
|
4
|
+
from tiny_object_storage.storage.base import BaseStorage, StoredObject
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class ObjectService:
|
|
8
|
+
# Handle object-related business logic.
|
|
9
|
+
|
|
10
|
+
def __init__(self, storage: BaseStorage):
|
|
11
|
+
# Store the configured storage backend.
|
|
12
|
+
self.storage = storage
|
|
13
|
+
self.logger = get_logger("tiny_object_storage.services.object")
|
|
14
|
+
|
|
15
|
+
def put_object(
|
|
16
|
+
self,
|
|
17
|
+
bucket_name: str,
|
|
18
|
+
object_key: str,
|
|
19
|
+
data: bytes | BinaryIO,
|
|
20
|
+
content_type: str | None = None,
|
|
21
|
+
) -> dict[str, Any]:
|
|
22
|
+
# Store an object and return its metadata.
|
|
23
|
+
self.logger.info(
|
|
24
|
+
"Uploading object bucket=%s key=%s content_type=%s",
|
|
25
|
+
bucket_name,
|
|
26
|
+
object_key,
|
|
27
|
+
content_type,
|
|
28
|
+
)
|
|
29
|
+
metadata = self.storage.put_object(
|
|
30
|
+
bucket_name=bucket_name,
|
|
31
|
+
object_key=object_key,
|
|
32
|
+
data=data,
|
|
33
|
+
content_type=content_type,
|
|
34
|
+
)
|
|
35
|
+
self.logger.info(
|
|
36
|
+
"Object uploaded bucket=%s key=%s size=%s etag=%s",
|
|
37
|
+
bucket_name,
|
|
38
|
+
object_key,
|
|
39
|
+
metadata["size"],
|
|
40
|
+
metadata["etag"],
|
|
41
|
+
)
|
|
42
|
+
return metadata
|
|
43
|
+
|
|
44
|
+
def get_object(self, bucket_name: str, object_key: str) -> StoredObject:
|
|
45
|
+
# Return a stored object from the backend.
|
|
46
|
+
self.logger.info("Downloading object bucket=%s key=%s", bucket_name, object_key)
|
|
47
|
+
stored_object = self.storage.get_object(bucket_name=bucket_name, object_key=object_key)
|
|
48
|
+
self.logger.info(
|
|
49
|
+
"Object downloaded bucket=%s key=%s size=%s",
|
|
50
|
+
bucket_name,
|
|
51
|
+
object_key,
|
|
52
|
+
stored_object.metadata["size"],
|
|
53
|
+
)
|
|
54
|
+
return stored_object
|
|
55
|
+
|
|
56
|
+
def delete_object(self, bucket_name: str, object_key: str) -> None:
|
|
57
|
+
# Delete an object from the backend.
|
|
58
|
+
self.logger.info("Deleting object bucket=%s key=%s", bucket_name, object_key)
|
|
59
|
+
self.storage.delete_object(bucket_name=bucket_name, object_key=object_key)
|
|
60
|
+
self.logger.info("Object deleted bucket=%s key=%s", bucket_name, object_key)
|
|
61
|
+
|
|
62
|
+
def list_objects(self, bucket_name: str) -> list[dict[str, Any]]:
|
|
63
|
+
# Return all objects for the requested bucket.
|
|
64
|
+
self.logger.info("Listing objects bucket=%s", bucket_name)
|
|
65
|
+
objects = self.storage.list_objects(bucket_name=bucket_name)
|
|
66
|
+
self.logger.info("Objects listed bucket=%s count=%s", bucket_name, len(objects))
|
|
67
|
+
return objects
|
|
68
|
+
|
|
69
|
+
def object_exists(self, bucket_name: str, object_key: str) -> bool:
|
|
70
|
+
# Return whether the given object exists.
|
|
71
|
+
self.logger.debug("Checking object existence bucket=%s key=%s", bucket_name, object_key)
|
|
72
|
+
exists = self.storage.object_exists(bucket_name=bucket_name, object_key=object_key)
|
|
73
|
+
self.logger.debug(
|
|
74
|
+
"Object existence checked bucket=%s key=%s exists=%s",
|
|
75
|
+
bucket_name,
|
|
76
|
+
object_key,
|
|
77
|
+
exists,
|
|
78
|
+
)
|
|
79
|
+
return exists
|
|
File without changes
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
from abc import ABC, abstractmethod
|
|
2
|
+
from dataclasses import dataclass
|
|
3
|
+
from typing import Any, BinaryIO
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
@dataclass(slots=True)
|
|
7
|
+
class StoredObject:
|
|
8
|
+
# Represent an object returned from the storage backend.
|
|
9
|
+
bucket_name: str
|
|
10
|
+
object_key: str
|
|
11
|
+
data: bytes
|
|
12
|
+
metadata: dict[str, Any]
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class BaseStorage(ABC):
|
|
16
|
+
# Define the storage backend contract.
|
|
17
|
+
|
|
18
|
+
@abstractmethod
|
|
19
|
+
def create_bucket(self, bucket_name: str) -> None:
|
|
20
|
+
# Create a new bucket.
|
|
21
|
+
raise NotImplementedError
|
|
22
|
+
|
|
23
|
+
@abstractmethod
|
|
24
|
+
def delete_bucket(self, bucket_name: str) -> None:
|
|
25
|
+
# Delete an existing bucket.
|
|
26
|
+
raise NotImplementedError
|
|
27
|
+
|
|
28
|
+
@abstractmethod
|
|
29
|
+
def list_buckets(self) -> list[dict[str, Any]]:
|
|
30
|
+
# Return all available buckets.
|
|
31
|
+
raise NotImplementedError
|
|
32
|
+
|
|
33
|
+
@abstractmethod
|
|
34
|
+
def bucket_exists(self, bucket_name: str) -> bool:
|
|
35
|
+
# Check whether a bucket exists.
|
|
36
|
+
raise NotImplementedError
|
|
37
|
+
|
|
38
|
+
@abstractmethod
|
|
39
|
+
def put_object(
|
|
40
|
+
self,
|
|
41
|
+
bucket_name: str,
|
|
42
|
+
object_key: str,
|
|
43
|
+
data: bytes | BinaryIO,
|
|
44
|
+
content_type: str | None = None,
|
|
45
|
+
) -> dict[str, Any]:
|
|
46
|
+
# Store an object and return its metadata.
|
|
47
|
+
raise NotImplementedError
|
|
48
|
+
|
|
49
|
+
@abstractmethod
|
|
50
|
+
def get_object(self, bucket_name: str, object_key: str) -> StoredObject:
|
|
51
|
+
# Read and return an object.
|
|
52
|
+
raise NotImplementedError
|
|
53
|
+
|
|
54
|
+
@abstractmethod
|
|
55
|
+
def delete_object(self, bucket_name: str, object_key: str) -> None:
|
|
56
|
+
# Delete an object.
|
|
57
|
+
raise NotImplementedError
|
|
58
|
+
|
|
59
|
+
@abstractmethod
|
|
60
|
+
def list_objects(self, bucket_name: str) -> list[dict[str, Any]]:
|
|
61
|
+
# Return all objects in a bucket.
|
|
62
|
+
raise NotImplementedError
|
|
63
|
+
|
|
64
|
+
@abstractmethod
|
|
65
|
+
def object_exists(self, bucket_name: str, object_key: str) -> bool:
|
|
66
|
+
# Check whether an object exists.
|
|
67
|
+
raise NotImplementedError
|
|
@@ -0,0 +1,318 @@
|
|
|
1
|
+
import mimetypes
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
from typing import Any, BinaryIO
|
|
4
|
+
|
|
5
|
+
from tiny_object_storage.core.errors import (
|
|
6
|
+
BucketAlreadyExistsError,
|
|
7
|
+
BucketNotEmptyError,
|
|
8
|
+
BucketNotFoundError,
|
|
9
|
+
ObjectNotFoundError,
|
|
10
|
+
)
|
|
11
|
+
from tiny_object_storage.core.utils import (
|
|
12
|
+
bucket_path,
|
|
13
|
+
compute_etag,
|
|
14
|
+
metadata_dir_path,
|
|
15
|
+
metadata_file_path,
|
|
16
|
+
object_path,
|
|
17
|
+
read_json,
|
|
18
|
+
utc_now_iso,
|
|
19
|
+
validate_bucket_name,
|
|
20
|
+
validate_object_key,
|
|
21
|
+
write_json,
|
|
22
|
+
)
|
|
23
|
+
from tiny_object_storage.logging_config import get_logger
|
|
24
|
+
from tiny_object_storage.storage.base import BaseStorage, StoredObject
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class FilesystemStorage(BaseStorage):
|
|
28
|
+
# Implement the storage backend using the local filesystem.
|
|
29
|
+
|
|
30
|
+
def __init__(self, base_dir: str | Path):
|
|
31
|
+
# Initialize the storage backend.
|
|
32
|
+
self.base_dir = Path(base_dir)
|
|
33
|
+
self.base_dir.mkdir(parents=True, exist_ok=True)
|
|
34
|
+
self.logger = get_logger("tiny_object_storage.storage.filesystem")
|
|
35
|
+
self.logger.info("Filesystem storage initialized base_dir=%s", self.base_dir)
|
|
36
|
+
|
|
37
|
+
def create_bucket(self, bucket_name: str) -> None:
|
|
38
|
+
# Create a new bucket as a directory with a metadata subdirectory.
|
|
39
|
+
validate_bucket_name(bucket_name)
|
|
40
|
+
path = bucket_path(self.base_dir, bucket_name)
|
|
41
|
+
|
|
42
|
+
if path.exists():
|
|
43
|
+
self.logger.warning("Bucket already exists bucket=%s", bucket_name)
|
|
44
|
+
raise BucketAlreadyExistsError(f"Bucket '{bucket_name}' already exists.")
|
|
45
|
+
|
|
46
|
+
path.mkdir(parents=True, exist_ok=False)
|
|
47
|
+
metadata_dir_path(self.base_dir, bucket_name).mkdir(parents=True, exist_ok=True)
|
|
48
|
+
|
|
49
|
+
bucket_metadata = {
|
|
50
|
+
"name": bucket_name,
|
|
51
|
+
"created_at": utc_now_iso(),
|
|
52
|
+
}
|
|
53
|
+
write_json(path / ".bucket.meta.json", bucket_metadata)
|
|
54
|
+
|
|
55
|
+
self.logger.info("Bucket created bucket=%s path=%s", bucket_name, path)
|
|
56
|
+
|
|
57
|
+
def delete_bucket(self, bucket_name: str) -> None:
|
|
58
|
+
# Delete an empty bucket.
|
|
59
|
+
path = self._require_bucket(bucket_name)
|
|
60
|
+
|
|
61
|
+
entries = [
|
|
62
|
+
entry for entry in path.iterdir() if entry.name not in {".meta", ".bucket.meta.json"}
|
|
63
|
+
]
|
|
64
|
+
if entries:
|
|
65
|
+
self.logger.warning("Bucket not empty bucket=%s", bucket_name)
|
|
66
|
+
raise BucketNotEmptyError(f"Bucket '{bucket_name}' is not empty.")
|
|
67
|
+
|
|
68
|
+
meta_dir = path / ".meta"
|
|
69
|
+
if meta_dir.exists():
|
|
70
|
+
for meta_file in meta_dir.iterdir():
|
|
71
|
+
meta_file.unlink()
|
|
72
|
+
meta_dir.rmdir()
|
|
73
|
+
|
|
74
|
+
bucket_meta_file = path / ".bucket.meta.json"
|
|
75
|
+
if bucket_meta_file.exists():
|
|
76
|
+
bucket_meta_file.unlink()
|
|
77
|
+
|
|
78
|
+
path.rmdir()
|
|
79
|
+
self.logger.info("Bucket deleted bucket=%s path=%s", bucket_name, path)
|
|
80
|
+
|
|
81
|
+
def list_buckets(self) -> list[dict[str, Any]]:
|
|
82
|
+
# List all buckets stored under the base directory.
|
|
83
|
+
buckets: list[dict[str, Any]] = []
|
|
84
|
+
|
|
85
|
+
for path in sorted(self.base_dir.iterdir(), key=lambda item: item.name):
|
|
86
|
+
if not path.is_dir():
|
|
87
|
+
continue
|
|
88
|
+
|
|
89
|
+
bucket_meta_file = path / ".bucket.meta.json"
|
|
90
|
+
created_at = None
|
|
91
|
+
if bucket_meta_file.exists():
|
|
92
|
+
try:
|
|
93
|
+
created_at = read_json(bucket_meta_file).get("created_at")
|
|
94
|
+
except Exception:
|
|
95
|
+
self.logger.exception("Failed to read bucket metadata bucket=%s", path.name)
|
|
96
|
+
|
|
97
|
+
buckets.append(
|
|
98
|
+
{
|
|
99
|
+
"name": path.name,
|
|
100
|
+
"created_at": created_at,
|
|
101
|
+
}
|
|
102
|
+
)
|
|
103
|
+
|
|
104
|
+
self.logger.info("Buckets listed count=%s", len(buckets))
|
|
105
|
+
return buckets
|
|
106
|
+
|
|
107
|
+
def bucket_exists(self, bucket_name: str) -> bool:
|
|
108
|
+
# Return whether the given bucket exists.
|
|
109
|
+
validate_bucket_name(bucket_name)
|
|
110
|
+
exists = bucket_path(self.base_dir, bucket_name).is_dir()
|
|
111
|
+
self.logger.debug("Bucket existence checked bucket=%s exists=%s", bucket_name, exists)
|
|
112
|
+
return exists
|
|
113
|
+
|
|
114
|
+
def put_object(
|
|
115
|
+
self,
|
|
116
|
+
bucket_name: str,
|
|
117
|
+
object_key: str,
|
|
118
|
+
data: bytes | BinaryIO,
|
|
119
|
+
content_type: str | None = None,
|
|
120
|
+
) -> dict[str, Any]:
|
|
121
|
+
# Write object data to disk and persist metadata.
|
|
122
|
+
self._require_bucket(bucket_name)
|
|
123
|
+
validate_object_key(object_key)
|
|
124
|
+
|
|
125
|
+
payload = self._read_bytes(data)
|
|
126
|
+
object_file_path = object_path(self.base_dir, bucket_name, object_key)
|
|
127
|
+
object_file_path.parent.mkdir(parents=True, exist_ok=True)
|
|
128
|
+
|
|
129
|
+
object_file_path.write_bytes(payload)
|
|
130
|
+
|
|
131
|
+
resolved_content_type = (
|
|
132
|
+
content_type or mimetypes.guess_type(object_key)[0] or "application/octet-stream"
|
|
133
|
+
)
|
|
134
|
+
now = utc_now_iso()
|
|
135
|
+
|
|
136
|
+
previous_metadata = self._read_metadata_if_exists(bucket_name, object_key)
|
|
137
|
+
created_at = previous_metadata.get("created_at", now) if previous_metadata else now
|
|
138
|
+
|
|
139
|
+
metadata = {
|
|
140
|
+
"bucket_name": bucket_name,
|
|
141
|
+
"object_key": object_key,
|
|
142
|
+
"size": len(payload),
|
|
143
|
+
"content_type": resolved_content_type,
|
|
144
|
+
"etag": compute_etag(payload),
|
|
145
|
+
"created_at": created_at,
|
|
146
|
+
"updated_at": now,
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
meta_path = metadata_file_path(self.base_dir, bucket_name, object_key)
|
|
150
|
+
write_json(meta_path, metadata)
|
|
151
|
+
|
|
152
|
+
self.logger.info(
|
|
153
|
+
"Object stored bucket=%s key=%s size=%s path=%s",
|
|
154
|
+
bucket_name,
|
|
155
|
+
object_key,
|
|
156
|
+
len(payload),
|
|
157
|
+
object_file_path,
|
|
158
|
+
)
|
|
159
|
+
return metadata
|
|
160
|
+
|
|
161
|
+
def get_object(self, bucket_name: str, object_key: str) -> StoredObject:
|
|
162
|
+
# Read an object from disk and return its bytes and metadata.
|
|
163
|
+
self._require_bucket(bucket_name)
|
|
164
|
+
validate_object_key(object_key)
|
|
165
|
+
|
|
166
|
+
object_file_path = object_path(self.base_dir, bucket_name, object_key)
|
|
167
|
+
if not object_file_path.is_file():
|
|
168
|
+
self.logger.warning("Object not found bucket=%s key=%s", bucket_name, object_key)
|
|
169
|
+
raise ObjectNotFoundError(
|
|
170
|
+
f"Object '{object_key}' was not found in bucket '{bucket_name}'."
|
|
171
|
+
)
|
|
172
|
+
|
|
173
|
+
data = object_file_path.read_bytes()
|
|
174
|
+
metadata = self._read_metadata_if_exists(bucket_name, object_key)
|
|
175
|
+
if not metadata:
|
|
176
|
+
metadata = {
|
|
177
|
+
"bucket_name": bucket_name,
|
|
178
|
+
"object_key": object_key,
|
|
179
|
+
"size": len(data),
|
|
180
|
+
"content_type": mimetypes.guess_type(object_key)[0] or "application/octet-stream",
|
|
181
|
+
"etag": compute_etag(data),
|
|
182
|
+
"created_at": None,
|
|
183
|
+
"updated_at": None,
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
self.logger.info("Object read bucket=%s key=%s size=%s", bucket_name, object_key, len(data))
|
|
187
|
+
return StoredObject(
|
|
188
|
+
bucket_name=bucket_name,
|
|
189
|
+
object_key=object_key,
|
|
190
|
+
data=data,
|
|
191
|
+
metadata=metadata,
|
|
192
|
+
)
|
|
193
|
+
|
|
194
|
+
def delete_object(self, bucket_name: str, object_key: str) -> None:
|
|
195
|
+
# Delete an object and its metadata.
|
|
196
|
+
self._require_bucket(bucket_name)
|
|
197
|
+
validate_object_key(object_key)
|
|
198
|
+
|
|
199
|
+
object_file_path = object_path(self.base_dir, bucket_name, object_key)
|
|
200
|
+
if not object_file_path.is_file():
|
|
201
|
+
self.logger.warning(
|
|
202
|
+
"Object not found for deletion bucket=%s key=%s", bucket_name, object_key
|
|
203
|
+
)
|
|
204
|
+
raise ObjectNotFoundError(
|
|
205
|
+
f"Object '{object_key}' was not found in bucket '{bucket_name}'."
|
|
206
|
+
)
|
|
207
|
+
|
|
208
|
+
object_file_path.unlink()
|
|
209
|
+
|
|
210
|
+
meta_path = metadata_file_path(self.base_dir, bucket_name, object_key)
|
|
211
|
+
if meta_path.exists():
|
|
212
|
+
meta_path.unlink()
|
|
213
|
+
|
|
214
|
+
self._cleanup_empty_parent_directories(bucket_name=bucket_name, object_key=object_key)
|
|
215
|
+
|
|
216
|
+
self.logger.info("Object deleted bucket=%s key=%s", bucket_name, object_key)
|
|
217
|
+
|
|
218
|
+
def list_objects(self, bucket_name: str) -> list[dict[str, Any]]:
|
|
219
|
+
# List all objects in a bucket recursively.
|
|
220
|
+
bucket_dir = self._require_bucket(bucket_name)
|
|
221
|
+
results: list[dict[str, Any]] = []
|
|
222
|
+
|
|
223
|
+
for path in sorted(bucket_dir.rglob("*"), key=lambda item: str(item)):
|
|
224
|
+
if not path.is_file():
|
|
225
|
+
continue
|
|
226
|
+
|
|
227
|
+
if path.name == ".bucket.meta.json":
|
|
228
|
+
continue
|
|
229
|
+
|
|
230
|
+
if ".meta" in path.parts:
|
|
231
|
+
continue
|
|
232
|
+
|
|
233
|
+
object_key = str(path.relative_to(bucket_dir)).replace("\\", "/")
|
|
234
|
+
metadata = self._read_metadata_if_exists(bucket_name, object_key)
|
|
235
|
+
|
|
236
|
+
if not metadata:
|
|
237
|
+
data = path.read_bytes()
|
|
238
|
+
metadata = {
|
|
239
|
+
"bucket_name": bucket_name,
|
|
240
|
+
"object_key": object_key,
|
|
241
|
+
"size": len(data),
|
|
242
|
+
"content_type": mimetypes.guess_type(object_key)[0]
|
|
243
|
+
or "application/octet-stream",
|
|
244
|
+
"etag": compute_etag(data),
|
|
245
|
+
"created_at": None,
|
|
246
|
+
"updated_at": None,
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
results.append(metadata)
|
|
250
|
+
|
|
251
|
+
self.logger.info("Objects listed bucket=%s count=%s", bucket_name, len(results))
|
|
252
|
+
return results
|
|
253
|
+
|
|
254
|
+
def object_exists(self, bucket_name: str, object_key: str) -> bool:
|
|
255
|
+
# Return whether the given object exists.
|
|
256
|
+
self._require_bucket(bucket_name)
|
|
257
|
+
validate_object_key(object_key)
|
|
258
|
+
|
|
259
|
+
exists = object_path(self.base_dir, bucket_name, object_key).is_file()
|
|
260
|
+
self.logger.debug(
|
|
261
|
+
"Object existence checked bucket=%s key=%s exists=%s",
|
|
262
|
+
bucket_name,
|
|
263
|
+
object_key,
|
|
264
|
+
exists,
|
|
265
|
+
)
|
|
266
|
+
return exists
|
|
267
|
+
|
|
268
|
+
def _require_bucket(self, bucket_name: str) -> Path:
|
|
269
|
+
# Return the bucket path or raise if the bucket does not exist.
|
|
270
|
+
validate_bucket_name(bucket_name)
|
|
271
|
+
path = bucket_path(self.base_dir, bucket_name)
|
|
272
|
+
|
|
273
|
+
if not path.is_dir():
|
|
274
|
+
self.logger.warning("Bucket not found bucket=%s", bucket_name)
|
|
275
|
+
raise BucketNotFoundError(f"Bucket '{bucket_name}' does not exist.")
|
|
276
|
+
|
|
277
|
+
return path
|
|
278
|
+
|
|
279
|
+
def _read_bytes(self, data: bytes | BinaryIO) -> bytes:
|
|
280
|
+
# Normalize supported input types into bytes.
|
|
281
|
+
if isinstance(data, bytes):
|
|
282
|
+
return data
|
|
283
|
+
|
|
284
|
+
return data.read()
|
|
285
|
+
|
|
286
|
+
def _read_metadata_if_exists(self, bucket_name: str, object_key: str) -> dict[str, Any] | None:
|
|
287
|
+
# Read metadata if the file exists.
|
|
288
|
+
meta_path = metadata_file_path(self.base_dir, bucket_name, object_key)
|
|
289
|
+
if not meta_path.exists():
|
|
290
|
+
return None
|
|
291
|
+
|
|
292
|
+
try:
|
|
293
|
+
metadata = read_json(meta_path)
|
|
294
|
+
self.logger.debug(
|
|
295
|
+
"Metadata read bucket=%s key=%s path=%s", bucket_name, object_key, meta_path
|
|
296
|
+
)
|
|
297
|
+
return metadata
|
|
298
|
+
except Exception:
|
|
299
|
+
self.logger.exception(
|
|
300
|
+
"Failed to read metadata bucket=%s key=%s path=%s",
|
|
301
|
+
bucket_name,
|
|
302
|
+
object_key,
|
|
303
|
+
meta_path,
|
|
304
|
+
)
|
|
305
|
+
return None
|
|
306
|
+
|
|
307
|
+
def _cleanup_empty_parent_directories(self, bucket_name: str, object_key: str) -> None:
|
|
308
|
+
# Remove empty object directories after deletion.
|
|
309
|
+
bucket_dir = bucket_path(self.base_dir, bucket_name)
|
|
310
|
+
object_file_path = object_path(self.base_dir, bucket_name, object_key)
|
|
311
|
+
|
|
312
|
+
current = object_file_path.parent
|
|
313
|
+
while current != bucket_dir and current.exists():
|
|
314
|
+
if any(current.iterdir()):
|
|
315
|
+
break
|
|
316
|
+
current.rmdir()
|
|
317
|
+
self.logger.debug("Removed empty directory path=%s", current)
|
|
318
|
+
current = current.parent
|
|
@@ -0,0 +1,228 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: tiny-object-storage
|
|
3
|
+
Version: 0.1.1
|
|
4
|
+
Summary: A lightweight S3-compatible object storage server for local development and learning.
|
|
5
|
+
Author: Ehsan Bitaraf
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/EhsanBitaraf/tiny-object-storage
|
|
8
|
+
Project-URL: Repository, https://github.com/EhsanBitaraf/tiny-object-storage
|
|
9
|
+
Project-URL: Bug Tracker, https://github.com/EhsanBitaraf/tiny-object-storage/issues
|
|
10
|
+
Keywords: s3,object-storage,minio,aws-s3,emulator,s3-compatible,mock,testing,fastapi
|
|
11
|
+
Classifier: Development Status :: 4 - Beta
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: Topic :: Software Development :: Testing
|
|
14
|
+
Classifier: Topic :: Internet :: WWW/HTTP
|
|
15
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Classifier: Operating System :: OS Independent
|
|
20
|
+
Requires-Python: >=3.11
|
|
21
|
+
Description-Content-Type: text/markdown
|
|
22
|
+
License-File: LICENSE
|
|
23
|
+
Requires-Dist: fastapi<1.0.0,>=0.115.0
|
|
24
|
+
Requires-Dist: uvicorn[standard]<1.0.0,>=0.30.0
|
|
25
|
+
Requires-Dist: pydantic<3.0.0,>=2.8.0
|
|
26
|
+
Requires-Dist: pydantic-settings<3.0.0,>=2.4.0
|
|
27
|
+
Requires-Dist: python-dotenv<2.0.0,>=1.0.1
|
|
28
|
+
Requires-Dist: python-multipart<1.0.0,>=0.0.9
|
|
29
|
+
Provides-Extra: dev
|
|
30
|
+
Requires-Dist: pytest<9.0.0,>=8.3.0; extra == "dev"
|
|
31
|
+
Requires-Dist: pytest-cov<6.0.0,>=5.0.0; extra == "dev"
|
|
32
|
+
Requires-Dist: httpx<1.0.0,>=0.27.0; extra == "dev"
|
|
33
|
+
Requires-Dist: ruff<1.0.0,>=0.6.0; extra == "dev"
|
|
34
|
+
Requires-Dist: mypy<2.0.0,>=1.11.0; extra == "dev"
|
|
35
|
+
Requires-Dist: bandit<2.0.0,>=1.7.9; extra == "dev"
|
|
36
|
+
Requires-Dist: safety<4.0.0,>=3.2.0; extra == "dev"
|
|
37
|
+
Requires-Dist: pre-commit<4.0.0,>=3.8.0; extra == "dev"
|
|
38
|
+
Dynamic: license-file
|
|
39
|
+
|
|
40
|
+
<div align="center">
|
|
41
|
+
<img src="assets/logo.png" alt="tiny-object-storage logo" width="180" height="auto" />
|
|
42
|
+
<h1>🪣 tiny-object-storage</h1>
|
|
43
|
+
<p><i>A lightweight, S3-compatible object storage server for local development, testing, and learning.</i></p>
|
|
44
|
+
|
|
45
|
+
[](https://www.python.org)
|
|
46
|
+
[](https://fastapi.tiangolo.com)
|
|
47
|
+
[](https://github.com/astral-sh/ruff)
|
|
48
|
+
[](https://mypy.readthedocs.io/en/stable/)
|
|
49
|
+
[](https://github.com/PyCQA/bandit)
|
|
50
|
+
[](https://github.com/pre-commit/pre-commit)
|
|
51
|
+
[](https://pytest-cov.readthedocs.io/en/latest/)
|
|
52
|
+
[](https://opensource.org/licenses/MIT)
|
|
53
|
+
|
|
54
|
+
</div>
|
|
55
|
+
|
|
56
|
+
---
|
|
57
|
+
|
|
58
|
+
## 📖 Overview
|
|
59
|
+
|
|
60
|
+
`tiny-object-storage` is a fast, minimalistic object storage server written in Python using FastAPI. It acts as a local emulator that mimics the core functionality of Amazon S3, allowing developers to test S3-integrated applications locally without relying on external cloud services or heavy containers like MinIO.
|
|
61
|
+
|
|
62
|
+
## ✨ Features
|
|
63
|
+
|
|
64
|
+
- **S3 Compatibility**: Supports core S3 XML responses and basic HTTP methods (`GET`, `PUT`, `HEAD`, `DELETE`).
|
|
65
|
+
- **Bucket Management**: Create, list, and delete buckets.
|
|
66
|
+
- **Object Operations**: Upload, download, metadata lookup, listing (`list-type=2`), and deletion.
|
|
67
|
+
- **Persistent Storage**: Filesystem-based persistence with dedicated metadata and data separation.
|
|
68
|
+
- **Developer Experience**: Interactive Swagger UI and OpenAPI schema built-in.
|
|
69
|
+
- **Enterprise Code Quality**: Enforced static typing (Mypy strict), linting (Ruff), and security analysis (Bandit).
|
|
70
|
+
- **Flexible Configuration**: Configurable via CLI arguments, environment variables, or `.env` files.
|
|
71
|
+
|
|
72
|
+
---
|
|
73
|
+
|
|
74
|
+
## 🚀 Quick Start
|
|
75
|
+
|
|
76
|
+
### Installation
|
|
77
|
+
|
|
78
|
+
**From PyPI (Recommended)**
|
|
79
|
+
Install the package globally or in a virtual environment using `pip`:
|
|
80
|
+
|
|
81
|
+
```bash
|
|
82
|
+
pip install tiny-object-storage
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
**From Source (Development)**
|
|
86
|
+
Clone the repository and install it in a virtual environment:
|
|
87
|
+
|
|
88
|
+
```bash
|
|
89
|
+
git clone https://github.com/EhsanBitaraf/tiny-object-storage.git
|
|
90
|
+
cd tiny-object-storage
|
|
91
|
+
python -m venv .venv
|
|
92
|
+
|
|
93
|
+
# On Linux/macOS:
|
|
94
|
+
source .venv/bin/activate
|
|
95
|
+
# On Windows:
|
|
96
|
+
.venv\Scripts\activate
|
|
97
|
+
|
|
98
|
+
# Install the application and development dependencies
|
|
99
|
+
pip install -e ".[dev]"
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
### Running the Server
|
|
103
|
+
|
|
104
|
+
Start the server using the built-in CLI:
|
|
105
|
+
|
|
106
|
+
```bash
|
|
107
|
+
tos-server
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
Or run it directly with `uvicorn`:
|
|
111
|
+
```bash
|
|
112
|
+
uvicorn tiny_object_storage.main:app --reload --host 0.0.0.0 --port 9000
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
---
|
|
116
|
+
|
|
117
|
+
## ⚙️ Configuration
|
|
118
|
+
|
|
119
|
+
The server reads configuration in the following priority (highest to lowest):
|
|
120
|
+
1. **Command-line arguments**
|
|
121
|
+
2. **Environment variables**
|
|
122
|
+
3. **`.env` file**
|
|
123
|
+
|
|
124
|
+
### CLI Usage Examples
|
|
125
|
+
|
|
126
|
+
```bash
|
|
127
|
+
# View all available options
|
|
128
|
+
tos-server --help
|
|
129
|
+
|
|
130
|
+
# Load from a specific .env file
|
|
131
|
+
tos-server --env-file .env.custom
|
|
132
|
+
|
|
133
|
+
# Override host and port
|
|
134
|
+
tos-server --host 127.0.0.1 --port 9001 --reload
|
|
135
|
+
|
|
136
|
+
# Customize storage directories and log level
|
|
137
|
+
tos-server --data-dir ./storage_data --log-dir ./logs --log-level debug
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
---
|
|
141
|
+
|
|
142
|
+
## 🔌 S3 Client Compatibility
|
|
143
|
+
|
|
144
|
+
`tiny-object-storage` acts as a local emulator and has been successfully verified against the following official SDKs and tools:
|
|
145
|
+
|
|
146
|
+
- **MinIO Python SDK**
|
|
147
|
+
- **Boto3 (AWS SDK for Python)**
|
|
148
|
+
- **AWS CLI**
|
|
149
|
+
|
|
150
|
+
> **Note:** For official SDKs to interact locally, configure them with `secure=False` (or `--no-verify-ssl`) and point the `endpoint_url` to `http://127.0.0.1:9000` with any dummy credentials.
|
|
151
|
+
|
|
152
|
+
Sample scripts for verifying SDK interactions are included in the repository root (e.g., `test_boto3_sdk.py`, `test_minio_sdk.py`).
|
|
153
|
+
|
|
154
|
+
---
|
|
155
|
+
|
|
156
|
+
## 🛠 Testing & Code Quality
|
|
157
|
+
|
|
158
|
+
The project includes an extensive test suite built with `pytest` achieving **100% test pass rate** over 63 edge-case scenarios.
|
|
159
|
+
|
|
160
|
+
### Running Tests
|
|
161
|
+
|
|
162
|
+
```bash
|
|
163
|
+
pytest
|
|
164
|
+
```
|
|
165
|
+
*(Optionally, use `pytest --cov=src` to generate a coverage report).*
|
|
166
|
+
|
|
167
|
+
### Code Quality Tools
|
|
168
|
+
|
|
169
|
+
We use `pre-commit` to maintain our code quality. The suite includes:
|
|
170
|
+
- **Ruff**: For ultra-fast linting and code formatting.
|
|
171
|
+
- **Mypy**: For strict static type checking.
|
|
172
|
+
- **Bandit**: For security vulnerability scanning.
|
|
173
|
+
|
|
174
|
+
To run checks manually on all files:
|
|
175
|
+
```bash
|
|
176
|
+
pre-commit run --all-files
|
|
177
|
+
```
|
|
178
|
+
|
|
179
|
+
---
|
|
180
|
+
|
|
181
|
+
## 🌐 API Endpoints (Examples)
|
|
182
|
+
|
|
183
|
+
After starting the server, visit `/docs` for the interactive Swagger UI. Below are standard `curl` examples targeting common S3 actions:
|
|
184
|
+
|
|
185
|
+
**Health check**
|
|
186
|
+
```bash
|
|
187
|
+
curl http://127.0.0.1:9000/health
|
|
188
|
+
```
|
|
189
|
+
|
|
190
|
+
**Create a bucket**
|
|
191
|
+
```bash
|
|
192
|
+
curl -X PUT http://127.0.0.1:9000/photos
|
|
193
|
+
```
|
|
194
|
+
|
|
195
|
+
**Upload an object**
|
|
196
|
+
```bash
|
|
197
|
+
curl -X PUT --data-binary @"cat.jpg" http://127.0.0.1:9000/photos/cat.jpg
|
|
198
|
+
```
|
|
199
|
+
|
|
200
|
+
**Download an object**
|
|
201
|
+
```bash
|
|
202
|
+
curl http://127.0.0.1:9000/photos/cat.jpg --output cat.jpg
|
|
203
|
+
```
|
|
204
|
+
|
|
205
|
+
**List objects (S3 Format)**
|
|
206
|
+
```bash
|
|
207
|
+
curl "http://127.0.0.1:9000/photos?list-type=2"
|
|
208
|
+
```
|
|
209
|
+
|
|
210
|
+
**Debug endpoints (JSON format)**
|
|
211
|
+
```bash
|
|
212
|
+
curl http://127.0.0.1:9000/_debug/buckets
|
|
213
|
+
curl http://127.0.0.1:9000/_debug/photos
|
|
214
|
+
```
|
|
215
|
+
|
|
216
|
+
---
|
|
217
|
+
|
|
218
|
+
## 🤝 Contributing
|
|
219
|
+
|
|
220
|
+
Contributions are welcome and highly appreciated! Please review our [Contributing Guidelines](CONTRIBUTING.md) to get started. By participating in this project, you agree to abide by the [Code of Conduct](.github/CODE_OF_CONDUCT.md).
|
|
221
|
+
|
|
222
|
+
## 🔒 Security
|
|
223
|
+
|
|
224
|
+
For instructions on reporting security vulnerabilities, please refer to our [Security Policy](SECURITY.md).
|
|
225
|
+
|
|
226
|
+
## 📜 License
|
|
227
|
+
|
|
228
|
+
This project is licensed under the MIT License - see the `LICENSE` file for details.
|