nexo-storage 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- nexo_storage-0.3.0/LICENSE +21 -0
- nexo_storage-0.3.0/PKG-INFO +98 -0
- nexo_storage-0.3.0/nexo_storage.egg-info/PKG-INFO +98 -0
- nexo_storage-0.3.0/nexo_storage.egg-info/SOURCES.txt +14 -0
- nexo_storage-0.3.0/nexo_storage.egg-info/dependency_links.txt +1 -0
- nexo_storage-0.3.0/nexo_storage.egg-info/requires.txt +88 -0
- nexo_storage-0.3.0/nexo_storage.egg-info/top_level.txt +1 -0
- nexo_storage-0.3.0/pyproject.toml +126 -0
- nexo_storage-0.3.0/setup.cfg +4 -0
- nexo_storage-0.3.0/src/__init__.py +0 -0
- nexo_storage-0.3.0/src/common.py +12 -0
- nexo_storage-0.3.0/src/config.py +7 -0
- nexo_storage-0.3.0/src/enums.py +6 -0
- nexo_storage-0.3.0/src/google.py +335 -0
- nexo_storage-0.3.0/src/minio.py +326 -0
- nexo_storage-0.3.0/src/schemas.py +11 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 hubagrayuda
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: nexo-storage
|
|
3
|
+
Version: 0.3.0
|
|
4
|
+
Summary: Storage package for Nexo
|
|
5
|
+
Author-email: Agra Bima Yuda <hub.agrayuda@gmail.com>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Requires-Python: >=3.12
|
|
8
|
+
Description-Content-Type: text/markdown
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Requires-Dist: aioredis>=2.0.1
|
|
11
|
+
Requires-Dist: annotated-doc>=0.0.4
|
|
12
|
+
Requires-Dist: annotated-types>=0.7.0
|
|
13
|
+
Requires-Dist: anyio>=4.13.0
|
|
14
|
+
Requires-Dist: argon2-cffi>=25.1.0
|
|
15
|
+
Requires-Dist: argon2-cffi-bindings>=25.1.0
|
|
16
|
+
Requires-Dist: async-timeout>=5.0.1
|
|
17
|
+
Requires-Dist: bcrypt>=5.0.0
|
|
18
|
+
Requires-Dist: certifi>=2026.4.22
|
|
19
|
+
Requires-Dist: cffi>=2.0.0
|
|
20
|
+
Requires-Dist: cfgv>=3.5.0
|
|
21
|
+
Requires-Dist: charset-normalizer>=3.4.7
|
|
22
|
+
Requires-Dist: cryptography>=48.0.0
|
|
23
|
+
Requires-Dist: distlib>=0.4.0
|
|
24
|
+
Requires-Dist: dnspython>=2.8.0
|
|
25
|
+
Requires-Dist: elastic-transport>=9.4.0
|
|
26
|
+
Requires-Dist: elasticsearch>=9.4.0
|
|
27
|
+
Requires-Dist: fastapi>=0.136.1
|
|
28
|
+
Requires-Dist: filelock>=3.29.0
|
|
29
|
+
Requires-Dist: google-api-core>=2.30.3
|
|
30
|
+
Requires-Dist: google-auth>=2.52.0
|
|
31
|
+
Requires-Dist: google-cloud-appengine-logging>=1.9.0
|
|
32
|
+
Requires-Dist: google-cloud-audit-log>=0.5.0
|
|
33
|
+
Requires-Dist: google-cloud-core>=2.6.0
|
|
34
|
+
Requires-Dist: google-cloud-logging>=3.15.0
|
|
35
|
+
Requires-Dist: google-cloud-pubsub>=2.38.0
|
|
36
|
+
Requires-Dist: google-cloud-storage>=3.10.1
|
|
37
|
+
Requires-Dist: google-crc32c>=1.8.0
|
|
38
|
+
Requires-Dist: google-oauth>=1.0.1
|
|
39
|
+
Requires-Dist: google-resumable-media>=2.9.0
|
|
40
|
+
Requires-Dist: googleapis-common-protos>=1.75.0
|
|
41
|
+
Requires-Dist: greenlet>=3.5.0
|
|
42
|
+
Requires-Dist: grpc-google-iam-v1>=0.14.4
|
|
43
|
+
Requires-Dist: grpcio>=1.80.0
|
|
44
|
+
Requires-Dist: grpcio-status>=1.80.0
|
|
45
|
+
Requires-Dist: h11>=0.16.0
|
|
46
|
+
Requires-Dist: httpcore>=1.0.9
|
|
47
|
+
Requires-Dist: httpx>=0.28.1
|
|
48
|
+
Requires-Dist: identify>=2.6.19
|
|
49
|
+
Requires-Dist: idna>=3.14
|
|
50
|
+
Requires-Dist: importlib_metadata>=8.7.1
|
|
51
|
+
Requires-Dist: minio>=7.2.20
|
|
52
|
+
Requires-Dist: motor>=3.7.1
|
|
53
|
+
Requires-Dist: nexo-crypto>=0.0.40
|
|
54
|
+
Requires-Dist: nexo-database>=0.3.0
|
|
55
|
+
Requires-Dist: nexo-enums>=0.0.40
|
|
56
|
+
Requires-Dist: nexo-logging>=0.0.40
|
|
57
|
+
Requires-Dist: nexo-schemas>=0.3.0
|
|
58
|
+
Requires-Dist: nexo-types>=0.0.40
|
|
59
|
+
Requires-Dist: nexo-utils>=0.0.40
|
|
60
|
+
Requires-Dist: nodeenv>=1.10.0
|
|
61
|
+
Requires-Dist: opentelemetry-api>=1.41.1
|
|
62
|
+
Requires-Dist: opentelemetry-sdk>=1.41.1
|
|
63
|
+
Requires-Dist: opentelemetry-semantic-conventions>=0.62b1
|
|
64
|
+
Requires-Dist: pika>=1.4.0
|
|
65
|
+
Requires-Dist: platformdirs>=4.9.6
|
|
66
|
+
Requires-Dist: pre_commit>=4.6.0
|
|
67
|
+
Requires-Dist: proto-plus>=1.28.0
|
|
68
|
+
Requires-Dist: protobuf>=6.33.6
|
|
69
|
+
Requires-Dist: pyasn1>=0.6.3
|
|
70
|
+
Requires-Dist: pyasn1_modules>=0.4.2
|
|
71
|
+
Requires-Dist: pycparser>=3.0
|
|
72
|
+
Requires-Dist: pycryptodome>=3.23.0
|
|
73
|
+
Requires-Dist: pydantic>=2.13.4
|
|
74
|
+
Requires-Dist: pydantic-settings>=2.14.1
|
|
75
|
+
Requires-Dist: pydantic_core>=2.46.4
|
|
76
|
+
Requires-Dist: PyJWT>=2.12.1
|
|
77
|
+
Requires-Dist: pymongo>=4.17.0
|
|
78
|
+
Requires-Dist: pyOpenSSL>=26.2.0
|
|
79
|
+
Requires-Dist: python-dateutil>=2.9.0.post0
|
|
80
|
+
Requires-Dist: python-discovery>=1.3.0
|
|
81
|
+
Requires-Dist: python-dotenv>=1.2.2
|
|
82
|
+
Requires-Dist: PyYAML>=6.0.3
|
|
83
|
+
Requires-Dist: redis>=7.4.0
|
|
84
|
+
Requires-Dist: requests>=2.33.1
|
|
85
|
+
Requires-Dist: ruff>=0.15.12
|
|
86
|
+
Requires-Dist: six>=1.17.0
|
|
87
|
+
Requires-Dist: sniffio>=1.3.1
|
|
88
|
+
Requires-Dist: SQLAlchemy>=2.0.49
|
|
89
|
+
Requires-Dist: starlette>=1.0.0
|
|
90
|
+
Requires-Dist: typing-inspection>=0.4.2
|
|
91
|
+
Requires-Dist: typing_extensions>=4.15.0
|
|
92
|
+
Requires-Dist: ua-parser>=1.0.2
|
|
93
|
+
Requires-Dist: ua-parser-builtins>=202605
|
|
94
|
+
Requires-Dist: urllib3>=2.7.0
|
|
95
|
+
Requires-Dist: user-agents>=2.2.0
|
|
96
|
+
Requires-Dist: virtualenv>=21.3.1
|
|
97
|
+
Requires-Dist: zipp>=3.23.1
|
|
98
|
+
Dynamic: license-file
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: nexo-storage
|
|
3
|
+
Version: 0.3.0
|
|
4
|
+
Summary: Storage package for Nexo
|
|
5
|
+
Author-email: Agra Bima Yuda <hub.agrayuda@gmail.com>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Requires-Python: >=3.12
|
|
8
|
+
Description-Content-Type: text/markdown
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Requires-Dist: aioredis>=2.0.1
|
|
11
|
+
Requires-Dist: annotated-doc>=0.0.4
|
|
12
|
+
Requires-Dist: annotated-types>=0.7.0
|
|
13
|
+
Requires-Dist: anyio>=4.13.0
|
|
14
|
+
Requires-Dist: argon2-cffi>=25.1.0
|
|
15
|
+
Requires-Dist: argon2-cffi-bindings>=25.1.0
|
|
16
|
+
Requires-Dist: async-timeout>=5.0.1
|
|
17
|
+
Requires-Dist: bcrypt>=5.0.0
|
|
18
|
+
Requires-Dist: certifi>=2026.4.22
|
|
19
|
+
Requires-Dist: cffi>=2.0.0
|
|
20
|
+
Requires-Dist: cfgv>=3.5.0
|
|
21
|
+
Requires-Dist: charset-normalizer>=3.4.7
|
|
22
|
+
Requires-Dist: cryptography>=48.0.0
|
|
23
|
+
Requires-Dist: distlib>=0.4.0
|
|
24
|
+
Requires-Dist: dnspython>=2.8.0
|
|
25
|
+
Requires-Dist: elastic-transport>=9.4.0
|
|
26
|
+
Requires-Dist: elasticsearch>=9.4.0
|
|
27
|
+
Requires-Dist: fastapi>=0.136.1
|
|
28
|
+
Requires-Dist: filelock>=3.29.0
|
|
29
|
+
Requires-Dist: google-api-core>=2.30.3
|
|
30
|
+
Requires-Dist: google-auth>=2.52.0
|
|
31
|
+
Requires-Dist: google-cloud-appengine-logging>=1.9.0
|
|
32
|
+
Requires-Dist: google-cloud-audit-log>=0.5.0
|
|
33
|
+
Requires-Dist: google-cloud-core>=2.6.0
|
|
34
|
+
Requires-Dist: google-cloud-logging>=3.15.0
|
|
35
|
+
Requires-Dist: google-cloud-pubsub>=2.38.0
|
|
36
|
+
Requires-Dist: google-cloud-storage>=3.10.1
|
|
37
|
+
Requires-Dist: google-crc32c>=1.8.0
|
|
38
|
+
Requires-Dist: google-oauth>=1.0.1
|
|
39
|
+
Requires-Dist: google-resumable-media>=2.9.0
|
|
40
|
+
Requires-Dist: googleapis-common-protos>=1.75.0
|
|
41
|
+
Requires-Dist: greenlet>=3.5.0
|
|
42
|
+
Requires-Dist: grpc-google-iam-v1>=0.14.4
|
|
43
|
+
Requires-Dist: grpcio>=1.80.0
|
|
44
|
+
Requires-Dist: grpcio-status>=1.80.0
|
|
45
|
+
Requires-Dist: h11>=0.16.0
|
|
46
|
+
Requires-Dist: httpcore>=1.0.9
|
|
47
|
+
Requires-Dist: httpx>=0.28.1
|
|
48
|
+
Requires-Dist: identify>=2.6.19
|
|
49
|
+
Requires-Dist: idna>=3.14
|
|
50
|
+
Requires-Dist: importlib_metadata>=8.7.1
|
|
51
|
+
Requires-Dist: minio>=7.2.20
|
|
52
|
+
Requires-Dist: motor>=3.7.1
|
|
53
|
+
Requires-Dist: nexo-crypto>=0.0.40
|
|
54
|
+
Requires-Dist: nexo-database>=0.3.0
|
|
55
|
+
Requires-Dist: nexo-enums>=0.0.40
|
|
56
|
+
Requires-Dist: nexo-logging>=0.0.40
|
|
57
|
+
Requires-Dist: nexo-schemas>=0.3.0
|
|
58
|
+
Requires-Dist: nexo-types>=0.0.40
|
|
59
|
+
Requires-Dist: nexo-utils>=0.0.40
|
|
60
|
+
Requires-Dist: nodeenv>=1.10.0
|
|
61
|
+
Requires-Dist: opentelemetry-api>=1.41.1
|
|
62
|
+
Requires-Dist: opentelemetry-sdk>=1.41.1
|
|
63
|
+
Requires-Dist: opentelemetry-semantic-conventions>=0.62b1
|
|
64
|
+
Requires-Dist: pika>=1.4.0
|
|
65
|
+
Requires-Dist: platformdirs>=4.9.6
|
|
66
|
+
Requires-Dist: pre_commit>=4.6.0
|
|
67
|
+
Requires-Dist: proto-plus>=1.28.0
|
|
68
|
+
Requires-Dist: protobuf>=6.33.6
|
|
69
|
+
Requires-Dist: pyasn1>=0.6.3
|
|
70
|
+
Requires-Dist: pyasn1_modules>=0.4.2
|
|
71
|
+
Requires-Dist: pycparser>=3.0
|
|
72
|
+
Requires-Dist: pycryptodome>=3.23.0
|
|
73
|
+
Requires-Dist: pydantic>=2.13.4
|
|
74
|
+
Requires-Dist: pydantic-settings>=2.14.1
|
|
75
|
+
Requires-Dist: pydantic_core>=2.46.4
|
|
76
|
+
Requires-Dist: PyJWT>=2.12.1
|
|
77
|
+
Requires-Dist: pymongo>=4.17.0
|
|
78
|
+
Requires-Dist: pyOpenSSL>=26.2.0
|
|
79
|
+
Requires-Dist: python-dateutil>=2.9.0.post0
|
|
80
|
+
Requires-Dist: python-discovery>=1.3.0
|
|
81
|
+
Requires-Dist: python-dotenv>=1.2.2
|
|
82
|
+
Requires-Dist: PyYAML>=6.0.3
|
|
83
|
+
Requires-Dist: redis>=7.4.0
|
|
84
|
+
Requires-Dist: requests>=2.33.1
|
|
85
|
+
Requires-Dist: ruff>=0.15.12
|
|
86
|
+
Requires-Dist: six>=1.17.0
|
|
87
|
+
Requires-Dist: sniffio>=1.3.1
|
|
88
|
+
Requires-Dist: SQLAlchemy>=2.0.49
|
|
89
|
+
Requires-Dist: starlette>=1.0.0
|
|
90
|
+
Requires-Dist: typing-inspection>=0.4.2
|
|
91
|
+
Requires-Dist: typing_extensions>=4.15.0
|
|
92
|
+
Requires-Dist: ua-parser>=1.0.2
|
|
93
|
+
Requires-Dist: ua-parser-builtins>=202605
|
|
94
|
+
Requires-Dist: urllib3>=2.7.0
|
|
95
|
+
Requires-Dist: user-agents>=2.2.0
|
|
96
|
+
Requires-Dist: virtualenv>=21.3.1
|
|
97
|
+
Requires-Dist: zipp>=3.23.1
|
|
98
|
+
Dynamic: license-file
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
LICENSE
|
|
2
|
+
pyproject.toml
|
|
3
|
+
nexo_storage.egg-info/PKG-INFO
|
|
4
|
+
nexo_storage.egg-info/SOURCES.txt
|
|
5
|
+
nexo_storage.egg-info/dependency_links.txt
|
|
6
|
+
nexo_storage.egg-info/requires.txt
|
|
7
|
+
nexo_storage.egg-info/top_level.txt
|
|
8
|
+
src/__init__.py
|
|
9
|
+
src/common.py
|
|
10
|
+
src/config.py
|
|
11
|
+
src/enums.py
|
|
12
|
+
src/google.py
|
|
13
|
+
src/minio.py
|
|
14
|
+
src/schemas.py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
aioredis>=2.0.1
|
|
2
|
+
annotated-doc>=0.0.4
|
|
3
|
+
annotated-types>=0.7.0
|
|
4
|
+
anyio>=4.13.0
|
|
5
|
+
argon2-cffi>=25.1.0
|
|
6
|
+
argon2-cffi-bindings>=25.1.0
|
|
7
|
+
async-timeout>=5.0.1
|
|
8
|
+
bcrypt>=5.0.0
|
|
9
|
+
certifi>=2026.4.22
|
|
10
|
+
cffi>=2.0.0
|
|
11
|
+
cfgv>=3.5.0
|
|
12
|
+
charset-normalizer>=3.4.7
|
|
13
|
+
cryptography>=48.0.0
|
|
14
|
+
distlib>=0.4.0
|
|
15
|
+
dnspython>=2.8.0
|
|
16
|
+
elastic-transport>=9.4.0
|
|
17
|
+
elasticsearch>=9.4.0
|
|
18
|
+
fastapi>=0.136.1
|
|
19
|
+
filelock>=3.29.0
|
|
20
|
+
google-api-core>=2.30.3
|
|
21
|
+
google-auth>=2.52.0
|
|
22
|
+
google-cloud-appengine-logging>=1.9.0
|
|
23
|
+
google-cloud-audit-log>=0.5.0
|
|
24
|
+
google-cloud-core>=2.6.0
|
|
25
|
+
google-cloud-logging>=3.15.0
|
|
26
|
+
google-cloud-pubsub>=2.38.0
|
|
27
|
+
google-cloud-storage>=3.10.1
|
|
28
|
+
google-crc32c>=1.8.0
|
|
29
|
+
google-oauth>=1.0.1
|
|
30
|
+
google-resumable-media>=2.9.0
|
|
31
|
+
googleapis-common-protos>=1.75.0
|
|
32
|
+
greenlet>=3.5.0
|
|
33
|
+
grpc-google-iam-v1>=0.14.4
|
|
34
|
+
grpcio>=1.80.0
|
|
35
|
+
grpcio-status>=1.80.0
|
|
36
|
+
h11>=0.16.0
|
|
37
|
+
httpcore>=1.0.9
|
|
38
|
+
httpx>=0.28.1
|
|
39
|
+
identify>=2.6.19
|
|
40
|
+
idna>=3.14
|
|
41
|
+
importlib_metadata>=8.7.1
|
|
42
|
+
minio>=7.2.20
|
|
43
|
+
motor>=3.7.1
|
|
44
|
+
nexo-crypto>=0.0.40
|
|
45
|
+
nexo-database>=0.3.0
|
|
46
|
+
nexo-enums>=0.0.40
|
|
47
|
+
nexo-logging>=0.0.40
|
|
48
|
+
nexo-schemas>=0.3.0
|
|
49
|
+
nexo-types>=0.0.40
|
|
50
|
+
nexo-utils>=0.0.40
|
|
51
|
+
nodeenv>=1.10.0
|
|
52
|
+
opentelemetry-api>=1.41.1
|
|
53
|
+
opentelemetry-sdk>=1.41.1
|
|
54
|
+
opentelemetry-semantic-conventions>=0.62b1
|
|
55
|
+
pika>=1.4.0
|
|
56
|
+
platformdirs>=4.9.6
|
|
57
|
+
pre_commit>=4.6.0
|
|
58
|
+
proto-plus>=1.28.0
|
|
59
|
+
protobuf>=6.33.6
|
|
60
|
+
pyasn1>=0.6.3
|
|
61
|
+
pyasn1_modules>=0.4.2
|
|
62
|
+
pycparser>=3.0
|
|
63
|
+
pycryptodome>=3.23.0
|
|
64
|
+
pydantic>=2.13.4
|
|
65
|
+
pydantic-settings>=2.14.1
|
|
66
|
+
pydantic_core>=2.46.4
|
|
67
|
+
PyJWT>=2.12.1
|
|
68
|
+
pymongo>=4.17.0
|
|
69
|
+
pyOpenSSL>=26.2.0
|
|
70
|
+
python-dateutil>=2.9.0.post0
|
|
71
|
+
python-discovery>=1.3.0
|
|
72
|
+
python-dotenv>=1.2.2
|
|
73
|
+
PyYAML>=6.0.3
|
|
74
|
+
redis>=7.4.0
|
|
75
|
+
requests>=2.33.1
|
|
76
|
+
ruff>=0.15.12
|
|
77
|
+
six>=1.17.0
|
|
78
|
+
sniffio>=1.3.1
|
|
79
|
+
SQLAlchemy>=2.0.49
|
|
80
|
+
starlette>=1.0.0
|
|
81
|
+
typing-inspection>=0.4.2
|
|
82
|
+
typing_extensions>=4.15.0
|
|
83
|
+
ua-parser>=1.0.2
|
|
84
|
+
ua-parser-builtins>=202605
|
|
85
|
+
urllib3>=2.7.0
|
|
86
|
+
user-agents>=2.2.0
|
|
87
|
+
virtualenv>=21.3.1
|
|
88
|
+
zipp>=3.23.1
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
nexo
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "nexo-storage"
|
|
7
|
+
version = "0.3.0"
|
|
8
|
+
description = "Storage package for Nexo"
|
|
9
|
+
authors = [
|
|
10
|
+
{ name = "Agra Bima Yuda", email = "hub.agrayuda@gmail.com" }
|
|
11
|
+
]
|
|
12
|
+
license = "MIT"
|
|
13
|
+
readme = "README.md"
|
|
14
|
+
requires-python = ">=3.12"
|
|
15
|
+
dependencies = [
|
|
16
|
+
"aioredis>=2.0.1",
|
|
17
|
+
"annotated-doc>=0.0.4",
|
|
18
|
+
"annotated-types>=0.7.0",
|
|
19
|
+
"anyio>=4.13.0",
|
|
20
|
+
"argon2-cffi>=25.1.0",
|
|
21
|
+
"argon2-cffi-bindings>=25.1.0",
|
|
22
|
+
"async-timeout>=5.0.1",
|
|
23
|
+
"bcrypt>=5.0.0",
|
|
24
|
+
"certifi>=2026.4.22",
|
|
25
|
+
"cffi>=2.0.0",
|
|
26
|
+
"cfgv>=3.5.0",
|
|
27
|
+
"charset-normalizer>=3.4.7",
|
|
28
|
+
"cryptography>=48.0.0",
|
|
29
|
+
"distlib>=0.4.0",
|
|
30
|
+
"dnspython>=2.8.0",
|
|
31
|
+
"elastic-transport>=9.4.0",
|
|
32
|
+
"elasticsearch>=9.4.0",
|
|
33
|
+
"fastapi>=0.136.1",
|
|
34
|
+
"filelock>=3.29.0",
|
|
35
|
+
"google-api-core>=2.30.3",
|
|
36
|
+
"google-auth>=2.52.0",
|
|
37
|
+
"google-cloud-appengine-logging>=1.9.0",
|
|
38
|
+
"google-cloud-audit-log>=0.5.0",
|
|
39
|
+
"google-cloud-core>=2.6.0",
|
|
40
|
+
"google-cloud-logging>=3.15.0",
|
|
41
|
+
"google-cloud-pubsub>=2.38.0",
|
|
42
|
+
"google-cloud-storage>=3.10.1",
|
|
43
|
+
"google-crc32c>=1.8.0",
|
|
44
|
+
"google-oauth>=1.0.1",
|
|
45
|
+
"google-resumable-media>=2.9.0",
|
|
46
|
+
"googleapis-common-protos>=1.75.0",
|
|
47
|
+
"greenlet>=3.5.0",
|
|
48
|
+
"grpc-google-iam-v1>=0.14.4",
|
|
49
|
+
"grpcio>=1.80.0",
|
|
50
|
+
"grpcio-status>=1.80.0",
|
|
51
|
+
"h11>=0.16.0",
|
|
52
|
+
"httpcore>=1.0.9",
|
|
53
|
+
"httpx>=0.28.1",
|
|
54
|
+
"identify>=2.6.19",
|
|
55
|
+
"idna>=3.14",
|
|
56
|
+
"importlib_metadata>=8.7.1",
|
|
57
|
+
"minio>=7.2.20",
|
|
58
|
+
"motor>=3.7.1",
|
|
59
|
+
"nexo-crypto>=0.0.40",
|
|
60
|
+
"nexo-database>=0.3.0",
|
|
61
|
+
"nexo-enums>=0.0.40",
|
|
62
|
+
"nexo-logging>=0.0.40",
|
|
63
|
+
"nexo-schemas>=0.3.0",
|
|
64
|
+
"nexo-types>=0.0.40",
|
|
65
|
+
"nexo-utils>=0.0.40",
|
|
66
|
+
"nodeenv>=1.10.0",
|
|
67
|
+
"opentelemetry-api>=1.41.1",
|
|
68
|
+
"opentelemetry-sdk>=1.41.1",
|
|
69
|
+
"opentelemetry-semantic-conventions>=0.62b1",
|
|
70
|
+
"pika>=1.4.0",
|
|
71
|
+
"platformdirs>=4.9.6",
|
|
72
|
+
"pre_commit>=4.6.0",
|
|
73
|
+
"proto-plus>=1.28.0",
|
|
74
|
+
"protobuf>=6.33.6",
|
|
75
|
+
"pyasn1>=0.6.3",
|
|
76
|
+
"pyasn1_modules>=0.4.2",
|
|
77
|
+
"pycparser>=3.0",
|
|
78
|
+
"pycryptodome>=3.23.0",
|
|
79
|
+
"pydantic>=2.13.4",
|
|
80
|
+
"pydantic-settings>=2.14.1",
|
|
81
|
+
"pydantic_core>=2.46.4",
|
|
82
|
+
"PyJWT>=2.12.1",
|
|
83
|
+
"pymongo>=4.17.0",
|
|
84
|
+
"pyOpenSSL>=26.2.0",
|
|
85
|
+
"python-dateutil>=2.9.0.post0",
|
|
86
|
+
"python-discovery>=1.3.0",
|
|
87
|
+
"python-dotenv>=1.2.2",
|
|
88
|
+
"PyYAML>=6.0.3",
|
|
89
|
+
"redis>=7.4.0",
|
|
90
|
+
"requests>=2.33.1",
|
|
91
|
+
"ruff>=0.15.12",
|
|
92
|
+
"six>=1.17.0",
|
|
93
|
+
"sniffio>=1.3.1",
|
|
94
|
+
"SQLAlchemy>=2.0.49",
|
|
95
|
+
"starlette>=1.0.0",
|
|
96
|
+
"typing-inspection>=0.4.2",
|
|
97
|
+
"typing_extensions>=4.15.0",
|
|
98
|
+
"ua-parser>=1.0.2",
|
|
99
|
+
"ua-parser-builtins>=202605",
|
|
100
|
+
"urllib3>=2.7.0",
|
|
101
|
+
"user-agents>=2.2.0",
|
|
102
|
+
"virtualenv>=21.3.1",
|
|
103
|
+
"zipp>=3.23.1",
|
|
104
|
+
]
|
|
105
|
+
|
|
106
|
+
[tool.setuptools]
|
|
107
|
+
packages = [
|
|
108
|
+
"nexo.storage",
|
|
109
|
+
]
|
|
110
|
+
|
|
111
|
+
[tool.setuptools.package-data]
|
|
112
|
+
"nexo.storage" = ["*.json", "*.yaml"]
|
|
113
|
+
|
|
114
|
+
[tool.setuptools.package-dir]
|
|
115
|
+
"nexo.storage" = "src"
|
|
116
|
+
|
|
117
|
+
[tool.ruff]
|
|
118
|
+
target-version = "py312"
|
|
119
|
+
|
|
120
|
+
[tool.ruff.format]
|
|
121
|
+
quote-style = "double"
|
|
122
|
+
indent-style = "space"
|
|
123
|
+
|
|
124
|
+
[tool.ruff.lint]
|
|
125
|
+
select = ["E", "F", "I", "B", "UP"]
|
|
126
|
+
ignore = []
|
|
File without changes
|
|
@@ -0,0 +1,335 @@
|
|
|
1
|
+
from copy import deepcopy
|
|
2
|
+
from datetime import UTC, datetime, timedelta
|
|
3
|
+
from uuid import uuid4
|
|
4
|
+
|
|
5
|
+
from nexo.database.enums import Connection
|
|
6
|
+
from nexo.database.handlers import RedisHandler
|
|
7
|
+
from nexo.database.utils import build_cache_key
|
|
8
|
+
from nexo.enums.expiration import Expiration
|
|
9
|
+
from nexo.logging.config import LogConfig
|
|
10
|
+
from nexo.logging.logger import Client as ClientLogger
|
|
11
|
+
from nexo.schemas.application import ApplicationContext, OptApplicationContext
|
|
12
|
+
from nexo.schemas.bus import ListOfAnyPublishers
|
|
13
|
+
from nexo.schemas.connection import OptConnectionContext
|
|
14
|
+
from nexo.schemas.error.enums import ErrorCode
|
|
15
|
+
from nexo.schemas.exception.factory import MaleoExceptionFactory
|
|
16
|
+
from nexo.schemas.extractor import extract_details
|
|
17
|
+
from nexo.schemas.operation.context import DefaultOperationContext
|
|
18
|
+
from nexo.schemas.operation.enums import OperationType
|
|
19
|
+
from nexo.schemas.operation.enums import Target as OperationTarget
|
|
20
|
+
from nexo.schemas.operation.mixins import Timestamp
|
|
21
|
+
from nexo.schemas.operation.resource import (
|
|
22
|
+
CreateResourceOperationAction,
|
|
23
|
+
CreateSingleResourceOperation,
|
|
24
|
+
ReadResourceOperationAction,
|
|
25
|
+
ReadSingleResourceOperation,
|
|
26
|
+
)
|
|
27
|
+
from nexo.schemas.resource import AggregateField, ResourceIdentifier
|
|
28
|
+
from nexo.schemas.response import SingleDataResponse
|
|
29
|
+
from nexo.schemas.security.authentication import OptAnyAuthentication
|
|
30
|
+
from nexo.schemas.security.authorization import OptAnyAuthorization
|
|
31
|
+
from nexo.schemas.security.impersonation import OptImpersonation
|
|
32
|
+
from nexo.types.misc import OptPathOrStr
|
|
33
|
+
from nexo.types.string import OptStr
|
|
34
|
+
from nexo.types.uuid import OptUUID
|
|
35
|
+
from nexo.utils.loaders.google import load, validate
|
|
36
|
+
|
|
37
|
+
from google.cloud.storage import Bucket, Client
|
|
38
|
+
from google.oauth2.service_account import Credentials
|
|
39
|
+
|
|
40
|
+
from .common import STORAGE_RESOURCE
|
|
41
|
+
from .enums import Disposition
|
|
42
|
+
from .schemas import Asset, BucketName
|
|
43
|
+
|
|
44
|
+
GOOGLE_CLOUD_STORAGE_RESOURCE = deepcopy(STORAGE_RESOURCE)
|
|
45
|
+
GOOGLE_CLOUD_STORAGE_RESOURCE.identifiers.append(
|
|
46
|
+
ResourceIdentifier(
|
|
47
|
+
key="google_cloud_storage",
|
|
48
|
+
name="Google Cloud Storage",
|
|
49
|
+
slug="google-cloud-storage",
|
|
50
|
+
)
|
|
51
|
+
)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
class GoogleCloudStorageConfig(BucketName):
|
|
55
|
+
pass
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
OptGoogleCloudStorageConfig = GoogleCloudStorageConfig | None
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
class GoogleCloudStorage:
|
|
62
|
+
def __init__(
|
|
63
|
+
self,
|
|
64
|
+
config: GoogleCloudStorageConfig,
|
|
65
|
+
log_config: LogConfig,
|
|
66
|
+
*,
|
|
67
|
+
application_context: OptApplicationContext = None,
|
|
68
|
+
credentials: Credentials | None = None,
|
|
69
|
+
credentials_path: OptPathOrStr = None,
|
|
70
|
+
redis: RedisHandler,
|
|
71
|
+
publishers: ListOfAnyPublishers,
|
|
72
|
+
) -> None:
|
|
73
|
+
self._config = config
|
|
74
|
+
|
|
75
|
+
self._application_context = (
|
|
76
|
+
application_context
|
|
77
|
+
if application_context is not None
|
|
78
|
+
else ApplicationContext.new()
|
|
79
|
+
)
|
|
80
|
+
|
|
81
|
+
self._logger = ClientLogger(
|
|
82
|
+
log_config,
|
|
83
|
+
environment=self._application_context.environment,
|
|
84
|
+
service_key=self._application_context.service_key,
|
|
85
|
+
client_key=GOOGLE_CLOUD_STORAGE_RESOURCE.identifiers[-1].key,
|
|
86
|
+
)
|
|
87
|
+
|
|
88
|
+
self._operation_context = DefaultOperationContext.CLIENT_SERVICE_INTERNAL.value
|
|
89
|
+
|
|
90
|
+
if (credentials is None and credentials_path is None) or (
|
|
91
|
+
credentials is not None and credentials_path is not None
|
|
92
|
+
):
|
|
93
|
+
raise ValueError(
|
|
94
|
+
"Only either 'credentials' and 'credentials_path' must be given"
|
|
95
|
+
)
|
|
96
|
+
|
|
97
|
+
if credentials is not None:
|
|
98
|
+
self._credentials = credentials
|
|
99
|
+
else:
|
|
100
|
+
self._credentials = load(credentials_path)
|
|
101
|
+
|
|
102
|
+
validate(self._credentials)
|
|
103
|
+
|
|
104
|
+
self._client = Client(credentials=self._credentials)
|
|
105
|
+
|
|
106
|
+
self._bucket = self._client.lookup_bucket(bucket_name=self._config.bucket_name)
|
|
107
|
+
if self._bucket is None:
|
|
108
|
+
self._client.close()
|
|
109
|
+
raise ValueError(f"Bucket '{self._config.bucket_name}' does not exist.")
|
|
110
|
+
|
|
111
|
+
self._redis = redis
|
|
112
|
+
self._namespace = self._redis.config.build_client_namespace(
|
|
113
|
+
GOOGLE_CLOUD_STORAGE_RESOURCE.aggregate(AggregateField.KEY, sep=":"),
|
|
114
|
+
client=GOOGLE_CLOUD_STORAGE_RESOURCE.identifiers[-1].key,
|
|
115
|
+
)
|
|
116
|
+
|
|
117
|
+
self._publishers = publishers
|
|
118
|
+
|
|
119
|
+
self._root_location = self._application_context.service_key
|
|
120
|
+
|
|
121
|
+
@property
|
|
122
|
+
def bucket(self) -> Bucket:
|
|
123
|
+
if self._bucket is None:
|
|
124
|
+
raise ValueError("Bucket has not been initialized.")
|
|
125
|
+
return self._bucket
|
|
126
|
+
|
|
127
|
+
async def upload(
|
|
128
|
+
self,
|
|
129
|
+
content: bytes,
|
|
130
|
+
location: str,
|
|
131
|
+
content_type: OptStr = None,
|
|
132
|
+
*,
|
|
133
|
+
operation_id: OptUUID = None,
|
|
134
|
+
is_ai: bool = False,
|
|
135
|
+
connection_context: OptConnectionContext = None,
|
|
136
|
+
authentication: OptAnyAuthentication = None,
|
|
137
|
+
authorization: OptAnyAuthorization = None,
|
|
138
|
+
impersonation: OptImpersonation = None,
|
|
139
|
+
root_location_override: OptStr = None,
|
|
140
|
+
make_public: bool = False,
|
|
141
|
+
expiration: Expiration = Expiration.EXP_15MN,
|
|
142
|
+
) -> SingleDataResponse[Asset, None]:
|
|
143
|
+
operation_id = operation_id if operation_id is not None else uuid4()
|
|
144
|
+
operation_action = CreateResourceOperationAction()
|
|
145
|
+
|
|
146
|
+
executed_at = datetime.now(tz=UTC)
|
|
147
|
+
|
|
148
|
+
if root_location_override is None or (
|
|
149
|
+
isinstance(root_location_override, str) and len(root_location_override) <= 0
|
|
150
|
+
):
|
|
151
|
+
blob_name = f"{self._root_location}/{location}"
|
|
152
|
+
else:
|
|
153
|
+
blob_name = f"{root_location_override}/{location}"
|
|
154
|
+
|
|
155
|
+
resource = deepcopy(GOOGLE_CLOUD_STORAGE_RESOURCE)
|
|
156
|
+
resource.details = {"location": location, "blob_name": blob_name}
|
|
157
|
+
|
|
158
|
+
try:
|
|
159
|
+
blob = self.bucket.blob(blob_name=blob_name)
|
|
160
|
+
blob.upload_from_string(content, content_type=content_type or "text/plain")
|
|
161
|
+
|
|
162
|
+
if make_public:
|
|
163
|
+
blob.make_public()
|
|
164
|
+
url = blob.public_url
|
|
165
|
+
else:
|
|
166
|
+
url = blob.generate_signed_url(
|
|
167
|
+
version="v4",
|
|
168
|
+
expiration=timedelta(seconds=expiration.value),
|
|
169
|
+
method="GET",
|
|
170
|
+
)
|
|
171
|
+
|
|
172
|
+
client = self._redis.manager.client.get(Connection.ASYNC)
|
|
173
|
+
cache_key = build_cache_key(blob_name, namespace=self._namespace)
|
|
174
|
+
await client.set(name=cache_key, value=url, ex=expiration.value)
|
|
175
|
+
|
|
176
|
+
asset = Asset(url=url)
|
|
177
|
+
response = SingleDataResponse[Asset, None].new(data=asset)
|
|
178
|
+
operation = CreateSingleResourceOperation[Asset, None](
|
|
179
|
+
application_context=self._application_context,
|
|
180
|
+
id=operation_id,
|
|
181
|
+
is_ai=is_ai,
|
|
182
|
+
context=self._operation_context,
|
|
183
|
+
action=operation_action,
|
|
184
|
+
timestamp=Timestamp.completed_now(executed_at),
|
|
185
|
+
summary=f"Successfully uploaded object to '{location}'",
|
|
186
|
+
connection_context=connection_context,
|
|
187
|
+
authentication=authentication,
|
|
188
|
+
authorization=authorization,
|
|
189
|
+
impersonation=impersonation,
|
|
190
|
+
resource=resource,
|
|
191
|
+
response=response,
|
|
192
|
+
)
|
|
193
|
+
operation.log_and_publish(self._logger, self._publishers)
|
|
194
|
+
return response
|
|
195
|
+
except Exception as e:
|
|
196
|
+
exc = MaleoExceptionFactory.from_code(
|
|
197
|
+
ErrorCode.INTERNAL_SERVER_ERROR,
|
|
198
|
+
details=extract_details(e),
|
|
199
|
+
operation_type=OperationType.RESOURCE,
|
|
200
|
+
application_context=self._application_context,
|
|
201
|
+
operation_id=operation_id,
|
|
202
|
+
is_ai_operation=is_ai,
|
|
203
|
+
operation_context=self._operation_context,
|
|
204
|
+
operation_action=operation_action,
|
|
205
|
+
resource=resource,
|
|
206
|
+
operation_timestamp=Timestamp.completed_now(executed_at),
|
|
207
|
+
operation_summary=(f"Error raised while uploading to '{location}'"),
|
|
208
|
+
connection_context=connection_context,
|
|
209
|
+
authentication=authentication,
|
|
210
|
+
authorization=authorization,
|
|
211
|
+
impersonation=impersonation,
|
|
212
|
+
)
|
|
213
|
+
exc.log_and_publish_operation(self._logger, self._publishers)
|
|
214
|
+
raise exc from e
|
|
215
|
+
|
|
216
|
+
async def generate_signed_url(
|
|
217
|
+
self,
|
|
218
|
+
location: str,
|
|
219
|
+
disposition: Disposition = Disposition.ATTACHMENT,
|
|
220
|
+
filename: OptStr = None,
|
|
221
|
+
*,
|
|
222
|
+
operation_id: OptUUID = None,
|
|
223
|
+
is_ai: bool = False,
|
|
224
|
+
connection_context: OptConnectionContext = None,
|
|
225
|
+
authentication: OptAnyAuthentication = None,
|
|
226
|
+
authorization: OptAnyAuthorization = None,
|
|
227
|
+
impersonation: OptImpersonation = None,
|
|
228
|
+
root_location_override: OptStr = None,
|
|
229
|
+
use_cache: bool = True,
|
|
230
|
+
expiration: Expiration = Expiration.EXP_15MN,
|
|
231
|
+
) -> SingleDataResponse[Asset, None]:
|
|
232
|
+
operation_id = operation_id if operation_id is not None else uuid4()
|
|
233
|
+
operation_action = ReadResourceOperationAction()
|
|
234
|
+
|
|
235
|
+
executed_at = datetime.now(tz=UTC)
|
|
236
|
+
|
|
237
|
+
if root_location_override is None or (
|
|
238
|
+
isinstance(root_location_override, str) and len(root_location_override) <= 0
|
|
239
|
+
):
|
|
240
|
+
blob_name = f"{self._root_location}/{location}"
|
|
241
|
+
else:
|
|
242
|
+
blob_name = f"{root_location_override}/{location}"
|
|
243
|
+
|
|
244
|
+
resource = deepcopy(GOOGLE_CLOUD_STORAGE_RESOURCE)
|
|
245
|
+
resource.details = {"location": location, "blob_name": blob_name}
|
|
246
|
+
|
|
247
|
+
if use_cache:
|
|
248
|
+
client = self._redis.manager.client.get(Connection.ASYNC)
|
|
249
|
+
cache_key = build_cache_key(blob_name, namespace=self._namespace)
|
|
250
|
+
url = await client.get(cache_key)
|
|
251
|
+
if url is not None:
|
|
252
|
+
operation_context = deepcopy(self._operation_context)
|
|
253
|
+
operation_context.target.type = OperationTarget.CACHE
|
|
254
|
+
asset = Asset(url=url)
|
|
255
|
+
response = SingleDataResponse[Asset, None].new(data=asset)
|
|
256
|
+
operation = ReadSingleResourceOperation[Asset, None](
|
|
257
|
+
application_context=self._application_context,
|
|
258
|
+
id=operation_id,
|
|
259
|
+
is_ai=is_ai,
|
|
260
|
+
context=operation_context,
|
|
261
|
+
action=operation_action,
|
|
262
|
+
resource=resource,
|
|
263
|
+
timestamp=Timestamp.completed_now(executed_at),
|
|
264
|
+
summary=f"Retrieved signed url for '{location}' from cache",
|
|
265
|
+
connection_context=connection_context,
|
|
266
|
+
authentication=authentication,
|
|
267
|
+
authorization=authorization,
|
|
268
|
+
impersonation=impersonation,
|
|
269
|
+
response=response,
|
|
270
|
+
)
|
|
271
|
+
operation.log_and_publish(self._logger, self._publishers)
|
|
272
|
+
|
|
273
|
+
return response
|
|
274
|
+
|
|
275
|
+
blob = self.bucket.blob(blob_name=blob_name)
|
|
276
|
+
if not blob.exists():
|
|
277
|
+
exc = MaleoExceptionFactory.from_code(
|
|
278
|
+
ErrorCode.NOT_FOUND,
|
|
279
|
+
operation_type=OperationType.RESOURCE,
|
|
280
|
+
application_context=self._application_context,
|
|
281
|
+
operation_id=operation_id,
|
|
282
|
+
is_ai_operation=is_ai,
|
|
283
|
+
operation_context=self._operation_context,
|
|
284
|
+
operation_action=operation_action,
|
|
285
|
+
resource=resource,
|
|
286
|
+
operation_timestamp=Timestamp.completed_now(executed_at),
|
|
287
|
+
operation_summary=f"Asset '{location}' not found",
|
|
288
|
+
connection_context=connection_context,
|
|
289
|
+
authentication=authentication,
|
|
290
|
+
authorization=authorization,
|
|
291
|
+
impersonation=impersonation,
|
|
292
|
+
)
|
|
293
|
+
exc.log_and_publish_operation(self._logger, self._publishers)
|
|
294
|
+
raise exc
|
|
295
|
+
|
|
296
|
+
if disposition is Disposition.ATTACHMENT:
|
|
297
|
+
if filename is None:
|
|
298
|
+
response_disposition = None
|
|
299
|
+
else:
|
|
300
|
+
response_disposition = (
|
|
301
|
+
f"{Disposition.ATTACHMENT.value}; filename={filename}"
|
|
302
|
+
)
|
|
303
|
+
elif disposition is Disposition.INLINE:
|
|
304
|
+
response_disposition = "inline"
|
|
305
|
+
|
|
306
|
+
url = blob.generate_signed_url(
|
|
307
|
+
expiration=timedelta(seconds=expiration.value),
|
|
308
|
+
response_disposition=response_disposition,
|
|
309
|
+
version="v4",
|
|
310
|
+
)
|
|
311
|
+
|
|
312
|
+
client = self._redis.manager.client.get(Connection.ASYNC)
|
|
313
|
+
cache_key = build_cache_key(blob_name, namespace=self._namespace)
|
|
314
|
+
await client.set(name=cache_key, value=url, ex=expiration.value)
|
|
315
|
+
|
|
316
|
+
asset = Asset(url=url)
|
|
317
|
+
response = SingleDataResponse[Asset, None].new(data=asset)
|
|
318
|
+
operation = ReadSingleResourceOperation[Asset, None](
|
|
319
|
+
application_context=self._application_context,
|
|
320
|
+
id=operation_id,
|
|
321
|
+
is_ai=is_ai,
|
|
322
|
+
context=self._operation_context,
|
|
323
|
+
action=operation_action,
|
|
324
|
+
resource=resource,
|
|
325
|
+
timestamp=Timestamp.completed_now(executed_at),
|
|
326
|
+
summary=f"Successfully generated signed url for asset '{location}'",
|
|
327
|
+
connection_context=connection_context,
|
|
328
|
+
authentication=authentication,
|
|
329
|
+
authorization=authorization,
|
|
330
|
+
impersonation=impersonation,
|
|
331
|
+
response=response,
|
|
332
|
+
)
|
|
333
|
+
operation.log_and_publish(self._logger, self._publishers)
|
|
334
|
+
|
|
335
|
+
return response
|
|
@@ -0,0 +1,326 @@
|
|
|
1
|
+
import io
|
|
2
|
+
from copy import deepcopy
|
|
3
|
+
from datetime import UTC, datetime, timedelta
|
|
4
|
+
from typing import Annotated
|
|
5
|
+
from uuid import uuid4
|
|
6
|
+
|
|
7
|
+
from nexo.database.enums import Connection
|
|
8
|
+
from nexo.database.handlers import RedisHandler
|
|
9
|
+
from nexo.database.utils import build_cache_key
|
|
10
|
+
from nexo.enums.expiration import Expiration
|
|
11
|
+
from nexo.logging.config import LogConfig
|
|
12
|
+
from nexo.logging.logger import Client as ClientLogger
|
|
13
|
+
from nexo.schemas.application import ApplicationContext, OptApplicationContext
|
|
14
|
+
from nexo.schemas.bus import ListOfAnyPublishers
|
|
15
|
+
from nexo.schemas.connection import OptConnectionContext
|
|
16
|
+
from nexo.schemas.error.enums import ErrorCode
|
|
17
|
+
from nexo.schemas.exception.factory import MaleoExceptionFactory
|
|
18
|
+
from nexo.schemas.extractor import extract_details
|
|
19
|
+
from nexo.schemas.operation.context import DefaultOperationContext
|
|
20
|
+
from nexo.schemas.operation.enums import OperationType
|
|
21
|
+
from nexo.schemas.operation.enums import Target as OperationTarget
|
|
22
|
+
from nexo.schemas.operation.mixins import Timestamp
|
|
23
|
+
from nexo.schemas.operation.resource import (
|
|
24
|
+
CreateResourceOperationAction,
|
|
25
|
+
CreateSingleResourceOperation,
|
|
26
|
+
ReadResourceOperationAction,
|
|
27
|
+
ReadSingleResourceOperation,
|
|
28
|
+
)
|
|
29
|
+
from nexo.schemas.resource import AggregateField, ResourceIdentifier
|
|
30
|
+
from nexo.schemas.response import SingleDataResponse
|
|
31
|
+
from nexo.schemas.security.authentication import OptAnyAuthentication
|
|
32
|
+
from nexo.schemas.security.authorization import OptAnyAuthorization
|
|
33
|
+
from nexo.schemas.security.impersonation import OptImpersonation
|
|
34
|
+
from nexo.types.string import OptStr
|
|
35
|
+
from nexo.types.uuid import OptUUID
|
|
36
|
+
from pydantic import Field
|
|
37
|
+
|
|
38
|
+
from minio import Minio
|
|
39
|
+
from minio.error import S3Error
|
|
40
|
+
from minio.helpers import DictType
|
|
41
|
+
|
|
42
|
+
from .common import STORAGE_RESOURCE
|
|
43
|
+
from .enums import Disposition
|
|
44
|
+
from .schemas import Asset, BucketName
|
|
45
|
+
|
|
46
|
+
MINIO_STORAGE_RESOURCE = deepcopy(STORAGE_RESOURCE)
|
|
47
|
+
MINIO_STORAGE_RESOURCE.identifiers.append(
|
|
48
|
+
ResourceIdentifier(
|
|
49
|
+
key="minio",
|
|
50
|
+
name="MinIO",
|
|
51
|
+
slug="minio",
|
|
52
|
+
)
|
|
53
|
+
)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
class MinIOConfig(BucketName):
|
|
57
|
+
endpoint: Annotated[str, Field(..., description="MinIO server endpoint")]
|
|
58
|
+
access_key: Annotated[str, Field(..., description="MinIO access key")]
|
|
59
|
+
secret_key: Annotated[str, Field(..., description="MinIO secret key")]
|
|
60
|
+
secure: Annotated[bool, Field(True, description="Use HTTPS")] = True
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
class MinIOStorage:
|
|
64
|
+
def __init__(
|
|
65
|
+
self,
|
|
66
|
+
config: MinIOConfig,
|
|
67
|
+
log_config: LogConfig,
|
|
68
|
+
*,
|
|
69
|
+
application_context: OptApplicationContext = None,
|
|
70
|
+
redis: RedisHandler,
|
|
71
|
+
publishers: ListOfAnyPublishers,
|
|
72
|
+
) -> None:
|
|
73
|
+
self._config = config
|
|
74
|
+
|
|
75
|
+
self._application_context = (
|
|
76
|
+
application_context
|
|
77
|
+
if application_context is not None
|
|
78
|
+
else ApplicationContext.new()
|
|
79
|
+
)
|
|
80
|
+
|
|
81
|
+
self._logger = ClientLogger(
|
|
82
|
+
log_config,
|
|
83
|
+
environment=self._application_context.environment,
|
|
84
|
+
service_key=self._application_context.service_key,
|
|
85
|
+
client_key=MINIO_STORAGE_RESOURCE.identifiers[-1].key,
|
|
86
|
+
)
|
|
87
|
+
|
|
88
|
+
self._operation_context = DefaultOperationContext.CLIENT_SERVICE_INTERNAL.value
|
|
89
|
+
|
|
90
|
+
self._client = Minio(**self._config.model_dump(exclude={"bucket_name"}))
|
|
91
|
+
|
|
92
|
+
if not self._client.bucket_exists(self._config.bucket_name):
|
|
93
|
+
raise ValueError(f"Bucket '{self._config.bucket_name}' does not exist.")
|
|
94
|
+
|
|
95
|
+
self._redis = redis
|
|
96
|
+
self._namespace = self._redis.config.build_client_namespace(
|
|
97
|
+
MINIO_STORAGE_RESOURCE.aggregate(AggregateField.KEY, sep=":"),
|
|
98
|
+
client=MINIO_STORAGE_RESOURCE.identifiers[-1].key,
|
|
99
|
+
)
|
|
100
|
+
|
|
101
|
+
self._publishers = publishers
|
|
102
|
+
self._root_location = self._application_context.service_key
|
|
103
|
+
|
|
104
|
+
def _resolve_object_name(
|
|
105
|
+
self,
|
|
106
|
+
location: str,
|
|
107
|
+
root_location_override: OptStr = None,
|
|
108
|
+
) -> str:
|
|
109
|
+
if root_location_override is None or (
|
|
110
|
+
isinstance(root_location_override, str) and len(root_location_override) <= 0
|
|
111
|
+
):
|
|
112
|
+
return f"{self._root_location}/{location}"
|
|
113
|
+
return f"{root_location_override}/{location}"
|
|
114
|
+
|
|
115
|
+
async def upload(
|
|
116
|
+
self,
|
|
117
|
+
content: bytes,
|
|
118
|
+
location: str,
|
|
119
|
+
content_type: OptStr = None,
|
|
120
|
+
*,
|
|
121
|
+
operation_id: OptUUID = None,
|
|
122
|
+
is_ai: bool = False,
|
|
123
|
+
connection_context: OptConnectionContext = None,
|
|
124
|
+
authentication: OptAnyAuthentication = None,
|
|
125
|
+
authorization: OptAnyAuthorization = None,
|
|
126
|
+
impersonation: OptImpersonation = None,
|
|
127
|
+
root_location_override: OptStr = None,
|
|
128
|
+
make_public: bool = False,
|
|
129
|
+
expiration: Expiration = Expiration.EXP_15MN,
|
|
130
|
+
) -> SingleDataResponse[Asset, None]:
|
|
131
|
+
operation_id = operation_id if operation_id is not None else uuid4()
|
|
132
|
+
operation_action = CreateResourceOperationAction()
|
|
133
|
+
executed_at = datetime.now(tz=UTC)
|
|
134
|
+
|
|
135
|
+
object_name = self._resolve_object_name(location, root_location_override)
|
|
136
|
+
|
|
137
|
+
resource = deepcopy(MINIO_STORAGE_RESOURCE)
|
|
138
|
+
resource.details = {"location": location, "object_name": object_name}
|
|
139
|
+
|
|
140
|
+
try:
|
|
141
|
+
data = io.BytesIO(content)
|
|
142
|
+
self._client.put_object(
|
|
143
|
+
bucket_name=self._config.bucket_name,
|
|
144
|
+
object_name=object_name,
|
|
145
|
+
data=data,
|
|
146
|
+
length=len(content),
|
|
147
|
+
content_type=content_type or "text/plain",
|
|
148
|
+
)
|
|
149
|
+
|
|
150
|
+
if make_public:
|
|
151
|
+
# MinIO public URL — no signing, direct endpoint access
|
|
152
|
+
scheme = "https" if self._config.secure else "http"
|
|
153
|
+
url = f"{scheme}://{self._config.endpoint}/{self._config.bucket_name}/{object_name}"
|
|
154
|
+
else:
|
|
155
|
+
url = self._client.presigned_get_object(
|
|
156
|
+
bucket_name=self._config.bucket_name,
|
|
157
|
+
object_name=object_name,
|
|
158
|
+
expires=timedelta(seconds=expiration.value),
|
|
159
|
+
)
|
|
160
|
+
|
|
161
|
+
client = self._redis.manager.client.get(Connection.ASYNC)
|
|
162
|
+
cache_key = build_cache_key(object_name, namespace=self._namespace)
|
|
163
|
+
await client.set(name=cache_key, value=url, ex=expiration.value)
|
|
164
|
+
|
|
165
|
+
asset = Asset(url=url)
|
|
166
|
+
response = SingleDataResponse[Asset, None].new(data=asset)
|
|
167
|
+
operation = CreateSingleResourceOperation[Asset, None](
|
|
168
|
+
application_context=self._application_context,
|
|
169
|
+
id=operation_id,
|
|
170
|
+
is_ai=is_ai,
|
|
171
|
+
context=self._operation_context,
|
|
172
|
+
action=operation_action,
|
|
173
|
+
timestamp=Timestamp.completed_now(executed_at),
|
|
174
|
+
summary=f"Successfully uploaded object to '{location}'",
|
|
175
|
+
connection_context=connection_context,
|
|
176
|
+
authentication=authentication,
|
|
177
|
+
authorization=authorization,
|
|
178
|
+
impersonation=impersonation,
|
|
179
|
+
resource=resource,
|
|
180
|
+
response=response,
|
|
181
|
+
)
|
|
182
|
+
operation.log_and_publish(self._logger, self._publishers)
|
|
183
|
+
return response
|
|
184
|
+
|
|
185
|
+
except Exception as e:
|
|
186
|
+
exc = MaleoExceptionFactory.from_code(
|
|
187
|
+
ErrorCode.INTERNAL_SERVER_ERROR,
|
|
188
|
+
details=extract_details(e),
|
|
189
|
+
operation_type=OperationType.RESOURCE,
|
|
190
|
+
application_context=self._application_context,
|
|
191
|
+
operation_id=operation_id,
|
|
192
|
+
is_ai_operation=is_ai,
|
|
193
|
+
operation_context=self._operation_context,
|
|
194
|
+
operation_action=operation_action,
|
|
195
|
+
resource=resource,
|
|
196
|
+
operation_timestamp=Timestamp.completed_now(executed_at),
|
|
197
|
+
operation_summary=f"Error raised while uploading to '{location}'",
|
|
198
|
+
connection_context=connection_context,
|
|
199
|
+
authentication=authentication,
|
|
200
|
+
authorization=authorization,
|
|
201
|
+
impersonation=impersonation,
|
|
202
|
+
)
|
|
203
|
+
exc.log_and_publish_operation(self._logger, self._publishers)
|
|
204
|
+
raise exc from e
|
|
205
|
+
|
|
206
|
+
async def generate_signed_url(
|
|
207
|
+
self,
|
|
208
|
+
location: str,
|
|
209
|
+
disposition: Disposition = Disposition.ATTACHMENT,
|
|
210
|
+
filename: OptStr = None,
|
|
211
|
+
*,
|
|
212
|
+
operation_id: OptUUID = None,
|
|
213
|
+
is_ai: bool = False,
|
|
214
|
+
connection_context: OptConnectionContext = None,
|
|
215
|
+
authentication: OptAnyAuthentication = None,
|
|
216
|
+
authorization: OptAnyAuthorization = None,
|
|
217
|
+
impersonation: OptImpersonation = None,
|
|
218
|
+
root_location_override: OptStr = None,
|
|
219
|
+
use_cache: bool = True,
|
|
220
|
+
expiration: Expiration = Expiration.EXP_15MN,
|
|
221
|
+
) -> SingleDataResponse[Asset, None]:
|
|
222
|
+
operation_id = operation_id if operation_id is not None else uuid4()
|
|
223
|
+
operation_action = ReadResourceOperationAction()
|
|
224
|
+
executed_at = datetime.now(tz=UTC)
|
|
225
|
+
|
|
226
|
+
object_name = self._resolve_object_name(location, root_location_override)
|
|
227
|
+
|
|
228
|
+
resource = deepcopy(MINIO_STORAGE_RESOURCE)
|
|
229
|
+
resource.details = {"location": location, "object_name": object_name}
|
|
230
|
+
|
|
231
|
+
if use_cache:
|
|
232
|
+
client = self._redis.manager.client.get(Connection.ASYNC)
|
|
233
|
+
cache_key = build_cache_key(object_name, namespace=self._namespace)
|
|
234
|
+
url = await client.get(cache_key)
|
|
235
|
+
if url is not None:
|
|
236
|
+
operation_context = deepcopy(self._operation_context)
|
|
237
|
+
operation_context.target.type = OperationTarget.CACHE
|
|
238
|
+
asset = Asset(url=url)
|
|
239
|
+
response = SingleDataResponse[Asset, None].new(data=asset)
|
|
240
|
+
operation = ReadSingleResourceOperation[Asset, None](
|
|
241
|
+
application_context=self._application_context,
|
|
242
|
+
id=operation_id,
|
|
243
|
+
is_ai=is_ai,
|
|
244
|
+
context=operation_context,
|
|
245
|
+
action=operation_action,
|
|
246
|
+
resource=resource,
|
|
247
|
+
timestamp=Timestamp.completed_now(executed_at),
|
|
248
|
+
summary=f"Retrieved signed url for '{location}' from cache",
|
|
249
|
+
connection_context=connection_context,
|
|
250
|
+
authentication=authentication,
|
|
251
|
+
authorization=authorization,
|
|
252
|
+
impersonation=impersonation,
|
|
253
|
+
response=response,
|
|
254
|
+
)
|
|
255
|
+
operation.log_and_publish(self._logger, self._publishers)
|
|
256
|
+
return response
|
|
257
|
+
|
|
258
|
+
# Check object existence via stat
|
|
259
|
+
try:
|
|
260
|
+
self._client.stat_object(self._config.bucket_name, object_name)
|
|
261
|
+
except S3Error as e:
|
|
262
|
+
if e.code == "NoSuchKey":
|
|
263
|
+
exc = MaleoExceptionFactory.from_code(
|
|
264
|
+
ErrorCode.NOT_FOUND,
|
|
265
|
+
operation_type=OperationType.RESOURCE,
|
|
266
|
+
application_context=self._application_context,
|
|
267
|
+
operation_id=operation_id,
|
|
268
|
+
is_ai_operation=is_ai,
|
|
269
|
+
operation_context=self._operation_context,
|
|
270
|
+
operation_action=operation_action,
|
|
271
|
+
resource=resource,
|
|
272
|
+
operation_timestamp=Timestamp.completed_now(executed_at),
|
|
273
|
+
operation_summary=f"Asset '{location}' not found",
|
|
274
|
+
connection_context=connection_context,
|
|
275
|
+
authentication=authentication,
|
|
276
|
+
authorization=authorization,
|
|
277
|
+
impersonation=impersonation,
|
|
278
|
+
)
|
|
279
|
+
exc.log_and_publish_operation(self._logger, self._publishers)
|
|
280
|
+
raise exc from e
|
|
281
|
+
raise
|
|
282
|
+
|
|
283
|
+
# Build response-content-disposition header
|
|
284
|
+
if disposition is Disposition.ATTACHMENT:
|
|
285
|
+
response_disposition = (
|
|
286
|
+
f"attachment; filename={filename}" if filename else None
|
|
287
|
+
)
|
|
288
|
+
else:
|
|
289
|
+
response_disposition = "inline"
|
|
290
|
+
|
|
291
|
+
extra_query_params: DictType | None = (
|
|
292
|
+
{"response-content-disposition": response_disposition}
|
|
293
|
+
if response_disposition
|
|
294
|
+
else None
|
|
295
|
+
)
|
|
296
|
+
|
|
297
|
+
url = self._client.presigned_get_object(
|
|
298
|
+
bucket_name=self._config.bucket_name,
|
|
299
|
+
object_name=object_name,
|
|
300
|
+
expires=timedelta(seconds=expiration.value),
|
|
301
|
+
extra_query_params=extra_query_params,
|
|
302
|
+
)
|
|
303
|
+
|
|
304
|
+
client = self._redis.manager.client.get(Connection.ASYNC)
|
|
305
|
+
cache_key = build_cache_key(object_name, namespace=self._namespace)
|
|
306
|
+
await client.set(name=cache_key, value=url, ex=expiration.value)
|
|
307
|
+
|
|
308
|
+
asset = Asset(url=url)
|
|
309
|
+
response = SingleDataResponse[Asset, None].new(data=asset)
|
|
310
|
+
operation = ReadSingleResourceOperation[Asset, None](
|
|
311
|
+
application_context=self._application_context,
|
|
312
|
+
id=operation_id,
|
|
313
|
+
is_ai=is_ai,
|
|
314
|
+
context=self._operation_context,
|
|
315
|
+
action=operation_action,
|
|
316
|
+
resource=resource,
|
|
317
|
+
timestamp=Timestamp.completed_now(executed_at),
|
|
318
|
+
summary=f"Successfully generated signed url for asset '{location}'",
|
|
319
|
+
connection_context=connection_context,
|
|
320
|
+
authentication=authentication,
|
|
321
|
+
authorization=authorization,
|
|
322
|
+
impersonation=impersonation,
|
|
323
|
+
response=response,
|
|
324
|
+
)
|
|
325
|
+
operation.log_and_publish(self._logger, self._publishers)
|
|
326
|
+
return response
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
from typing import Annotated
|
|
2
|
+
|
|
3
|
+
from pydantic import BaseModel, Field
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class Asset(BaseModel):
|
|
7
|
+
url: Annotated[str, Field(..., description="Asset's URL")]
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class BucketName(BaseModel):
|
|
11
|
+
bucket_name: Annotated[str, Field(..., description="Bucket's Name")]
|