swarmauri_storage_s3_over_sftp 0.11.0.dev5__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- swarmauri_storage_s3_over_sftp-0.11.0.dev5/LICENSE +3 -0
- swarmauri_storage_s3_over_sftp-0.11.0.dev5/PKG-INFO +89 -0
- swarmauri_storage_s3_over_sftp-0.11.0.dev5/README.md +62 -0
- swarmauri_storage_s3_over_sftp-0.11.0.dev5/pyproject.toml +89 -0
- swarmauri_storage_s3_over_sftp-0.11.0.dev5/swarmauri_storage_s3_over_sftp/__init__.py +8 -0
- swarmauri_storage_s3_over_sftp-0.11.0.dev5/swarmauri_storage_s3_over_sftp/s3_over_sftp_storage_adapter.py +297 -0
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: swarmauri_storage_s3_over_sftp
|
|
3
|
+
Version: 0.11.0.dev5
|
|
4
|
+
Summary: S3-style Swarmauri storage adapter backed by SFTP
|
|
5
|
+
License-Expression: Apache-2.0
|
|
6
|
+
License-File: LICENSE
|
|
7
|
+
Keywords: swarmauri,sdk,standards,storage,s3,sftp,object-storage,ssh,remote-filesystem
|
|
8
|
+
Author: Jacob Stewart
|
|
9
|
+
Author-email: jacob@swarmauri.com
|
|
10
|
+
Requires-Python: >=3.10,<3.13
|
|
11
|
+
Classifier: Development Status :: 1 - Planning
|
|
12
|
+
Classifier: License :: OSI Approved :: Apache Software License
|
|
13
|
+
Classifier: Operating System :: OS Independent
|
|
14
|
+
Classifier: Programming Language :: Python
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
20
|
+
Requires-Dist: paramiko (>=3.4,<4.0)
|
|
21
|
+
Requires-Dist: pycparser (<3)
|
|
22
|
+
Requires-Dist: swarmauri_base
|
|
23
|
+
Requires-Dist: swarmauri_core
|
|
24
|
+
Requires-Dist: swarmauri_standard
|
|
25
|
+
Description-Content-Type: text/markdown
|
|
26
|
+
|
|
27
|
+

|
|
28
|
+
|
|
29
|
+
<p align="center">
|
|
30
|
+
<a href="https://pepy.tech/project/swarmauri_storage_s3_over_sftp/">
|
|
31
|
+
<img src="https://static.pepy.tech/badge/swarmauri_storage_s3_over_sftp/month" alt="PyPI - Downloads"/></a>
|
|
32
|
+
<a href="https://hits.sh/github.com/swarmauri/swarmauri-sdk/tree/master/pkgs/standards/swarmauri_storage_s3_over_sftp/">
|
|
33
|
+
<img alt="Hits" src="https://hits.sh/github.com/swarmauri/swarmauri-sdk/tree/master/pkgs/standards/swarmauri_storage_s3_over_sftp.svg"/></a>
|
|
34
|
+
<a href="https://pypi.org/project/swarmauri_storage_s3_over_sftp/">
|
|
35
|
+
<img src="https://img.shields.io/badge/python-3.10%20%7C%203.11%20%7C%203.12-blue" alt="Supported Python Versions"/></a>
|
|
36
|
+
<a href="https://pypi.org/project/swarmauri_storage_s3_over_sftp/">
|
|
37
|
+
<img src="https://img.shields.io/pypi/l/swarmauri_storage_s3_over_sftp" alt="License"/></a>
|
|
38
|
+
<a href="https://pypi.org/project/swarmauri_storage_s3_over_sftp/">
|
|
39
|
+
<img src="https://img.shields.io/pypi/v/swarmauri_storage_s3_over_sftp?label=swarmauri_storage_s3_over_sftp&color=green" alt="Release Version"/></a>
|
|
40
|
+
<a href="https://discord.gg/N4UpBuQv8T">
|
|
41
|
+
<img src="https://img.shields.io/badge/Discord-Join%20Chat-5865F2?logo=discord&logoColor=white" alt="Discord"/></a>
|
|
42
|
+
</p>
|
|
43
|
+
|
|
44
|
+
# Swarmauri Storage S3 Over SFTP
|
|
45
|
+
|
|
46
|
+
S3-style object storage adapter that persists bucket-prefixed objects through SFTP.
|
|
47
|
+
|
|
48
|
+
## Features
|
|
49
|
+
|
|
50
|
+
- Exposes Swarmauri storage adapter methods with S3-like `bucket/key` addressing over an SFTP directory tree.
|
|
51
|
+
- Supports password, key-file, and injected Paramiko transport or SFTP clients for managed connection lifecycles.
|
|
52
|
+
- Provides upload, download, recursive push and pull, byte-range reads, deletion, and bucket directory creation.
|
|
53
|
+
- Uses Swarmauri storage-key normalization helpers to reject unsafe keys before resolving remote or local paths.
|
|
54
|
+
- Provides discoverable `swarmauri.storage_adapters` and `peagen.plugins.storage_adapters` entry points.
|
|
55
|
+
|
|
56
|
+
## Installation
|
|
57
|
+
|
|
58
|
+
Install this package with `uv` or `pip`.
|
|
59
|
+
|
|
60
|
+
```bash
|
|
61
|
+
uv add swarmauri_storage_s3_over_sftp
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
```bash
|
|
65
|
+
pip install swarmauri_storage_s3_over_sftp
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
## Usage
|
|
69
|
+
|
|
70
|
+
Use this adapter when a workflow expects object-storage semantics but the backing system is reachable only through SFTP.
|
|
71
|
+
|
|
72
|
+
```python
|
|
73
|
+
from swarmauri_storage_s3_over_sftp import S3OverSftpStorageAdapter
|
|
74
|
+
|
|
75
|
+
adapter = S3OverSftpStorageAdapter.from_uri(
|
|
76
|
+
"s3+sftp://deploy@example.com/artifacts/runs?root_dir=/srv/object-store"
|
|
77
|
+
)
|
|
78
|
+
|
|
79
|
+
uri = adapter.put_blob("latest/result.json", b'{"status":"ok"}')
|
|
80
|
+
payload = adapter.get_blob("latest/result.json")
|
|
81
|
+
|
|
82
|
+
print(uri)
|
|
83
|
+
print(payload)
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
The URI path is interpreted as `/<bucket>/<prefix>`. The optional `root_dir` query value points at the remote SFTP directory under which bucket directories are stored.
|
|
87
|
+
|
|
88
|
+
License: Apache-2.0. See `LICENSE`.
|
|
89
|
+
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+

|
|
2
|
+
|
|
3
|
+
<p align="center">
|
|
4
|
+
<a href="https://pepy.tech/project/swarmauri_storage_s3_over_sftp/">
|
|
5
|
+
<img src="https://static.pepy.tech/badge/swarmauri_storage_s3_over_sftp/month" alt="PyPI - Downloads"/></a>
|
|
6
|
+
<a href="https://hits.sh/github.com/swarmauri/swarmauri-sdk/tree/master/pkgs/standards/swarmauri_storage_s3_over_sftp/">
|
|
7
|
+
<img alt="Hits" src="https://hits.sh/github.com/swarmauri/swarmauri-sdk/tree/master/pkgs/standards/swarmauri_storage_s3_over_sftp.svg"/></a>
|
|
8
|
+
<a href="https://pypi.org/project/swarmauri_storage_s3_over_sftp/">
|
|
9
|
+
<img src="https://img.shields.io/badge/python-3.10%20%7C%203.11%20%7C%203.12-blue" alt="Supported Python Versions"/></a>
|
|
10
|
+
<a href="https://pypi.org/project/swarmauri_storage_s3_over_sftp/">
|
|
11
|
+
<img src="https://img.shields.io/pypi/l/swarmauri_storage_s3_over_sftp" alt="License"/></a>
|
|
12
|
+
<a href="https://pypi.org/project/swarmauri_storage_s3_over_sftp/">
|
|
13
|
+
<img src="https://img.shields.io/pypi/v/swarmauri_storage_s3_over_sftp?label=swarmauri_storage_s3_over_sftp&color=green" alt="Release Version"/></a>
|
|
14
|
+
<a href="https://discord.gg/N4UpBuQv8T">
|
|
15
|
+
<img src="https://img.shields.io/badge/Discord-Join%20Chat-5865F2?logo=discord&logoColor=white" alt="Discord"/></a>
|
|
16
|
+
</p>
|
|
17
|
+
|
|
18
|
+
# Swarmauri Storage S3 Over SFTP
|
|
19
|
+
|
|
20
|
+
S3-style object storage adapter that persists bucket-prefixed objects through SFTP.
|
|
21
|
+
|
|
22
|
+
## Features
|
|
23
|
+
|
|
24
|
+
- Exposes Swarmauri storage adapter methods with S3-like `bucket/key` addressing over an SFTP directory tree.
|
|
25
|
+
- Supports password, key-file, and injected Paramiko transport or SFTP clients for managed connection lifecycles.
|
|
26
|
+
- Provides upload, download, recursive push and pull, byte-range reads, deletion, and bucket directory creation.
|
|
27
|
+
- Uses Swarmauri storage-key normalization helpers to reject unsafe keys before resolving remote or local paths.
|
|
28
|
+
- Provides discoverable `swarmauri.storage_adapters` and `peagen.plugins.storage_adapters` entry points.
|
|
29
|
+
|
|
30
|
+
## Installation
|
|
31
|
+
|
|
32
|
+
Install this package with `uv` or `pip`.
|
|
33
|
+
|
|
34
|
+
```bash
|
|
35
|
+
uv add swarmauri_storage_s3_over_sftp
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
```bash
|
|
39
|
+
pip install swarmauri_storage_s3_over_sftp
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
## Usage
|
|
43
|
+
|
|
44
|
+
Use this adapter when a workflow expects object-storage semantics but the backing system is reachable only through SFTP.
|
|
45
|
+
|
|
46
|
+
```python
|
|
47
|
+
from swarmauri_storage_s3_over_sftp import S3OverSftpStorageAdapter
|
|
48
|
+
|
|
49
|
+
adapter = S3OverSftpStorageAdapter.from_uri(
|
|
50
|
+
"s3+sftp://deploy@example.com/artifacts/runs?root_dir=/srv/object-store"
|
|
51
|
+
)
|
|
52
|
+
|
|
53
|
+
uri = adapter.put_blob("latest/result.json", b'{"status":"ok"}')
|
|
54
|
+
payload = adapter.get_blob("latest/result.json")
|
|
55
|
+
|
|
56
|
+
print(uri)
|
|
57
|
+
print(payload)
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
The URI path is interpreted as `/<bucket>/<prefix>`. The optional `root_dir` query value points at the remote SFTP directory under which bucket directories are stored.
|
|
61
|
+
|
|
62
|
+
License: Apache-2.0. See `LICENSE`.
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "swarmauri_storage_s3_over_sftp"
|
|
3
|
+
version = "0.11.0.dev5"
|
|
4
|
+
authors = [{ name = "Jacob Stewart", email = "jacob@swarmauri.com" }]
|
|
5
|
+
description = "S3-style Swarmauri storage adapter backed by SFTP"
|
|
6
|
+
readme = "README.md"
|
|
7
|
+
license = "Apache-2.0"
|
|
8
|
+
repository = "http://github.com/swarmauri/swarmauri-sdk"
|
|
9
|
+
requires-python = ">=3.10,<3.13"
|
|
10
|
+
classifiers = [
|
|
11
|
+
"Development Status :: 1 - Planning",
|
|
12
|
+
"License :: OSI Approved :: Apache Software License",
|
|
13
|
+
"Operating System :: OS Independent",
|
|
14
|
+
"Programming Language :: Python",
|
|
15
|
+
"Programming Language :: Python :: 3",
|
|
16
|
+
"Programming Language :: Python :: 3 :: Only",
|
|
17
|
+
"Programming Language :: Python :: 3.10",
|
|
18
|
+
"Programming Language :: Python :: 3.11",
|
|
19
|
+
"Programming Language :: Python :: 3.12",
|
|
20
|
+
]
|
|
21
|
+
dependencies = [
|
|
22
|
+
"paramiko>=3.4,<4.0",
|
|
23
|
+
"pycparser<3",
|
|
24
|
+
"swarmauri_core",
|
|
25
|
+
"swarmauri_base",
|
|
26
|
+
"swarmauri_standard",
|
|
27
|
+
]
|
|
28
|
+
keywords = [
|
|
29
|
+
"swarmauri",
|
|
30
|
+
"sdk",
|
|
31
|
+
"standards",
|
|
32
|
+
"storage",
|
|
33
|
+
"s3",
|
|
34
|
+
"sftp",
|
|
35
|
+
"object-storage",
|
|
36
|
+
"ssh",
|
|
37
|
+
"remote-filesystem",
|
|
38
|
+
]
|
|
39
|
+
|
|
40
|
+
[tool.uv.sources]
|
|
41
|
+
swarmauri_core = { workspace = true }
|
|
42
|
+
swarmauri_base = { workspace = true }
|
|
43
|
+
swarmauri_standard = { workspace = true }
|
|
44
|
+
|
|
45
|
+
[project.entry-points."swarmauri.storage_adapters"]
|
|
46
|
+
S3OverSftpStorageAdapter = "swarmauri_storage_s3_over_sftp.s3_over_sftp_storage_adapter:S3OverSftpStorageAdapter"
|
|
47
|
+
|
|
48
|
+
[project.entry-points."peagen.plugins.storage_adapters"]
|
|
49
|
+
s3-over-sftp = "swarmauri_storage_s3_over_sftp.s3_over_sftp_storage_adapter:S3OverSftpStorageAdapter"
|
|
50
|
+
|
|
51
|
+
[tool.pytest.ini_options]
|
|
52
|
+
addopts = "--basetemp=.pytest-tmp"
|
|
53
|
+
norecursedirs = [".uv-cache", "combined", "scripts"]
|
|
54
|
+
markers = [
|
|
55
|
+
"test: standard test",
|
|
56
|
+
"unit: Unit tests",
|
|
57
|
+
"i9n: Integration tests",
|
|
58
|
+
"r8n: Regression tests",
|
|
59
|
+
"timeout: mark test to timeout after X seconds",
|
|
60
|
+
"xpass: Expected passes",
|
|
61
|
+
"xfail: Expected failures",
|
|
62
|
+
"acceptance: Acceptance tests",
|
|
63
|
+
"perf: Performance tests that measure execution time and resource usage",
|
|
64
|
+
"example: Example usage tests",
|
|
65
|
+
]
|
|
66
|
+
timeout = 300
|
|
67
|
+
log_cli = true
|
|
68
|
+
log_cli_level = "INFO"
|
|
69
|
+
log_cli_format = "%(asctime)s [%(levelname)s] %(message)s"
|
|
70
|
+
log_cli_date_format = "%Y-%m-%d %H:%M:%S"
|
|
71
|
+
asyncio_default_fixture_loop_scope = "function"
|
|
72
|
+
|
|
73
|
+
[build-system]
|
|
74
|
+
requires = ["poetry-core>=1.0.0"]
|
|
75
|
+
build-backend = "poetry.core.masonry.api"
|
|
76
|
+
|
|
77
|
+
[dependency-groups]
|
|
78
|
+
dev = [
|
|
79
|
+
"pytest>=8.0",
|
|
80
|
+
"pytest-asyncio>=0.24.0",
|
|
81
|
+
"pytest-xdist>=3.6.1",
|
|
82
|
+
"pytest-json-report>=1.5.0",
|
|
83
|
+
"python-dotenv",
|
|
84
|
+
"httpx>=0.27",
|
|
85
|
+
"flake8>=7.0",
|
|
86
|
+
"pytest-timeout>=2.3.1",
|
|
87
|
+
"ruff>=0.9.9",
|
|
88
|
+
"pytest-benchmark>=4.0.0",
|
|
89
|
+
]
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
try:
|
|
2
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
3
|
+
except ImportError: # pragma: no cover
|
|
4
|
+
from importlib_metadata import PackageNotFoundError, version
|
|
5
|
+
|
|
6
|
+
from .s3_over_sftp_storage_adapter import S3OverSftpStorageAdapter
|
|
7
|
+
|
|
8
|
+
__all__ = ["PackageNotFoundError", "S3OverSftpStorageAdapter", "version"]
|
|
@@ -0,0 +1,297 @@
|
|
|
1
|
+
"""S3-style object storage adapter backed by SFTP."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import io
|
|
6
|
+
import os
|
|
7
|
+
import posixpath
|
|
8
|
+
import shutil
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
from typing import BinaryIO, Literal
|
|
11
|
+
from urllib.parse import parse_qs, unquote, urlparse
|
|
12
|
+
|
|
13
|
+
import paramiko
|
|
14
|
+
from pydantic import SecretStr
|
|
15
|
+
|
|
16
|
+
from swarmauri_base.ComponentBase import ComponentBase
|
|
17
|
+
from swarmauri_base.storage import StorageAdapterBase
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
@ComponentBase.register_type(StorageAdapterBase, "S3OverSftpStorageAdapter")
|
|
21
|
+
class S3OverSftpStorageAdapter(StorageAdapterBase):
|
|
22
|
+
"""Persist S3-like bucket objects below a remote SFTP directory."""
|
|
23
|
+
|
|
24
|
+
type: Literal["S3OverSftpStorageAdapter"] = "S3OverSftpStorageAdapter"
|
|
25
|
+
|
|
26
|
+
def __init__(
|
|
27
|
+
self,
|
|
28
|
+
bucket: str,
|
|
29
|
+
*,
|
|
30
|
+
host: str | None = None,
|
|
31
|
+
port: int = 22,
|
|
32
|
+
username: str | None = None,
|
|
33
|
+
password: SecretStr | str | None = None,
|
|
34
|
+
key_filename: str | os.PathLike | None = None,
|
|
35
|
+
root_dir: str = ".",
|
|
36
|
+
prefix: str = "",
|
|
37
|
+
transport: paramiko.Transport | None = None,
|
|
38
|
+
sftp_client: paramiko.SFTPClient | None = None,
|
|
39
|
+
connect_kwargs: dict | None = None,
|
|
40
|
+
**kwargs,
|
|
41
|
+
) -> None:
|
|
42
|
+
super().__init__(**kwargs)
|
|
43
|
+
self._bucket = self.normalize_key(bucket)
|
|
44
|
+
self._prefix = self.normalize_prefix(prefix)
|
|
45
|
+
self._host = host
|
|
46
|
+
self._port = port
|
|
47
|
+
self._username = username
|
|
48
|
+
self._password = self._secret_value(password)
|
|
49
|
+
self._key_filename = (
|
|
50
|
+
str(key_filename) if key_filename is not None else None
|
|
51
|
+
)
|
|
52
|
+
self._root_dir = self._normalize_remote_root(root_dir)
|
|
53
|
+
self._owns_connection = transport is None and sftp_client is None
|
|
54
|
+
self._owns_sftp = sftp_client is None
|
|
55
|
+
self._transport = transport
|
|
56
|
+
self._ssh_client: paramiko.SSHClient | None = None
|
|
57
|
+
self._sftp = sftp_client or self._open_sftp(connect_kwargs or {})
|
|
58
|
+
|
|
59
|
+
@staticmethod
|
|
60
|
+
def _secret_value(value: SecretStr | str | None) -> str | None:
|
|
61
|
+
if isinstance(value, SecretStr):
|
|
62
|
+
return value.get_secret_value() or None
|
|
63
|
+
return value or None
|
|
64
|
+
|
|
65
|
+
@staticmethod
|
|
66
|
+
def _normalize_remote_root(root_dir: str) -> str:
|
|
67
|
+
root = posixpath.normpath(root_dir or ".")
|
|
68
|
+
return "" if root == "." else root.rstrip("/")
|
|
69
|
+
|
|
70
|
+
def _open_sftp(self, connect_kwargs: dict) -> paramiko.SFTPClient:
|
|
71
|
+
if self._transport is not None:
|
|
72
|
+
return paramiko.SFTPClient.from_transport(self._transport)
|
|
73
|
+
if self._host is None:
|
|
74
|
+
raise ValueError(
|
|
75
|
+
"host is required unless transport or sftp_client is provided"
|
|
76
|
+
)
|
|
77
|
+
|
|
78
|
+
self._ssh_client = paramiko.SSHClient()
|
|
79
|
+
self._ssh_client.load_system_host_keys()
|
|
80
|
+
options = dict(connect_kwargs)
|
|
81
|
+
options.setdefault("hostname", self._host)
|
|
82
|
+
options.setdefault("port", self._port)
|
|
83
|
+
options.setdefault("username", self._username)
|
|
84
|
+
options.setdefault("password", self._password)
|
|
85
|
+
if self._key_filename is not None:
|
|
86
|
+
options.setdefault("key_filename", self._key_filename)
|
|
87
|
+
self._ssh_client.connect(
|
|
88
|
+
**{
|
|
89
|
+
key: value
|
|
90
|
+
for key, value in options.items()
|
|
91
|
+
if value is not None
|
|
92
|
+
}
|
|
93
|
+
)
|
|
94
|
+
return self._ssh_client.open_sftp()
|
|
95
|
+
|
|
96
|
+
def close(self) -> None:
|
|
97
|
+
"""Close owned SFTP resources."""
|
|
98
|
+
if self._owns_sftp:
|
|
99
|
+
self._sftp.close()
|
|
100
|
+
if self._owns_connection and self._ssh_client is not None:
|
|
101
|
+
self._ssh_client.close()
|
|
102
|
+
if self._owns_connection and self._transport is not None:
|
|
103
|
+
self._transport.close()
|
|
104
|
+
|
|
105
|
+
def _remote_join(self, *parts: str, allow_empty: bool = False) -> str:
|
|
106
|
+
key = self.compose_key(*parts, allow_empty=allow_empty)
|
|
107
|
+
path_parts = [part for part in (self._root_dir, key) if part]
|
|
108
|
+
return posixpath.join(*path_parts) if path_parts else "."
|
|
109
|
+
|
|
110
|
+
def _bucket_key(self, key: str, *, allow_empty: bool = False) -> str:
|
|
111
|
+
return self.compose_key(
|
|
112
|
+
self._bucket, self._prefix, key, allow_empty=allow_empty
|
|
113
|
+
)
|
|
114
|
+
|
|
115
|
+
def _remote_path(self, key: str, *, allow_empty: bool = False) -> str:
|
|
116
|
+
return self._remote_join(
|
|
117
|
+
self._bucket_key(key, allow_empty=allow_empty)
|
|
118
|
+
)
|
|
119
|
+
|
|
120
|
+
def _relative_key(self, remote_path: str) -> str:
|
|
121
|
+
base = self._remote_path("", allow_empty=True).rstrip("/")
|
|
122
|
+
rel = (
|
|
123
|
+
remote_path[len(base) :].lstrip("/")
|
|
124
|
+
if base and remote_path.startswith(base)
|
|
125
|
+
else remote_path
|
|
126
|
+
)
|
|
127
|
+
return self.normalize_key(rel, allow_empty=True)
|
|
128
|
+
|
|
129
|
+
@staticmethod
|
|
130
|
+
def _is_missing(exc: OSError) -> bool:
|
|
131
|
+
return getattr(exc, "errno", None) == 2 or "No such file" in str(exc)
|
|
132
|
+
|
|
133
|
+
def _mkdir_p(self, remote_dir: str) -> None:
|
|
134
|
+
if remote_dir in {"", ".", "/"}:
|
|
135
|
+
return
|
|
136
|
+
parts = remote_dir.strip("/").split("/")
|
|
137
|
+
path = "/" if remote_dir.startswith("/") else ""
|
|
138
|
+
for part in parts:
|
|
139
|
+
path = posixpath.join(path, part) if path else part
|
|
140
|
+
try:
|
|
141
|
+
self._sftp.stat(path)
|
|
142
|
+
except OSError as exc:
|
|
143
|
+
if not self._is_missing(exc):
|
|
144
|
+
raise
|
|
145
|
+
self._sftp.mkdir(path)
|
|
146
|
+
|
|
147
|
+
@property
|
|
148
|
+
def root_uri(self) -> str:
|
|
149
|
+
"""Return the base ``s3+sftp://`` URI for this adapter."""
|
|
150
|
+
auth = ""
|
|
151
|
+
if self._username:
|
|
152
|
+
auth = f"{self._username}@"
|
|
153
|
+
host = self._host or "sftp"
|
|
154
|
+
port = f":{self._port}" if self._port != 22 else ""
|
|
155
|
+
key = self.compose_key(self._bucket, self._prefix, allow_empty=True)
|
|
156
|
+
return f"s3+sftp://{auth}{host}{port}/{key.rstrip('/')}/"
|
|
157
|
+
|
|
158
|
+
def upload(self, key: str, data: BinaryIO) -> str:
|
|
159
|
+
"""Upload *data* under *key* and return its object URI."""
|
|
160
|
+
normalized_key = self.normalize_key(key)
|
|
161
|
+
remote_path = self._remote_path(key)
|
|
162
|
+
self._mkdir_p(posixpath.dirname(remote_path))
|
|
163
|
+
with self._sftp.open(remote_path, "wb") as target:
|
|
164
|
+
shutil.copyfileobj(data, target)
|
|
165
|
+
return f"{self.root_uri}{normalized_key}"
|
|
166
|
+
|
|
167
|
+
def download(self, key: str) -> BinaryIO:
|
|
168
|
+
"""Download *key* into an in-memory binary stream."""
|
|
169
|
+
buffer = io.BytesIO()
|
|
170
|
+
try:
|
|
171
|
+
with self._sftp.open(self._remote_path(key), "rb") as source:
|
|
172
|
+
shutil.copyfileobj(source, buffer)
|
|
173
|
+
except OSError as exc:
|
|
174
|
+
if self._is_missing(exc):
|
|
175
|
+
raise FileNotFoundError(key) from exc
|
|
176
|
+
raise
|
|
177
|
+
buffer.seek(0)
|
|
178
|
+
return buffer
|
|
179
|
+
|
|
180
|
+
def upload_dir(self, src: str | os.PathLike, *, prefix: str = "") -> None:
|
|
181
|
+
"""Recursively upload files from *src* under ``prefix``."""
|
|
182
|
+
base = Path(src)
|
|
183
|
+
for path in base.rglob("*"):
|
|
184
|
+
if path.is_file():
|
|
185
|
+
rel = path.relative_to(base).as_posix()
|
|
186
|
+
key = self.compose_key(prefix, rel)
|
|
187
|
+
with path.open("rb") as handle:
|
|
188
|
+
self.upload(key, handle)
|
|
189
|
+
|
|
190
|
+
def iter_prefix(self, prefix: str):
|
|
191
|
+
"""Yield stored keys below ``prefix`` relative to the adapter root."""
|
|
192
|
+
remote_prefix = self._remote_path(
|
|
193
|
+
self.normalize_prefix(prefix), allow_empty=True
|
|
194
|
+
)
|
|
195
|
+
try:
|
|
196
|
+
for remote_path in self._walk(remote_prefix):
|
|
197
|
+
rel = self._relative_key(remote_path)
|
|
198
|
+
if rel:
|
|
199
|
+
yield rel
|
|
200
|
+
except OSError as exc:
|
|
201
|
+
if self._is_missing(exc):
|
|
202
|
+
return
|
|
203
|
+
raise
|
|
204
|
+
|
|
205
|
+
def _walk(self, remote_dir: str):
|
|
206
|
+
for entry in self._sftp.listdir_attr(remote_dir):
|
|
207
|
+
remote_path = posixpath.join(remote_dir, entry.filename)
|
|
208
|
+
if self._is_directory(entry):
|
|
209
|
+
yield from self._walk(remote_path)
|
|
210
|
+
else:
|
|
211
|
+
yield remote_path
|
|
212
|
+
|
|
213
|
+
@staticmethod
|
|
214
|
+
def _is_directory(entry) -> bool:
|
|
215
|
+
import stat
|
|
216
|
+
|
|
217
|
+
return stat.S_ISDIR(entry.st_mode)
|
|
218
|
+
|
|
219
|
+
def download_dir(self, prefix: str, dest_dir: str | os.PathLike) -> None:
|
|
220
|
+
"""Download all stored artifacts under ``prefix`` into ``dest_dir``."""
|
|
221
|
+
dest = Path(dest_dir)
|
|
222
|
+
normalized_prefix = self.normalize_prefix(prefix)
|
|
223
|
+
for rel_key in self.iter_prefix(prefix):
|
|
224
|
+
target_rel = rel_key
|
|
225
|
+
if normalized_prefix and rel_key.startswith(
|
|
226
|
+
f"{normalized_prefix}/"
|
|
227
|
+
):
|
|
228
|
+
target_rel = rel_key[len(normalized_prefix) + 1 :]
|
|
229
|
+
if not target_rel:
|
|
230
|
+
continue
|
|
231
|
+
target = self.download_target_for_key(dest, target_rel)
|
|
232
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
233
|
+
data = self.download(rel_key)
|
|
234
|
+
with target.open("wb") as handle:
|
|
235
|
+
shutil.copyfileobj(data, handle)
|
|
236
|
+
|
|
237
|
+
async def ensure_bucket(self) -> None:
|
|
238
|
+
"""Create the bucket directory when it does not already exist."""
|
|
239
|
+
self._mkdir_p(self._remote_join(self._bucket))
|
|
240
|
+
|
|
241
|
+
async def put_bytes(
|
|
242
|
+
self, object_key: str, data: bytes, content_type: str
|
|
243
|
+
) -> None:
|
|
244
|
+
"""Store raw bytes under *object_key*."""
|
|
245
|
+
del content_type
|
|
246
|
+
self.upload(object_key, io.BytesIO(data))
|
|
247
|
+
|
|
248
|
+
async def get_bytes(self, object_key: str) -> bytes:
|
|
249
|
+
"""Retrieve the stored object as bytes."""
|
|
250
|
+
return self.get_blob(object_key)
|
|
251
|
+
|
|
252
|
+
async def get_range(
|
|
253
|
+
self, object_key: str, start: int, length: int
|
|
254
|
+
) -> bytes:
|
|
255
|
+
"""Retrieve a byte range from the remote object."""
|
|
256
|
+
remote_path = self._remote_path(object_key)
|
|
257
|
+
try:
|
|
258
|
+
size = self._sftp.stat(remote_path).st_size
|
|
259
|
+
parsed_start, parsed_end = self._parse_range(start, length, size)
|
|
260
|
+
with self._sftp.open(remote_path, "rb") as source:
|
|
261
|
+
source.seek(parsed_start)
|
|
262
|
+
return source.read(parsed_end - parsed_start)
|
|
263
|
+
except OSError as exc:
|
|
264
|
+
if self._is_missing(exc):
|
|
265
|
+
raise FileNotFoundError(object_key) from exc
|
|
266
|
+
raise
|
|
267
|
+
|
|
268
|
+
async def remove_object(self, object_key: str) -> None:
|
|
269
|
+
"""Delete ``object_key`` when it exists."""
|
|
270
|
+
try:
|
|
271
|
+
self._sftp.remove(self._remote_path(object_key))
|
|
272
|
+
except OSError as exc:
|
|
273
|
+
if not self._is_missing(exc):
|
|
274
|
+
raise
|
|
275
|
+
|
|
276
|
+
@classmethod
|
|
277
|
+
def from_uri(cls, uri: str) -> "S3OverSftpStorageAdapter":
|
|
278
|
+
"""Instantiate the adapter from an ``s3+sftp://`` URI."""
|
|
279
|
+
parsed = urlparse(uri)
|
|
280
|
+
if parsed.scheme != "s3+sftp":
|
|
281
|
+
raise ValueError("URI must start with s3+sftp://")
|
|
282
|
+
|
|
283
|
+
path_parts = [unquote(part) for part in parsed.path.split("/") if part]
|
|
284
|
+
if not path_parts:
|
|
285
|
+
raise ValueError("URI path must include a bucket")
|
|
286
|
+
|
|
287
|
+
query = parse_qs(parsed.query)
|
|
288
|
+
root_dir = query.get("root_dir", ["."])[0]
|
|
289
|
+
return cls(
|
|
290
|
+
bucket=path_parts[0],
|
|
291
|
+
prefix="/".join(path_parts[1:]),
|
|
292
|
+
host=parsed.hostname,
|
|
293
|
+
port=parsed.port or 22,
|
|
294
|
+
username=unquote(parsed.username) if parsed.username else None,
|
|
295
|
+
password=unquote(parsed.password) if parsed.password else None,
|
|
296
|
+
root_dir=root_dir,
|
|
297
|
+
)
|