cesiumcloud 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cesiumcloud-0.1.0/PKG-INFO +67 -0
- cesiumcloud-0.1.0/README.md +51 -0
- cesiumcloud-0.1.0/cesiumcloud/__init__.py +6 -0
- cesiumcloud-0.1.0/cesiumcloud/__main__.py +3 -0
- cesiumcloud-0.1.0/cesiumcloud/cli.py +44 -0
- cesiumcloud-0.1.0/cesiumcloud/client.py +382 -0
- cesiumcloud-0.1.0/cesiumcloud.egg-info/PKG-INFO +67 -0
- cesiumcloud-0.1.0/cesiumcloud.egg-info/SOURCES.txt +11 -0
- cesiumcloud-0.1.0/cesiumcloud.egg-info/dependency_links.txt +1 -0
- cesiumcloud-0.1.0/cesiumcloud.egg-info/entry_points.txt +2 -0
- cesiumcloud-0.1.0/cesiumcloud.egg-info/top_level.txt +1 -0
- cesiumcloud-0.1.0/pyproject.toml +30 -0
- cesiumcloud-0.1.0/setup.cfg +4 -0
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: cesiumcloud
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: CesiumCloud cloud storage Python SDK and command-line client
|
|
5
|
+
Author: CesiumCloud
|
|
6
|
+
Project-URL: Homepage, https://cloud.cesiummc.space/
|
|
7
|
+
Project-URL: Documentation, https://api.cesiummc.space/docs
|
|
8
|
+
Keywords: cesiumcloud,cloud,storage,backup
|
|
9
|
+
Classifier: Programming Language :: Python :: 3
|
|
10
|
+
Classifier: Operating System :: OS Independent
|
|
11
|
+
Classifier: Environment :: Console
|
|
12
|
+
Classifier: Topic :: Internet
|
|
13
|
+
Classifier: Topic :: System :: Archiving :: Backup
|
|
14
|
+
Requires-Python: >=3.10
|
|
15
|
+
Description-Content-Type: text/markdown
|
|
16
|
+
|
|
17
|
+
# CesiumCloud for Python
|
|
18
|
+
|
|
19
|
+
CesiumCloud cloud storage client for Python 3.10+.
|
|
20
|
+
|
|
21
|
+
## Installation
|
|
22
|
+
|
|
23
|
+
Install the latest version from PyPI:
|
|
24
|
+
|
|
25
|
+
python -m pip install cesiumcloud
|
|
26
|
+
|
|
27
|
+
Or install the official wheel directly:
|
|
28
|
+
|
|
29
|
+
python3 -m pip install https://cloud.cesiummc.space/clients/cesiumcloud-0.1.0-py3-none-any.whl
|
|
30
|
+
|
|
31
|
+
## Command-line use
|
|
32
|
+
|
|
33
|
+
python -m cesiumcloud authenticate
|
|
34
|
+
python -m cesiumcloud upload "/home/user/Documents"
|
|
35
|
+
python -m cesiumcloud upload "/home/user/dogimages.zip"
|
|
36
|
+
python -m cesiumcloud download "/home/user/Restored"
|
|
37
|
+
python -m cesiumcloud whoami
|
|
38
|
+
|
|
39
|
+
The cesiumcloud-python command is equivalent.
|
|
40
|
+
|
|
41
|
+
## Python SDK
|
|
42
|
+
|
|
43
|
+
from cesiumcloud import CesiumCloudClient
|
|
44
|
+
|
|
45
|
+
client = CesiumCloudClient()
|
|
46
|
+
client.authenticate()
|
|
47
|
+
client.upload("/home/user/dogimages.zip")
|
|
48
|
+
files = client.list_files()
|
|
49
|
+
print([file["name"] for file in files])
|
|
50
|
+
client.download("/home/user/Restored")
|
|
51
|
+
|
|
52
|
+
Authentication uses your CesiumCloud account without copying an API key.
|
|
53
|
+
The Python implementation requires no Node.js runtime or third-party Python dependencies.
|
|
54
|
+
|
|
55
|
+
## Current limitations
|
|
56
|
+
|
|
57
|
+
Data is streamed directly from local files. The Python client does not yet maintain
|
|
58
|
+
restart-safe checkpoints across processes. Existing destination files are not
|
|
59
|
+
silently overwritten. Keep an independent backup of important data.
|
|
60
|
+
|
|
61
|
+
## Access management
|
|
62
|
+
|
|
63
|
+
Revoke credentials from the CesiumCloud dashboard. The logout command removes local credentials only.
|
|
64
|
+
|
|
65
|
+
## Existing accounts
|
|
66
|
+
|
|
67
|
+
CesiumCloud connects to the existing API and continues reading legacy ~/.tencloud/credentials.json for backwards compatibility. Existing TenCloud package installations are not modified.
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
# CesiumCloud for Python
|
|
2
|
+
|
|
3
|
+
CesiumCloud cloud storage client for Python 3.10+.
|
|
4
|
+
|
|
5
|
+
## Installation
|
|
6
|
+
|
|
7
|
+
Install the latest version from PyPI:
|
|
8
|
+
|
|
9
|
+
python -m pip install cesiumcloud
|
|
10
|
+
|
|
11
|
+
Or install the official wheel directly:
|
|
12
|
+
|
|
13
|
+
python3 -m pip install https://cloud.cesiummc.space/clients/cesiumcloud-0.1.0-py3-none-any.whl
|
|
14
|
+
|
|
15
|
+
## Command-line use
|
|
16
|
+
|
|
17
|
+
python -m cesiumcloud authenticate
|
|
18
|
+
python -m cesiumcloud upload "/home/user/Documents"
|
|
19
|
+
python -m cesiumcloud upload "/home/user/dogimages.zip"
|
|
20
|
+
python -m cesiumcloud download "/home/user/Restored"
|
|
21
|
+
python -m cesiumcloud whoami
|
|
22
|
+
|
|
23
|
+
The cesiumcloud-python command is equivalent.
|
|
24
|
+
|
|
25
|
+
## Python SDK
|
|
26
|
+
|
|
27
|
+
from cesiumcloud import CesiumCloudClient
|
|
28
|
+
|
|
29
|
+
client = CesiumCloudClient()
|
|
30
|
+
client.authenticate()
|
|
31
|
+
client.upload("/home/user/dogimages.zip")
|
|
32
|
+
files = client.list_files()
|
|
33
|
+
print([file["name"] for file in files])
|
|
34
|
+
client.download("/home/user/Restored")
|
|
35
|
+
|
|
36
|
+
Authentication uses your CesiumCloud account without copying an API key.
|
|
37
|
+
The Python implementation requires no Node.js runtime or third-party Python dependencies.
|
|
38
|
+
|
|
39
|
+
## Current limitations
|
|
40
|
+
|
|
41
|
+
Data is streamed directly from local files. The Python client does not yet maintain
|
|
42
|
+
restart-safe checkpoints across processes. Existing destination files are not
|
|
43
|
+
silently overwritten. Keep an independent backup of important data.
|
|
44
|
+
|
|
45
|
+
## Access management
|
|
46
|
+
|
|
47
|
+
Revoke credentials from the CesiumCloud dashboard. The logout command removes local credentials only.
|
|
48
|
+
|
|
49
|
+
## Existing accounts
|
|
50
|
+
|
|
51
|
+
CesiumCloud connects to the existing API and continues reading legacy ~/.tencloud/credentials.json for backwards compatibility. Existing TenCloud package installations are not modified.
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
"""CesiumCloud Python SDK (supports existing TenCloud credentials)."""
|
|
2
|
+
from .client import TenCloudClient, TenCloudError, plan_packs
|
|
3
|
+
CesiumCloudClient = TenCloudClient
|
|
4
|
+
CesiumCloudError = TenCloudError
|
|
5
|
+
__all__ = ["CesiumCloudClient", "CesiumCloudError",
|
|
6
|
+
"TenCloudClient", "TenCloudError", "plan_packs"]
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
"""Python entrypoint: python -m cesiumcloud."""
|
|
2
|
+
import argparse
|
|
3
|
+
import sys
|
|
4
|
+
from . import CesiumCloudClient, CesiumCloudError
|
|
5
|
+
|
|
6
|
+
def main():
|
|
7
|
+
parser = argparse.ArgumentParser(prog="cesiumcloud", description="CesiumCloud cloud storage client")
|
|
8
|
+
sub = parser.add_subparsers(dest="command")
|
|
9
|
+
sub.add_parser("authenticate", help="Authorize this computer")
|
|
10
|
+
upload = sub.add_parser("upload", help="Upload a file or folder")
|
|
11
|
+
upload.add_argument("source")
|
|
12
|
+
download = sub.add_parser("download", help="Restore files to a directory")
|
|
13
|
+
download.add_argument("directory")
|
|
14
|
+
sub.add_parser("whoami", help="Display your CesiumCloud account")
|
|
15
|
+
sub.add_parser("logout", help="Remove your saved credential")
|
|
16
|
+
args = parser.parse_args()
|
|
17
|
+
client = CesiumCloudClient()
|
|
18
|
+
if not args.command:
|
|
19
|
+
parser.print_help()
|
|
20
|
+
return
|
|
21
|
+
try:
|
|
22
|
+
if args.command == "authenticate":
|
|
23
|
+
profile = client.authenticate()
|
|
24
|
+
print("Authenticated as", profile.get("email", "CesiumCloud user"))
|
|
25
|
+
elif args.command == "upload":
|
|
26
|
+
def show(current, total):
|
|
27
|
+
percent = int(100 * current / total) if total else 100
|
|
28
|
+
print(f"\rUploading: {percent:3d}%", end="", flush=True)
|
|
29
|
+
done = client.upload(args.source, on_progress=show)
|
|
30
|
+
print(f"\nUpload complete ({done['packs']} transfers).")
|
|
31
|
+
elif args.command == "download":
|
|
32
|
+
restored = client.download(args.directory)
|
|
33
|
+
print(f"Restored {restored['files']} files.")
|
|
34
|
+
elif args.command == "whoami":
|
|
35
|
+
print(client.account().get("email", "CesiumCloud user"))
|
|
36
|
+
else:
|
|
37
|
+
client.logout()
|
|
38
|
+
print("Local authorization removed. Revoke the device through CesiumCloud if needed.")
|
|
39
|
+
except (CesiumCloudError, OSError) as error:
|
|
40
|
+
print(f"CesiumCloud: {error}", file=sys.stderr)
|
|
41
|
+
sys.exit(1)
|
|
42
|
+
|
|
43
|
+
if __name__ == "__main__":
|
|
44
|
+
main()
|
|
@@ -0,0 +1,382 @@
|
|
|
1
|
+
"""CesiumCloud Python client. Python 3.10+, standard library only."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import bisect
|
|
5
|
+
import gzip
|
|
6
|
+
import json
|
|
7
|
+
import os
|
|
8
|
+
import re
|
|
9
|
+
import struct
|
|
10
|
+
import time
|
|
11
|
+
import uuid
|
|
12
|
+
import urllib.error
|
|
13
|
+
import urllib.request
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
from dataclasses import dataclass
|
|
16
|
+
from typing import Callable, Iterator, Optional
|
|
17
|
+
|
|
18
|
+
API_URL = "https://api.cesiummc.space"
|
|
19
|
+
PACK_LIMIT = 64 * 1024**3
|
|
20
|
+
MANIFEST_LIMIT = 2 * 1024**2
|
|
21
|
+
SEGMENT_LIMIT = 10000
|
|
22
|
+
CHUNK_BYTES = 8 * 1024**2
|
|
23
|
+
TOKEN_RE = re.compile(r"^tc_live_[0-9a-f]{64}$")
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class TenCloudError(RuntimeError):
|
|
27
|
+
"""Error raised by a TenCloud client operation."""
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _call(url: str, method: str = "GET", data: bytes | None = None,
|
|
31
|
+
headers: dict[str, str] | None = None, timeout: int = 90):
|
|
32
|
+
req = urllib.request.Request(url, data=data,
|
|
33
|
+
headers={"User-Agent": "TenCloud-Python/0.1", **(headers or {})},
|
|
34
|
+
method=method)
|
|
35
|
+
try:
|
|
36
|
+
with urllib.request.urlopen(req, timeout=timeout) as response:
|
|
37
|
+
return response.status, response.headers, response.read()
|
|
38
|
+
except urllib.error.HTTPError as error:
|
|
39
|
+
with error:
|
|
40
|
+
return error.code, error.headers, error.read()
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _json_response(code: int, body: bytes):
|
|
44
|
+
try:
|
|
45
|
+
payload = json.loads(body.decode("utf-8"))
|
|
46
|
+
except (UnicodeDecodeError, json.JSONDecodeError):
|
|
47
|
+
raise TenCloudError(f"Invalid TenCloud response (HTTP {code})")
|
|
48
|
+
if not (200 <= code < 300):
|
|
49
|
+
error = payload.get("error", payload.get("message", f"HTTP {code}"))
|
|
50
|
+
if isinstance(error, dict):
|
|
51
|
+
error = error.get("message", str(error))
|
|
52
|
+
raise TenCloudError(str(error))
|
|
53
|
+
return payload
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _entries_for_root(root: Path):
|
|
57
|
+
root = root.expanduser().resolve(strict=True)
|
|
58
|
+
if root.is_file():
|
|
59
|
+
files = [(root, root.name)]
|
|
60
|
+
elif root.is_dir():
|
|
61
|
+
files = []
|
|
62
|
+
for directory, folders, names in os.walk(root, followlinks=False):
|
|
63
|
+
folders[:] = sorted(folder for folder in folders
|
|
64
|
+
if not (Path(directory) / folder).is_symlink())
|
|
65
|
+
for filename in sorted(names):
|
|
66
|
+
full = Path(directory) / filename
|
|
67
|
+
if full.is_file() and not full.is_symlink():
|
|
68
|
+
files.append((full, full.relative_to(root).as_posix()))
|
|
69
|
+
else:
|
|
70
|
+
raise TenCloudError("Select a regular file or directory")
|
|
71
|
+
for full, name in files:
|
|
72
|
+
if not name or len(name.encode("utf-8")) > 2048 or "\\" in name \
|
|
73
|
+
or any(part in ("", ".", "..") for part in name.split("/")):
|
|
74
|
+
raise TenCloudError("A selected filename is not supported")
|
|
75
|
+
stat = full.stat()
|
|
76
|
+
yield full, name, stat.st_size, int(stat.st_mtime_ns // 1_000_000)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
@dataclass
|
|
80
|
+
class Pack:
|
|
81
|
+
entries: list[dict]
|
|
82
|
+
paths: list[Path]
|
|
83
|
+
header: bytes
|
|
84
|
+
payload_bytes: int
|
|
85
|
+
|
|
86
|
+
@property
|
|
87
|
+
def total_bytes(self):
|
|
88
|
+
return len(self.header) + self.payload_bytes
|
|
89
|
+
|
|
90
|
+
def read(self, start: int, length: int) -> bytes:
|
|
91
|
+
if start < 0 or length <= 0 or start + length > self.total_bytes or length > CHUNK_BYTES:
|
|
92
|
+
raise TenCloudError("Upload chunk outside pack bounds")
|
|
93
|
+
result = bytearray(length)
|
|
94
|
+
consumed = 0
|
|
95
|
+
if start < len(self.header):
|
|
96
|
+
copy_count = min(length, len(self.header) - start)
|
|
97
|
+
result[:copy_count] = self.header[start:start + copy_count]
|
|
98
|
+
consumed += copy_count
|
|
99
|
+
start += copy_count
|
|
100
|
+
offsets = [e["d"] for e in self.entries]
|
|
101
|
+
while consumed < length:
|
|
102
|
+
offset = start - len(self.header)
|
|
103
|
+
index = max(0, bisect.bisect_right(offsets, offset) - 1)
|
|
104
|
+
while index < len(self.entries) and self.entries[index]["n"] == 0:
|
|
105
|
+
index += 1
|
|
106
|
+
if index >= len(self.entries):
|
|
107
|
+
raise TenCloudError("Upload data cannot be located")
|
|
108
|
+
entry = self.entries[index]
|
|
109
|
+
if not entry["d"] <= offset < entry["d"] + entry["n"]:
|
|
110
|
+
raise TenCloudError("Upload data index gap")
|
|
111
|
+
src = self.paths[index]
|
|
112
|
+
stat = src.stat()
|
|
113
|
+
if stat.st_size != entry["s"] or int(stat.st_mtime_ns // 1_000_000) != entry["m"]:
|
|
114
|
+
raise TenCloudError("Source file changed during upload: " + entry["p"])
|
|
115
|
+
local_start = entry["o"] + offset - entry["d"]
|
|
116
|
+
amount = min(length - consumed, entry["n"] - (offset - entry["d"]))
|
|
117
|
+
with src.open("rb") as stream:
|
|
118
|
+
stream.seek(local_start)
|
|
119
|
+
piece = stream.read(amount)
|
|
120
|
+
if len(piece) != amount:
|
|
121
|
+
raise TenCloudError("Source file changed while uploading")
|
|
122
|
+
result[consumed:consumed + amount] = piece
|
|
123
|
+
consumed += amount
|
|
124
|
+
start += amount
|
|
125
|
+
return bytes(result)
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def _make_pack(entries: list[dict], paths: list[Path], source_bytes: int) -> Pack:
|
|
129
|
+
index = {"v": 1, "sourceBytes": source_bytes, "entries": entries}
|
|
130
|
+
compressed = gzip.compress(json.dumps(index, separators=(",", ":"),
|
|
131
|
+
ensure_ascii=False).encode("utf-8"), mtime=0)
|
|
132
|
+
if len(compressed) > MANIFEST_LIMIT:
|
|
133
|
+
raise TenCloudError("File index exceeds the supported limit")
|
|
134
|
+
header = b"TENPACK1" + struct.pack("<II", len(compressed), len(entries)) + compressed
|
|
135
|
+
return Pack(entries.copy(), paths.copy(), header, source_bytes)
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def plan_packs(source: str | Path) -> Iterator[Pack]:
|
|
139
|
+
entries, paths, payload = [], [], 0
|
|
140
|
+
capacity = PACK_LIMIT - MANIFEST_LIMIT - 16
|
|
141
|
+
for full, name, size, modified in _entries_for_root(Path(source)):
|
|
142
|
+
position = 0
|
|
143
|
+
while position < size or (size == 0 and position == 0):
|
|
144
|
+
if len(entries) >= SEGMENT_LIMIT or payload >= capacity:
|
|
145
|
+
yield _make_pack(entries, paths, payload)
|
|
146
|
+
entries, paths, payload = [], [], 0
|
|
147
|
+
amount = min(size - position, capacity - payload) if size else 0
|
|
148
|
+
if size and amount == 0:
|
|
149
|
+
continue
|
|
150
|
+
entries.append({"p": name, "o": position, "n": amount, "s": size,
|
|
151
|
+
"m": modified, "d": payload})
|
|
152
|
+
paths.append(full)
|
|
153
|
+
payload += amount
|
|
154
|
+
position += amount
|
|
155
|
+
if size == 0:
|
|
156
|
+
break
|
|
157
|
+
if entries:
|
|
158
|
+
yield _make_pack(entries, paths, payload)
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
class TenCloudClient:
|
|
162
|
+
def __init__(self, api_url: str = API_URL, credential_path: str | Path | None = None):
|
|
163
|
+
self.api_url = api_url.rstrip("/")
|
|
164
|
+
self.credential_path = Path(credential_path or
|
|
165
|
+
(Path.home() / ".tencloud" / "credentials.json"))
|
|
166
|
+
self.token: str | None = None
|
|
167
|
+
|
|
168
|
+
def _saved_token(self) -> str | None:
|
|
169
|
+
env_token = os.environ.get("CESIUMCLOUD_API_KEY") or os.environ.get("TENCLOUD_API_KEY")
|
|
170
|
+
if env_token:
|
|
171
|
+
return env_token if TOKEN_RE.fullmatch(env_token) else None
|
|
172
|
+
if self.token:
|
|
173
|
+
return self.token
|
|
174
|
+
try:
|
|
175
|
+
entry = json.loads(self.credential_path.read_text(encoding="utf-8"))
|
|
176
|
+
except (OSError, ValueError):
|
|
177
|
+
return None
|
|
178
|
+
token = entry.get("token", "")
|
|
179
|
+
return token if entry.get("api") == self.api_url and TOKEN_RE.fullmatch(token) else None
|
|
180
|
+
|
|
181
|
+
def _api(self, endpoint: str, data=None, method: str = "GET",
|
|
182
|
+
authenticated: bool = True):
|
|
183
|
+
headers = {"Accept": "application/json"}
|
|
184
|
+
if authenticated:
|
|
185
|
+
token = self._saved_token()
|
|
186
|
+
if not token:
|
|
187
|
+
raise TenCloudError("Sign in first using: python -m tencloud authenticate")
|
|
188
|
+
headers["Authorization"] = "Bearer " + token
|
|
189
|
+
body = None
|
|
190
|
+
if data is not None:
|
|
191
|
+
body = json.dumps(data).encode("utf-8")
|
|
192
|
+
headers["Content-Type"] = "application/json"
|
|
193
|
+
code, _, content = _call(self.api_url + endpoint, method, body, headers)
|
|
194
|
+
return _json_response(code, content)
|
|
195
|
+
|
|
196
|
+
def authenticate(self, on_challenge: Optional[Callable[[str, str], None]] = None,
|
|
197
|
+
poll: float = 7.0):
|
|
198
|
+
start = self._api("/v1/device/start", {"label": os.environ.get("COMPUTERNAME")
|
|
199
|
+
or os.uname().nodename if hasattr(os, "uname")
|
|
200
|
+
else "TenCloud Python"}, "POST", False)
|
|
201
|
+
uri, code = start["verification_uri"], start["user_code"]
|
|
202
|
+
if on_challenge:
|
|
203
|
+
on_challenge(uri, code)
|
|
204
|
+
else:
|
|
205
|
+
print(f"Open {uri}\nEnter the code: {code}\nApprove this device in TenCloud.")
|
|
206
|
+
until = time.monotonic() + start["expires_in"]
|
|
207
|
+
while time.monotonic() < until:
|
|
208
|
+
time.sleep(max(7, start.get("interval", poll)))
|
|
209
|
+
try:
|
|
210
|
+
answer = self._api("/v1/device/token", {"device_code": start["device_code"]},
|
|
211
|
+
"POST", False)
|
|
212
|
+
token = answer["access_token"]
|
|
213
|
+
if not TOKEN_RE.fullmatch(token):
|
|
214
|
+
raise TenCloudError("Invalid authorization credential")
|
|
215
|
+
self.token = token
|
|
216
|
+
self.account()
|
|
217
|
+
self.credential_path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
|
|
218
|
+
self.credential_path.write_text(
|
|
219
|
+
json.dumps({"api": self.api_url, "token": token}), encoding="utf-8")
|
|
220
|
+
if os.name != "nt":
|
|
221
|
+
self.credential_path.chmod(0o600)
|
|
222
|
+
return self.account()
|
|
223
|
+
except TenCloudError as error:
|
|
224
|
+
if str(error) in ("authorization_pending", "slow_down"):
|
|
225
|
+
continue
|
|
226
|
+
raise
|
|
227
|
+
raise TenCloudError("Authorization expired. Try authenticating again.")
|
|
228
|
+
|
|
229
|
+
def logout(self):
|
|
230
|
+
self.credential_path.unlink(missing_ok=True)
|
|
231
|
+
self.token = None
|
|
232
|
+
|
|
233
|
+
def account(self):
|
|
234
|
+
return self._api("/v1/me")
|
|
235
|
+
|
|
236
|
+
def _resume_status(self, url: str, size: int):
|
|
237
|
+
try:
|
|
238
|
+
code, headers, body = _call(url, "PUT", b"",
|
|
239
|
+
{"Content-Range": f"bytes */{size}"}, 120)
|
|
240
|
+
except (TimeoutError, OSError, urllib.error.URLError):
|
|
241
|
+
return None, None
|
|
242
|
+
if code == 308:
|
|
243
|
+
value = headers.get("Range", "")
|
|
244
|
+
match = re.search(r"0-(\d+)$", value)
|
|
245
|
+
return int(match.group(1)) + 1 if match else 0, None
|
|
246
|
+
if code in (200, 201):
|
|
247
|
+
try:
|
|
248
|
+
return size, json.loads(body.decode("utf-8")).get("id")
|
|
249
|
+
except (ValueError, UnicodeDecodeError):
|
|
250
|
+
return size, None
|
|
251
|
+
return None, None
|
|
252
|
+
|
|
253
|
+
def _send_pack(self, pack: Pack, upload_url: str,
|
|
254
|
+
callback: Optional[Callable[[int, int], None]] = None):
|
|
255
|
+
offset, receipt, attempts = 0, None, 0
|
|
256
|
+
while offset < pack.total_bytes:
|
|
257
|
+
amount = min(CHUNK_BYTES, pack.total_bytes - offset)
|
|
258
|
+
chunk = pack.read(offset, amount)
|
|
259
|
+
headers = {"Content-Type": "application/octet-stream",
|
|
260
|
+
"Content-Range": f"bytes {offset}-{offset + amount - 1}/{pack.total_bytes}"}
|
|
261
|
+
try:
|
|
262
|
+
code, h, response = _call(upload_url, "PUT", chunk, headers, 120)
|
|
263
|
+
if code == 308:
|
|
264
|
+
match = re.search(r"0-(\d+)$", h.get("Range", ""))
|
|
265
|
+
offset = int(match.group(1)) + 1 if match else offset + amount
|
|
266
|
+
attempts = 0
|
|
267
|
+
elif 200 <= code < 300:
|
|
268
|
+
try:
|
|
269
|
+
receipt = json.loads(response.decode("utf-8")).get("id")
|
|
270
|
+
except (ValueError, UnicodeDecodeError):
|
|
271
|
+
receipt = None
|
|
272
|
+
offset = pack.total_bytes
|
|
273
|
+
else:
|
|
274
|
+
raise TenCloudError(f"Upload transfer HTTP {code}")
|
|
275
|
+
except (OSError, urllib.error.URLError, TenCloudError):
|
|
276
|
+
attempts += 1
|
|
277
|
+
if attempts > 5:
|
|
278
|
+
raise TenCloudError("Upload interrupted; retry the command")
|
|
279
|
+
position, completed_id = self._resume_status(upload_url, pack.total_bytes)
|
|
280
|
+
if position is None:
|
|
281
|
+
time.sleep(attempts * 1.5)
|
|
282
|
+
continue
|
|
283
|
+
offset = position
|
|
284
|
+
if completed_id:
|
|
285
|
+
receipt = completed_id
|
|
286
|
+
if offset < pack.total_bytes:
|
|
287
|
+
time.sleep(0.5)
|
|
288
|
+
if callback:
|
|
289
|
+
callback(min(pack.total_bytes - 1, offset), pack.total_bytes)
|
|
290
|
+
return receipt
|
|
291
|
+
|
|
292
|
+
def upload(self, source: str | Path, on_progress: Optional[Callable[[int, int], None]] = None):
|
|
293
|
+
count = 0
|
|
294
|
+
for plan in plan_packs(source):
|
|
295
|
+
count += 1
|
|
296
|
+
created = self._api("/v1/packs", {"sizeBytes": plan.total_bytes,
|
|
297
|
+
"sourceBytes": plan.payload_bytes,
|
|
298
|
+
"segments": len(plan.entries),
|
|
299
|
+
"requestId": str(uuid.uuid4())}, "POST")
|
|
300
|
+
if created.get("alreadyCommitted"):
|
|
301
|
+
continue
|
|
302
|
+
pack_id = created["packId"]
|
|
303
|
+
file_id = self._send_pack(plan, created["uploadUrl"], on_progress)
|
|
304
|
+
result = self._api("/v1/packs/" + pack_id + "/commit",
|
|
305
|
+
{"driveFileId": file_id} if file_id else {}, "POST")
|
|
306
|
+
if not result.get("ok"):
|
|
307
|
+
raise TenCloudError("Upload confirmation failed")
|
|
308
|
+
if on_progress:
|
|
309
|
+
on_progress(plan.total_bytes, plan.total_bytes)
|
|
310
|
+
return {"packs": count}
|
|
311
|
+
|
|
312
|
+
upload_file = upload
|
|
313
|
+
upload_folder = upload
|
|
314
|
+
|
|
315
|
+
def list_files(self):
|
|
316
|
+
pieces = []
|
|
317
|
+
cursor = {"pack": 0, "offset": 0}
|
|
318
|
+
while cursor:
|
|
319
|
+
result = self._api("/v1/virtual-files?pack=" + str(cursor["pack"]) +
|
|
320
|
+
"&offset=" + str(cursor["offset"]))
|
|
321
|
+
pieces.extend(result["files"])
|
|
322
|
+
cursor = result.get("next")
|
|
323
|
+
groups = []
|
|
324
|
+
by_name = {}
|
|
325
|
+
for part in pieces:
|
|
326
|
+
key = (part["name"], part["size_bytes"], part["modifiedAt"])
|
|
327
|
+
group = by_name.get(key)
|
|
328
|
+
if group is None or (part["fileOffset"] == 0 and
|
|
329
|
+
any(p["fileOffset"] == 0 for p in group["parts"])):
|
|
330
|
+
group = {"name": part["name"], "size": part["size_bytes"], "parts": []}
|
|
331
|
+
groups.append(group)
|
|
332
|
+
by_name[key] = group
|
|
333
|
+
group["parts"].append(part)
|
|
334
|
+
for group in groups:
|
|
335
|
+
group["parts"].sort(key=lambda p: p["fileOffset"])
|
|
336
|
+
return groups
|
|
337
|
+
|
|
338
|
+
def download(self, output_directory: str | Path):
|
|
339
|
+
target_root = Path(output_directory).expanduser().resolve()
|
|
340
|
+
target_root.mkdir(parents=True, exist_ok=True)
|
|
341
|
+
root_real = target_root.resolve()
|
|
342
|
+
count = 0
|
|
343
|
+
for group in self.list_files():
|
|
344
|
+
parts = group["parts"]
|
|
345
|
+
pos = 0
|
|
346
|
+
if not all(p["fileOffset"] == (0 if i == 0 else
|
|
347
|
+
parts[i - 1]["fileOffset"] + parts[i - 1]["part_bytes"])
|
|
348
|
+
for i, p in enumerate(parts)):
|
|
349
|
+
raise TenCloudError("Cannot restore an incomplete file: " + group["name"])
|
|
350
|
+
if sum(p["part_bytes"] for p in parts) != group["size"]:
|
|
351
|
+
raise TenCloudError("Cannot restore an incomplete file: " + group["name"])
|
|
352
|
+
name = group["name"]
|
|
353
|
+
dest = target_root.joinpath(*name.split("/"))
|
|
354
|
+
dest.parent.mkdir(parents=True, exist_ok=True)
|
|
355
|
+
if root_real not in dest.resolve().parents or dest.is_symlink():
|
|
356
|
+
raise TenCloudError("Unsafe output file path")
|
|
357
|
+
# Existing local files are not overwritten by default.
|
|
358
|
+
with dest.open("xb") as output:
|
|
359
|
+
for segment in parts:
|
|
360
|
+
if segment["part_bytes"] == 0:
|
|
361
|
+
continue
|
|
362
|
+
token = self._saved_token()
|
|
363
|
+
url = self.api_url + "/v1/virtual-files/" + segment["packId"] + \
|
|
364
|
+
"/segments/" + str(segment["segmentIndex"])
|
|
365
|
+
request = urllib.request.Request(url, headers={
|
|
366
|
+
"Authorization": "Bearer " + token,
|
|
367
|
+
"User-Agent": "TenCloud-Python/0.1"}, method="GET")
|
|
368
|
+
with urllib.request.urlopen(request, timeout=120) as stream:
|
|
369
|
+
copied = 0
|
|
370
|
+
while True:
|
|
371
|
+
chunk = stream.read(1024 * 1024)
|
|
372
|
+
if not chunk:
|
|
373
|
+
break
|
|
374
|
+
copied += len(chunk)
|
|
375
|
+
output.write(chunk)
|
|
376
|
+
if copied != segment["part_bytes"]:
|
|
377
|
+
raise TenCloudError("Incomplete download: " + name)
|
|
378
|
+
pos += copied
|
|
379
|
+
if pos != group["size"]:
|
|
380
|
+
raise TenCloudError("Incomplete download: " + name)
|
|
381
|
+
count += 1
|
|
382
|
+
return {"files": count}
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: cesiumcloud
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: CesiumCloud cloud storage Python SDK and command-line client
|
|
5
|
+
Author: CesiumCloud
|
|
6
|
+
Project-URL: Homepage, https://cloud.cesiummc.space/
|
|
7
|
+
Project-URL: Documentation, https://api.cesiummc.space/docs
|
|
8
|
+
Keywords: cesiumcloud,cloud,storage,backup
|
|
9
|
+
Classifier: Programming Language :: Python :: 3
|
|
10
|
+
Classifier: Operating System :: OS Independent
|
|
11
|
+
Classifier: Environment :: Console
|
|
12
|
+
Classifier: Topic :: Internet
|
|
13
|
+
Classifier: Topic :: System :: Archiving :: Backup
|
|
14
|
+
Requires-Python: >=3.10
|
|
15
|
+
Description-Content-Type: text/markdown
|
|
16
|
+
|
|
17
|
+
# CesiumCloud for Python
|
|
18
|
+
|
|
19
|
+
CesiumCloud cloud storage client for Python 3.10+.
|
|
20
|
+
|
|
21
|
+
## Installation
|
|
22
|
+
|
|
23
|
+
Install the latest version from PyPI:
|
|
24
|
+
|
|
25
|
+
python -m pip install cesiumcloud
|
|
26
|
+
|
|
27
|
+
Or install the official wheel directly:
|
|
28
|
+
|
|
29
|
+
python3 -m pip install https://cloud.cesiummc.space/clients/cesiumcloud-0.1.0-py3-none-any.whl
|
|
30
|
+
|
|
31
|
+
## Command-line use
|
|
32
|
+
|
|
33
|
+
python -m cesiumcloud authenticate
|
|
34
|
+
python -m cesiumcloud upload "/home/user/Documents"
|
|
35
|
+
python -m cesiumcloud upload "/home/user/dogimages.zip"
|
|
36
|
+
python -m cesiumcloud download "/home/user/Restored"
|
|
37
|
+
python -m cesiumcloud whoami
|
|
38
|
+
|
|
39
|
+
The cesiumcloud-python command is equivalent.
|
|
40
|
+
|
|
41
|
+
## Python SDK
|
|
42
|
+
|
|
43
|
+
from cesiumcloud import CesiumCloudClient
|
|
44
|
+
|
|
45
|
+
client = CesiumCloudClient()
|
|
46
|
+
client.authenticate()
|
|
47
|
+
client.upload("/home/user/dogimages.zip")
|
|
48
|
+
files = client.list_files()
|
|
49
|
+
print([file["name"] for file in files])
|
|
50
|
+
client.download("/home/user/Restored")
|
|
51
|
+
|
|
52
|
+
Authentication uses your CesiumCloud account without copying an API key.
|
|
53
|
+
The Python implementation requires no Node.js runtime or third-party Python dependencies.
|
|
54
|
+
|
|
55
|
+
## Current limitations
|
|
56
|
+
|
|
57
|
+
Data is streamed directly from local files. The Python client does not yet maintain
|
|
58
|
+
restart-safe checkpoints across processes. Existing destination files are not
|
|
59
|
+
silently overwritten. Keep an independent backup of important data.
|
|
60
|
+
|
|
61
|
+
## Access management
|
|
62
|
+
|
|
63
|
+
Revoke credentials from the CesiumCloud dashboard. The logout command removes local credentials only.
|
|
64
|
+
|
|
65
|
+
## Existing accounts
|
|
66
|
+
|
|
67
|
+
CesiumCloud connects to the existing API and continues reading legacy ~/.tencloud/credentials.json for backwards compatibility. Existing TenCloud package installations are not modified.
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
README.md
|
|
2
|
+
pyproject.toml
|
|
3
|
+
cesiumcloud/__init__.py
|
|
4
|
+
cesiumcloud/__main__.py
|
|
5
|
+
cesiumcloud/cli.py
|
|
6
|
+
cesiumcloud/client.py
|
|
7
|
+
cesiumcloud.egg-info/PKG-INFO
|
|
8
|
+
cesiumcloud.egg-info/SOURCES.txt
|
|
9
|
+
cesiumcloud.egg-info/dependency_links.txt
|
|
10
|
+
cesiumcloud.egg-info/entry_points.txt
|
|
11
|
+
cesiumcloud.egg-info/top_level.txt
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
cesiumcloud
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=70", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "cesiumcloud"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "CesiumCloud cloud storage Python SDK and command-line client"
|
|
9
|
+
requires-python = ">=3.10"
|
|
10
|
+
authors = [{name = "CesiumCloud"}]
|
|
11
|
+
readme = "README.md"
|
|
12
|
+
keywords = ["cesiumcloud", "cloud", "storage", "backup"]
|
|
13
|
+
classifiers = [
|
|
14
|
+
"Programming Language :: Python :: 3",
|
|
15
|
+
"Operating System :: OS Independent",
|
|
16
|
+
"Environment :: Console",
|
|
17
|
+
"Topic :: Internet",
|
|
18
|
+
"Topic :: System :: Archiving :: Backup"
|
|
19
|
+
]
|
|
20
|
+
|
|
21
|
+
[project.scripts]
|
|
22
|
+
cesiumcloud = "cesiumcloud.cli:main"
|
|
23
|
+
|
|
24
|
+
[project.urls]
|
|
25
|
+
Homepage = "https://cloud.cesiummc.space/"
|
|
26
|
+
Documentation = "https://api.cesiummc.space/docs"
|
|
27
|
+
|
|
28
|
+
[tool.setuptools.packages.find]
|
|
29
|
+
where = ["."]
|
|
30
|
+
include = ["cesiumcloud*"]
|