pyicloudx 0.13.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pyicloudx-0.13.0/LICENSE +21 -0
- pyicloudx-0.13.0/PKG-INFO +368 -0
- pyicloudx-0.13.0/README.md +316 -0
- pyicloudx-0.13.0/pyproject.toml +48 -0
- pyicloudx-0.13.0/setup.cfg +4 -0
- pyicloudx-0.13.0/src/pyicloudx/__init__.py +137 -0
- pyicloudx-0.13.0/src/pyicloudx/__main__.py +10 -0
- pyicloudx-0.13.0/src/pyicloudx/auth/__init__.py +36 -0
- pyicloudx-0.13.0/src/pyicloudx/auth/base.py +65 -0
- pyicloudx-0.13.0/src/pyicloudx/auth/inline.py +20 -0
- pyicloudx-0.13.0/src/pyicloudx/auth/interactive.py +37 -0
- pyicloudx-0.13.0/src/pyicloudx/auth/resource.py +88 -0
- pyicloudx-0.13.0/src/pyicloudx/auth/url.py +100 -0
- pyicloudx-0.13.0/src/pyicloudx/blobstore.py +171 -0
- pyicloudx-0.13.0/src/pyicloudx/cache.py +95 -0
- pyicloudx-0.13.0/src/pyicloudx/cli.py +488 -0
- pyicloudx-0.13.0/src/pyicloudx/client.py +209 -0
- pyicloudx-0.13.0/src/pyicloudx/cloud.py +335 -0
- pyicloudx-0.13.0/src/pyicloudx/daemon.py +568 -0
- pyicloudx-0.13.0/src/pyicloudx/errors.py +41 -0
- pyicloudx-0.13.0/src/pyicloudx/files.py +175 -0
- pyicloudx-0.13.0/src/pyicloudx/guard.py +297 -0
- pyicloudx-0.13.0/src/pyicloudx/keyring.py +231 -0
- pyicloudx-0.13.0/src/pyicloudx/logging.py +60 -0
- pyicloudx-0.13.0/src/pyicloudx/metrics.py +78 -0
- pyicloudx-0.13.0/src/pyicloudx/numpy_ops.py +84 -0
- pyicloudx-0.13.0/src/pyicloudx/protocol.py +378 -0
- pyicloudx-0.13.0/src/pyicloudx/providers.py +71 -0
- pyicloudx-0.13.0/src/pyicloudx/py.typed +0 -0
- pyicloudx-0.13.0/src/pyicloudx/remote.py +206 -0
- pyicloudx-0.13.0/src/pyicloudx/retry.py +82 -0
- pyicloudx-0.13.0/src/pyicloudx/routing.py +374 -0
- pyicloudx-0.13.0/src/pyicloudx/session.py +61 -0
- pyicloudx-0.13.0/src/pyicloudx/storage.py +89 -0
- pyicloudx-0.13.0/src/pyicloudx/sync.py +120 -0
- pyicloudx-0.13.0/src/pyicloudx/transport.py +102 -0
- pyicloudx-0.13.0/src/pyicloudx.egg-info/PKG-INFO +368 -0
- pyicloudx-0.13.0/src/pyicloudx.egg-info/SOURCES.txt +55 -0
- pyicloudx-0.13.0/src/pyicloudx.egg-info/dependency_links.txt +1 -0
- pyicloudx-0.13.0/src/pyicloudx.egg-info/entry_points.txt +2 -0
- pyicloudx-0.13.0/src/pyicloudx.egg-info/requires.txt +5 -0
- pyicloudx-0.13.0/src/pyicloudx.egg-info/top_level.txt +1 -0
- pyicloudx-0.13.0/tests/test_auth_strategies.py +142 -0
- pyicloudx-0.13.0/tests/test_blobstore.py +73 -0
- pyicloudx-0.13.0/tests/test_cli.py +162 -0
- pyicloudx-0.13.0/tests/test_client.py +241 -0
- pyicloudx-0.13.0/tests/test_cloud.py +104 -0
- pyicloudx-0.13.0/tests/test_daemon.py +139 -0
- pyicloudx-0.13.0/tests/test_extras.py +300 -0
- pyicloudx-0.13.0/tests/test_files.py +122 -0
- pyicloudx-0.13.0/tests/test_guard.py +180 -0
- pyicloudx-0.13.0/tests/test_keyring.py +129 -0
- pyicloudx-0.13.0/tests/test_protocol.py +214 -0
- pyicloudx-0.13.0/tests/test_remote.py +103 -0
- pyicloudx-0.13.0/tests/test_routing.py +204 -0
- pyicloudx-0.13.0/tests/test_session_store.py +89 -0
- pyicloudx-0.13.0/tests/test_sync.py +129 -0
pyicloudx-0.13.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 pyicloudx contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,368 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: pyicloudx
|
|
3
|
+
Version: 0.13.0
|
|
4
|
+
Summary: Vendor-neutral auth strategies, anti-abuse shield, PCP protocol, routing, keyring and a serialized JSON-RPC daemon (stdlib + numpy)
|
|
5
|
+
Author: pyicloudx contributors
|
|
6
|
+
License: MIT License
|
|
7
|
+
|
|
8
|
+
Copyright (c) 2026 pyicloudx contributors
|
|
9
|
+
|
|
10
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
11
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
12
|
+
in the Software without restriction, including without limitation the rights
|
|
13
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
14
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
15
|
+
furnished to do so, subject to the following conditions:
|
|
16
|
+
|
|
17
|
+
The above copyright notice and this permission notice shall be included in all
|
|
18
|
+
copies or substantial portions of the Software.
|
|
19
|
+
|
|
20
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
21
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
22
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
23
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
24
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
25
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
26
|
+
SOFTWARE.
|
|
27
|
+
|
|
28
|
+
Project-URL: Homepage, https://pypi.org/project/pyicloudx/
|
|
29
|
+
Project-URL: Repository, https://example.invalid/pyicloudx
|
|
30
|
+
Keywords: auth,security,2fa,session,rate-limit,protocol,cli
|
|
31
|
+
Classifier: Development Status :: 4 - Beta
|
|
32
|
+
Classifier: Intended Audience :: Developers
|
|
33
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
34
|
+
Classifier: Programming Language :: Python :: 3
|
|
35
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
36
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
37
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
38
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
39
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
40
|
+
Classifier: Operating System :: OS Independent
|
|
41
|
+
Classifier: Topic :: Internet :: WWW/HTTP :: Session
|
|
42
|
+
Classifier: Topic :: Security
|
|
43
|
+
Classifier: Typing :: Typed
|
|
44
|
+
Requires-Python: >=3.10
|
|
45
|
+
Description-Content-Type: text/markdown
|
|
46
|
+
License-File: LICENSE
|
|
47
|
+
Requires-Dist: numpy>=1.26
|
|
48
|
+
Provides-Extra: dev
|
|
49
|
+
Requires-Dist: build>=1.2; extra == "dev"
|
|
50
|
+
Requires-Dist: twine>=5.0; extra == "dev"
|
|
51
|
+
Dynamic: license-file
|
|
52
|
+
|
|
53
|
+
# pyicloudx — vendor-neutral auth, anti-abuse shield, PCP protocol, and a CLI
|
|
54
|
+
|
|
55
|
+
> Status: **runnable / tested** (2026-11-17) · version **0.6.0** · **numpy-powered analytics**
|
|
56
|
+
> Location: `work/python/pyICloud/` · import name: `pyicloudx` · distribution name: `pyicloudx`
|
|
57
|
+
|
|
58
|
+
## 1. Highlights
|
|
59
|
+
|
|
60
|
+
| # | Feature | Evidence |
|
|
61
|
+
|---|---|---|
|
|
62
|
+
| 1 | **Four auth strategies, one call**: `inc` / `int` / `res` / `url` | `ICloudClient.for_auth(...)` |
|
|
63
|
+
| 2 | **10-layer anti-abuse shield** | single check ≈ **6 µs** |
|
|
64
|
+
| 3 | **PCP binary protocol**: 20-byte header, CRC32, HMAC, zlib, multiplexing, chunking, RTT | **2.5 µs/frame** encode, 0.05% chunk overhead |
|
|
65
|
+
| 4 | **Any-file support**: content sniffing, no extension whitelist | unknown binaries are never rejected |
|
|
66
|
+
| 5 | **numpy analytics**: percentiles, histograms, window stats | `numpy_ops.py`, `metrics.py` |
|
|
67
|
+
| 6 | **Persistent sessions + first-class 2FA** | JSON session store (0600), `on_2fa` callback |
|
|
68
|
+
| 7 | **Offline-testable**: transport/input/clock injected | **208 tests**, zero network |
|
|
69
|
+
| 8 | **Persistent daemon**: JSON-RPC, serialized queue, audit log | one call at a time, never interleaved |
|
|
70
|
+
| 9 | **Provider-agnostic**: you bring the endpoints | no vendor login baked in |
|
|
71
|
+
|
|
72
|
+
## 2. Install
|
|
73
|
+
|
|
74
|
+
```bash
|
|
75
|
+
pip install pyicloudx # from PyPI (once uploaded)
|
|
76
|
+
pip install -e . # local development
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
> The distribution and import name is **pyicloudx** because `pyicloud` is already
|
|
80
|
+
> taken on PyPI by another project.
|
|
81
|
+
|
|
82
|
+
## 3. Authentication: four strategies
|
|
83
|
+
|
|
84
|
+
| Name | Class | Credentials come from | Typical use |
|
|
85
|
+
|---|---|---|---|
|
|
86
|
+
| `inc` | `InlineAuth` | code arguments | scripts, CI |
|
|
87
|
+
| `int` | `InteractiveAuth` | terminal prompts (password hidden) | manual tools |
|
|
88
|
+
| `res` | `ResourceAuth` | JSON file or `PYICLOUD_*` env vars | services, multiple accounts |
|
|
89
|
+
| `url` | `UrlAuth` | browser `Cookie:` header or callback URL | avoids repeated 2FA |
|
|
90
|
+
|
|
91
|
+
```python
|
|
92
|
+
from pyicloudx import ICloudClient
|
|
93
|
+
|
|
94
|
+
c = ICloudClient.for_auth("inc", username="you@example.com", password="******")
|
|
95
|
+
session = c.authenticate()
|
|
96
|
+
print(c.username, f"{session.remaining_days:.1f} days left")
|
|
97
|
+
resp = c.request("GET", "/shortcuts")
|
|
98
|
+
payload = c.request_json("GET", "/shortcuts") # JSON helper
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
## 4. Anti-abuse shield (10 layers)
|
|
102
|
+
|
|
103
|
+
```python
|
|
104
|
+
from pyicloudx import AntiAbuseGuard, GuardPolicy, Request
|
|
105
|
+
|
|
106
|
+
guard = AntiAbuseGuard(GuardPolicy(secret=b"your-key", rate=20, burst=40))
|
|
107
|
+
d = guard.check(Request(key=client_ip, path="/api/me", headers=request_headers))
|
|
108
|
+
if not d.allowed:
|
|
109
|
+
return d.status, {"retry_after": d.retry_after}
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
| Layer | Defense | Result |
|
|
113
|
+
|---|---|---|
|
|
114
|
+
| 1 | blocklist (exact + CIDR) | 403 |
|
|
115
|
+
| 2 | honeypot paths | 403 + escalating ban (30 s → 1 h) |
|
|
116
|
+
| 3 | body size limit (8 MB) | 413 |
|
|
117
|
+
| 4 | required headers | 400 |
|
|
118
|
+
| 5 | fingerprint consistency (UA vs `Sec-CH-UA`) | 400 |
|
|
119
|
+
| 6 | timestamp skew (> 120 s) | 401 |
|
|
120
|
+
| 7 | nonce replay cache | 409 |
|
|
121
|
+
| 8 | HMAC signature | 401 |
|
|
122
|
+
| 9 | token bucket + sliding window | 429 + `retry_after` |
|
|
123
|
+
| 10 | progressive challenge token | 401 `challenge_required` |
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
## 4. Providers (you bring the endpoints)
|
|
127
|
+
|
|
128
|
+
The client never assumes a vendor. A `Provider` carries three URLs:
|
|
129
|
+
|
|
130
|
+
```python
|
|
131
|
+
from pyicloudx import ICloudClient, Provider
|
|
132
|
+
|
|
133
|
+
my_cloud = Provider(
|
|
134
|
+
name="mycloud",
|
|
135
|
+
login_url="https://cloud.example.com/api/login",
|
|
136
|
+
verify_url="https://cloud.example.com/api/2fa",
|
|
137
|
+
base_url="https://cloud.example.com",
|
|
138
|
+
)
|
|
139
|
+
|
|
140
|
+
c = ICloudClient.for_auth("inc", username="u", password="p", provider=my_cloud)
|
|
141
|
+
```
|
|
142
|
+
|
|
143
|
+
- `GENERIC` (default) has no URLs: password auth fails fast with
|
|
144
|
+
`MissingConfiguration`; `url` (cookie) auth still works.
|
|
145
|
+
- `APPLE` is an optional preset for the iCloud endpoints — an example, not the
|
|
146
|
+
library's purpose.
|
|
147
|
+
|
|
148
|
+
## 5. PCP protocol
|
|
149
|
+
|
|
150
|
+
Frame layout: `magic(PCP1) | version | type | flags | stream_id | seq | length | payload`
|
|
151
|
+
plus optional CRC32 and HMAC-SHA256 trailers.
|
|
152
|
+
|
|
153
|
+
- sticky/partial reads are safe (`decode(bytearray)` keeps the remainder);
|
|
154
|
+
- garbage resynchronization (searches for the next magic);
|
|
155
|
+
- per-frame CRC and optional HMAC;
|
|
156
|
+
- zlib compression above a threshold (only when it shrinks);
|
|
157
|
+
- multiplexing via `stream_id`;
|
|
158
|
+
- chunked file transfer with 8-byte offsets and out-of-order reassembly;
|
|
159
|
+
- PING/PONG RTT meter with an injectable clock;
|
|
160
|
+
- HELLO capability negotiation; text debug mode.
|
|
161
|
+
|
|
162
|
+
```python
|
|
163
|
+
from pyicloudx import Codec, Frame, DATA, ChunkEncoder, ChunkAssembler
|
|
164
|
+
|
|
165
|
+
codec = Codec(secret=b"shared")
|
|
166
|
+
wire = codec.encode(Frame(DATA, stream_id=1, seq=1, payload=b"hello"))
|
|
167
|
+
for frame in codec.decode(bytearray(wire)):
|
|
168
|
+
print(frame.type_name, frame.payload)
|
|
169
|
+
```
|
|
170
|
+
|
|
171
|
+
Measured: encode **2.47 µs/frame**, decode 5.24 µs/frame, 1 MB chunked transfer
|
|
172
|
+
with **0.05%** protocol overhead.
|
|
173
|
+
|
|
174
|
+
## 6. Any file
|
|
175
|
+
|
|
176
|
+
```python
|
|
177
|
+
from pyicloudx import FilePayload
|
|
178
|
+
|
|
179
|
+
f = FilePayload.from_path("whatever.bin") # no extension whitelist
|
|
180
|
+
ctype, body = f.to_multipart("file") # RFC 5987 for non-ASCII names
|
|
181
|
+
ctype, parts = f.multipart_parts("file") # streaming alternative
|
|
182
|
+
```
|
|
183
|
+
|
|
184
|
+
Magic-number sniffing covers png/jpeg/gif/webp/bmp/pdf/zip/gzip/7z/rar/wav/avi,
|
|
185
|
+
plus text and unknown binary (never rejected). Filenames are sanitized
|
|
186
|
+
(`../../etc/passwd` → `passwd`).
|
|
187
|
+
|
|
188
|
+
## 7. Port routing grammar
|
|
189
|
+
|
|
190
|
+
```ini
|
|
191
|
+
[secret]
|
|
192
|
+
alice_pw = @env:ALICE_PW
|
|
193
|
+
|
|
194
|
+
[account alice]
|
|
195
|
+
user = alice@example.com
|
|
196
|
+
pass = $alice_pw
|
|
197
|
+
|
|
198
|
+
[route]
|
|
199
|
+
8080 = alice inc
|
|
200
|
+
8443 = bob res file=~/creds.json
|
|
201
|
+
9000-9010 = alice int
|
|
202
|
+
* = bob url cookie=@env:BOB_COOKIE
|
|
203
|
+
```
|
|
204
|
+
|
|
205
|
+
Precedence: exact port > range > `*`. Values support `@env:`, `@file:`,
|
|
206
|
+
`$secret`, quoted strings, and `#` comments. Errors carry line numbers.
|
|
207
|
+
|
|
208
|
+
## 8. CLI
|
|
209
|
+
|
|
210
|
+
```bash
|
|
211
|
+
python -m pyicloudx check --config routes.ini
|
|
212
|
+
python -m pyicloudx run --config routes.ini --port 8080 [--request /shortcuts]
|
|
213
|
+
python -m pyicloudx verify --config routes.ini --port 8080
|
|
214
|
+
python -m pyicloudx secrets set alice_pw # Windows DPAPI / macOS Keychain
|
|
215
|
+
python -m pyicloudx watch --config routes.ini --port 8080
|
|
216
|
+
python -m pyicloudx info | doctor | bench
|
|
217
|
+
```
|
|
218
|
+
|
|
219
|
+
## 9. Ops tooling
|
|
220
|
+
|
|
221
|
+
| Module | What it does |
|
|
222
|
+
|---|---|
|
|
223
|
+
| `metrics.py` | counters + numpy percentiles (p50/p95/p99) + histograms |
|
|
224
|
+
| `numpy_ops.py` | percentile stats, histograms, moving average, chunk plan, uint8 views |
|
|
225
|
+
| `logging.py` | redacting logger (passwords/cookies never reach logs) |
|
|
226
|
+
| `retry.py` | exponential backoff with jitter, honors `Retry-After` |
|
|
227
|
+
| `cache.py` | TTL response cache for GET 2xx |
|
|
228
|
+
| `keyring.py` | Windows DPAPI / macOS Keychain, `rotate()` / `has()` |
|
|
229
|
+
| `routing.py` | config grammar + `reload()` / `reload_if_changed()` / `describe()` |
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
## 9. Daemon (persistent, serialized, JSON-RPC)
|
|
233
|
+
|
|
234
|
+
```bash
|
|
235
|
+
python -m pyicloudx daemon start --port 8300 --token s3cret
|
|
236
|
+
python -m pyicloudx daemon status
|
|
237
|
+
python -m pyicloudx daemon rpc --method ping
|
|
238
|
+
python -m pyicloudx daemon stop
|
|
239
|
+
```
|
|
240
|
+
|
|
241
|
+
Every RPC runs through a **single worker queue**, so two callers can never
|
|
242
|
+
mutate the same session concurrently; each call produces one JSON envelope and
|
|
243
|
+
one atomic audit line (JSONL + TXT mirror).
|
|
244
|
+
|
|
245
|
+
```
|
|
246
|
+
POST /rpc {"id":"a1","method":"ping","params":{}}
|
|
247
|
+
-> {"ok":true,"id":"a1","method":"ping","ts":...,"elapsed_ms":0.3,
|
|
248
|
+
"result":{"pong":true,"pid":1234}}
|
|
249
|
+
```
|
|
250
|
+
|
|
251
|
+
Verified by tests: six concurrent 60 ms calls take >= 300 ms (strictly serialized),
|
|
252
|
+
ids stay unique, and every audit line parses.
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
## 10. Remote client + online deployment
|
|
256
|
+
|
|
257
|
+
`RemoteClient` keeps one keep-alive connection and measures its own latency with numpy:
|
|
258
|
+
|
|
259
|
+
```python
|
|
260
|
+
from pyicloudx.remote import RemoteClient
|
|
261
|
+
|
|
262
|
+
with RemoteClient("https://your-daemon.example.com", token="***") as rpc:
|
|
263
|
+
rpc.version()
|
|
264
|
+
for _ in range(100):
|
|
265
|
+
rpc.ping()
|
|
266
|
+
print(rpc.latency_summary())
|
|
267
|
+
```
|
|
268
|
+
|
|
269
|
+
Localhost measurement (300 pings, same machine):
|
|
270
|
+
|
|
271
|
+
| metric | value |
|
|
272
|
+
|---|---|
|
|
273
|
+
| avg round trip | **0.94 ms** |
|
|
274
|
+
| p50 | 0.67 ms |
|
|
275
|
+
| p95 | 2.19 ms |
|
|
276
|
+
|
|
277
|
+
Deployable as-is: `deploy/Dockerfile`, `deploy/fly.toml`, `deploy/render.yaml`,
|
|
278
|
+
`deploy/README.md`. The daemon **refuses to bind a public host without a token**,
|
|
279
|
+
gzip-compresses responses above 512 bytes, and writes audit logs asynchronously
|
|
280
|
+
so disk I/O never sits on the request path.
|
|
281
|
+
|
|
282
|
+
|
|
283
|
+
## 11. Cloud mode: one machine, auto ports, one gateway
|
|
284
|
+
|
|
285
|
+
```bash
|
|
286
|
+
python -m pyicloudx cloud up # foreground; prints the registry
|
|
287
|
+
python -m pyicloudx cloud up --token t # daemon requires the token
|
|
288
|
+
python -m pyicloudx cloud status # health of every service
|
|
289
|
+
python -m pyicloudx cloud down
|
|
290
|
+
```
|
|
291
|
+
|
|
292
|
+
What it does:
|
|
293
|
+
|
|
294
|
+
- allocates **three free ports** (daemon / site / gateway) and records them in
|
|
295
|
+
`~/.pyicloud/cloud.json`;
|
|
296
|
+
- starts the daemon as a detached child and a static-site server in-process;
|
|
297
|
+
- binds **one gateway port**: `/health` and `/rpc` go to the daemon, everything
|
|
298
|
+
else to the site;
|
|
299
|
+
- restarts the daemon if it exits (`serve_forever` supervisor loop);
|
|
300
|
+
- `cloud status` reports per-service health by probing the real endpoints.
|
|
301
|
+
|
|
302
|
+
```
|
|
303
|
+
~/.pyicloud/cloud.json
|
|
304
|
+
{
|
|
305
|
+
"gateway": {"port": 64143, "url": "http://127.0.0.1:64143/"},
|
|
306
|
+
"services": {
|
|
307
|
+
"daemon": {"port": 64141, "pid": 20652},
|
|
308
|
+
"site": {"port": 64142}
|
|
309
|
+
}
|
|
310
|
+
}
|
|
311
|
+
```
|
|
312
|
+
|
|
313
|
+
Measured round trip on localhost: **0.94 ms** average (300 pings).
|
|
314
|
+
|
|
315
|
+
|
|
316
|
+
## 12. Storage and cloud sync
|
|
317
|
+
|
|
318
|
+
Content-addressed blob store (`sha256` is the address) plus daemon-to-daemon sync:
|
|
319
|
+
|
|
320
|
+
```bash
|
|
321
|
+
# store a file locally
|
|
322
|
+
python -m pyicloudx store put photo.jpg --dir ~/.pyicloud/blobs
|
|
323
|
+
python -m pyicloudx store list --dir ~/.pyicloud/blobs
|
|
324
|
+
|
|
325
|
+
# push/pull against another daemon
|
|
326
|
+
python -m pyicloudx sync status --remote https://daemon-b.example.com --token t
|
|
327
|
+
python -m pyicloudx sync push --remote https://daemon-b.example.com --token t
|
|
328
|
+
python -m pyicloudx sync pull --remote https://daemon-b.example.com --token t
|
|
329
|
+
```
|
|
330
|
+
|
|
331
|
+
Why sync is cheap:
|
|
332
|
+
|
|
333
|
+
- the sha256 of the bytes is the address, so identical uploads deduplicate;
|
|
334
|
+
- sync is a **manifest set comparison**: local-only → push, remote-only → pull;
|
|
335
|
+
- two different files with the same name are just two different blobs (no conflicts);
|
|
336
|
+
- reads re-hash and reject corrupted blobs.
|
|
337
|
+
|
|
338
|
+
Daemon endpoints: `GET /blobs` (manifest), `GET /blobs/<sha>`, `POST /blobs`
|
|
339
|
+
(raw upload), `GET /metrics` (JSON counters). RPC adds `store_list`,
|
|
340
|
+
`store_stat`, `store_delete`.
|
|
341
|
+
|
|
342
|
+
|
|
343
|
+
## 13. Tests
|
|
344
|
+
|
|
345
|
+
```bash
|
|
346
|
+
python -m unittest discover -s tests -v
|
|
347
|
+
```
|
|
348
|
+
|
|
349
|
+
208 offline tests across auth strategies, client flows, session storage, shield,
|
|
350
|
+
files, PCP protocol, routing, keyring and the extras module.
|
|
351
|
+
|
|
352
|
+
## 14. Build & publish
|
|
353
|
+
|
|
354
|
+
```bash
|
|
355
|
+
pip install build twine
|
|
356
|
+
python -m build --no-isolation --outdir dist
|
|
357
|
+
twine check dist/*
|
|
358
|
+
twine upload dist/* # needs TWINE_PASSWORD=__token__ / pypi-...
|
|
359
|
+
```
|
|
360
|
+
|
|
361
|
+
## 15. Limitations
|
|
362
|
+
|
|
363
|
+
1. **No vendor login is baked in**: the default `GENERIC` provider has no URLs, so
|
|
364
|
+
password auth requires you to pass a `Provider` (or use cookie/url auth).
|
|
365
|
+
2. The shield is in-process (no Redis); it is not a WAF.
|
|
366
|
+
3. PCP is a single-connection protocol; retransmission and congestion control
|
|
367
|
+
are delegated to TCP.
|
|
368
|
+
4. Benchmark numbers are local and machine-specific.
|
|
@@ -0,0 +1,316 @@
|
|
|
1
|
+
# pyicloudx — vendor-neutral auth, anti-abuse shield, PCP protocol, and a CLI
|
|
2
|
+
|
|
3
|
+
> Status: **runnable / tested** (2026-11-17) · version **0.6.0** · **numpy-powered analytics**
|
|
4
|
+
> Location: `work/python/pyICloud/` · import name: `pyicloudx` · distribution name: `pyicloudx`
|
|
5
|
+
|
|
6
|
+
## 1. Highlights
|
|
7
|
+
|
|
8
|
+
| # | Feature | Evidence |
|
|
9
|
+
|---|---|---|
|
|
10
|
+
| 1 | **Four auth strategies, one call**: `inc` / `int` / `res` / `url` | `ICloudClient.for_auth(...)` |
|
|
11
|
+
| 2 | **10-layer anti-abuse shield** | single check ≈ **6 µs** |
|
|
12
|
+
| 3 | **PCP binary protocol**: 20-byte header, CRC32, HMAC, zlib, multiplexing, chunking, RTT | **2.5 µs/frame** encode, 0.05% chunk overhead |
|
|
13
|
+
| 4 | **Any-file support**: content sniffing, no extension whitelist | unknown binaries are never rejected |
|
|
14
|
+
| 5 | **numpy analytics**: percentiles, histograms, window stats | `numpy_ops.py`, `metrics.py` |
|
|
15
|
+
| 6 | **Persistent sessions + first-class 2FA** | JSON session store (0600), `on_2fa` callback |
|
|
16
|
+
| 7 | **Offline-testable**: transport/input/clock injected | **208 tests**, zero network |
|
|
17
|
+
| 8 | **Persistent daemon**: JSON-RPC, serialized queue, audit log | one call at a time, never interleaved |
|
|
18
|
+
| 9 | **Provider-agnostic**: you bring the endpoints | no vendor login baked in |
|
|
19
|
+
|
|
20
|
+
## 2. Install
|
|
21
|
+
|
|
22
|
+
```bash
|
|
23
|
+
pip install pyicloudx # from PyPI (once uploaded)
|
|
24
|
+
pip install -e . # local development
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
> The distribution and import name is **pyicloudx** because `pyicloud` is already
|
|
28
|
+
> taken on PyPI by another project.
|
|
29
|
+
|
|
30
|
+
## 3. Authentication: four strategies
|
|
31
|
+
|
|
32
|
+
| Name | Class | Credentials come from | Typical use |
|
|
33
|
+
|---|---|---|---|
|
|
34
|
+
| `inc` | `InlineAuth` | code arguments | scripts, CI |
|
|
35
|
+
| `int` | `InteractiveAuth` | terminal prompts (password hidden) | manual tools |
|
|
36
|
+
| `res` | `ResourceAuth` | JSON file or `PYICLOUD_*` env vars | services, multiple accounts |
|
|
37
|
+
| `url` | `UrlAuth` | browser `Cookie:` header or callback URL | avoids repeated 2FA |
|
|
38
|
+
|
|
39
|
+
```python
|
|
40
|
+
from pyicloudx import ICloudClient
|
|
41
|
+
|
|
42
|
+
c = ICloudClient.for_auth("inc", username="you@example.com", password="******")
|
|
43
|
+
session = c.authenticate()
|
|
44
|
+
print(c.username, f"{session.remaining_days:.1f} days left")
|
|
45
|
+
resp = c.request("GET", "/shortcuts")
|
|
46
|
+
payload = c.request_json("GET", "/shortcuts") # JSON helper
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
## 4. Anti-abuse shield (10 layers)
|
|
50
|
+
|
|
51
|
+
```python
|
|
52
|
+
from pyicloudx import AntiAbuseGuard, GuardPolicy, Request
|
|
53
|
+
|
|
54
|
+
guard = AntiAbuseGuard(GuardPolicy(secret=b"your-key", rate=20, burst=40))
|
|
55
|
+
d = guard.check(Request(key=client_ip, path="/api/me", headers=request_headers))
|
|
56
|
+
if not d.allowed:
|
|
57
|
+
return d.status, {"retry_after": d.retry_after}
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
| Layer | Defense | Result |
|
|
61
|
+
|---|---|---|
|
|
62
|
+
| 1 | blocklist (exact + CIDR) | 403 |
|
|
63
|
+
| 2 | honeypot paths | 403 + escalating ban (30 s → 1 h) |
|
|
64
|
+
| 3 | body size limit (8 MB) | 413 |
|
|
65
|
+
| 4 | required headers | 400 |
|
|
66
|
+
| 5 | fingerprint consistency (UA vs `Sec-CH-UA`) | 400 |
|
|
67
|
+
| 6 | timestamp skew (> 120 s) | 401 |
|
|
68
|
+
| 7 | nonce replay cache | 409 |
|
|
69
|
+
| 8 | HMAC signature | 401 |
|
|
70
|
+
| 9 | token bucket + sliding window | 429 + `retry_after` |
|
|
71
|
+
| 10 | progressive challenge token | 401 `challenge_required` |
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
## 4. Providers (you bring the endpoints)
|
|
75
|
+
|
|
76
|
+
The client never assumes a vendor. A `Provider` carries three URLs:
|
|
77
|
+
|
|
78
|
+
```python
|
|
79
|
+
from pyicloudx import ICloudClient, Provider
|
|
80
|
+
|
|
81
|
+
my_cloud = Provider(
|
|
82
|
+
name="mycloud",
|
|
83
|
+
login_url="https://cloud.example.com/api/login",
|
|
84
|
+
verify_url="https://cloud.example.com/api/2fa",
|
|
85
|
+
base_url="https://cloud.example.com",
|
|
86
|
+
)
|
|
87
|
+
|
|
88
|
+
c = ICloudClient.for_auth("inc", username="u", password="p", provider=my_cloud)
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
- `GENERIC` (default) has no URLs: password auth fails fast with
|
|
92
|
+
`MissingConfiguration`; `url` (cookie) auth still works.
|
|
93
|
+
- `APPLE` is an optional preset for the iCloud endpoints — an example, not the
|
|
94
|
+
library's purpose.
|
|
95
|
+
|
|
96
|
+
## 5. PCP protocol
|
|
97
|
+
|
|
98
|
+
Frame layout: `magic(PCP1) | version | type | flags | stream_id | seq | length | payload`
|
|
99
|
+
plus optional CRC32 and HMAC-SHA256 trailers.
|
|
100
|
+
|
|
101
|
+
- sticky/partial reads are safe (`decode(bytearray)` keeps the remainder);
|
|
102
|
+
- garbage resynchronization (searches for the next magic);
|
|
103
|
+
- per-frame CRC and optional HMAC;
|
|
104
|
+
- zlib compression above a threshold (only when it shrinks);
|
|
105
|
+
- multiplexing via `stream_id`;
|
|
106
|
+
- chunked file transfer with 8-byte offsets and out-of-order reassembly;
|
|
107
|
+
- PING/PONG RTT meter with an injectable clock;
|
|
108
|
+
- HELLO capability negotiation; text debug mode.
|
|
109
|
+
|
|
110
|
+
```python
|
|
111
|
+
from pyicloudx import Codec, Frame, DATA, ChunkEncoder, ChunkAssembler
|
|
112
|
+
|
|
113
|
+
codec = Codec(secret=b"shared")
|
|
114
|
+
wire = codec.encode(Frame(DATA, stream_id=1, seq=1, payload=b"hello"))
|
|
115
|
+
for frame in codec.decode(bytearray(wire)):
|
|
116
|
+
print(frame.type_name, frame.payload)
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
Measured: encode **2.47 µs/frame**, decode 5.24 µs/frame, 1 MB chunked transfer
|
|
120
|
+
with **0.05%** protocol overhead.
|
|
121
|
+
|
|
122
|
+
## 6. Any file
|
|
123
|
+
|
|
124
|
+
```python
|
|
125
|
+
from pyicloudx import FilePayload
|
|
126
|
+
|
|
127
|
+
f = FilePayload.from_path("whatever.bin") # no extension whitelist
|
|
128
|
+
ctype, body = f.to_multipart("file") # RFC 5987 for non-ASCII names
|
|
129
|
+
ctype, parts = f.multipart_parts("file") # streaming alternative
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
Magic-number sniffing covers png/jpeg/gif/webp/bmp/pdf/zip/gzip/7z/rar/wav/avi,
|
|
133
|
+
plus text and unknown binary (never rejected). Filenames are sanitized
|
|
134
|
+
(`../../etc/passwd` → `passwd`).
|
|
135
|
+
|
|
136
|
+
## 7. Port routing grammar
|
|
137
|
+
|
|
138
|
+
```ini
|
|
139
|
+
[secret]
|
|
140
|
+
alice_pw = @env:ALICE_PW
|
|
141
|
+
|
|
142
|
+
[account alice]
|
|
143
|
+
user = alice@example.com
|
|
144
|
+
pass = $alice_pw
|
|
145
|
+
|
|
146
|
+
[route]
|
|
147
|
+
8080 = alice inc
|
|
148
|
+
8443 = bob res file=~/creds.json
|
|
149
|
+
9000-9010 = alice int
|
|
150
|
+
* = bob url cookie=@env:BOB_COOKIE
|
|
151
|
+
```
|
|
152
|
+
|
|
153
|
+
Precedence: exact port > range > `*`. Values support `@env:`, `@file:`,
|
|
154
|
+
`$secret`, quoted strings, and `#` comments. Errors carry line numbers.
|
|
155
|
+
|
|
156
|
+
## 8. CLI
|
|
157
|
+
|
|
158
|
+
```bash
|
|
159
|
+
python -m pyicloudx check --config routes.ini
|
|
160
|
+
python -m pyicloudx run --config routes.ini --port 8080 [--request /shortcuts]
|
|
161
|
+
python -m pyicloudx verify --config routes.ini --port 8080
|
|
162
|
+
python -m pyicloudx secrets set alice_pw # Windows DPAPI / macOS Keychain
|
|
163
|
+
python -m pyicloudx watch --config routes.ini --port 8080
|
|
164
|
+
python -m pyicloudx info | doctor | bench
|
|
165
|
+
```
|
|
166
|
+
|
|
167
|
+
## 9. Ops tooling
|
|
168
|
+
|
|
169
|
+
| Module | What it does |
|
|
170
|
+
|---|---|
|
|
171
|
+
| `metrics.py` | counters + numpy percentiles (p50/p95/p99) + histograms |
|
|
172
|
+
| `numpy_ops.py` | percentile stats, histograms, moving average, chunk plan, uint8 views |
|
|
173
|
+
| `logging.py` | redacting logger (passwords/cookies never reach logs) |
|
|
174
|
+
| `retry.py` | exponential backoff with jitter, honors `Retry-After` |
|
|
175
|
+
| `cache.py` | TTL response cache for GET 2xx |
|
|
176
|
+
| `keyring.py` | Windows DPAPI / macOS Keychain, `rotate()` / `has()` |
|
|
177
|
+
| `routing.py` | config grammar + `reload()` / `reload_if_changed()` / `describe()` |
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
## 9. Daemon (persistent, serialized, JSON-RPC)
|
|
181
|
+
|
|
182
|
+
```bash
|
|
183
|
+
python -m pyicloudx daemon start --port 8300 --token s3cret
|
|
184
|
+
python -m pyicloudx daemon status
|
|
185
|
+
python -m pyicloudx daemon rpc --method ping
|
|
186
|
+
python -m pyicloudx daemon stop
|
|
187
|
+
```
|
|
188
|
+
|
|
189
|
+
Every RPC runs through a **single worker queue**, so two callers can never
|
|
190
|
+
mutate the same session concurrently; each call produces one JSON envelope and
|
|
191
|
+
one atomic audit line (JSONL + TXT mirror).
|
|
192
|
+
|
|
193
|
+
```
|
|
194
|
+
POST /rpc {"id":"a1","method":"ping","params":{}}
|
|
195
|
+
-> {"ok":true,"id":"a1","method":"ping","ts":...,"elapsed_ms":0.3,
|
|
196
|
+
"result":{"pong":true,"pid":1234}}
|
|
197
|
+
```
|
|
198
|
+
|
|
199
|
+
Verified by tests: six concurrent 60 ms calls take >= 300 ms (strictly serialized),
|
|
200
|
+
ids stay unique, and every audit line parses.
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
## 10. Remote client + online deployment
|
|
204
|
+
|
|
205
|
+
`RemoteClient` keeps one keep-alive connection and measures its own latency with numpy:
|
|
206
|
+
|
|
207
|
+
```python
|
|
208
|
+
from pyicloudx.remote import RemoteClient
|
|
209
|
+
|
|
210
|
+
with RemoteClient("https://your-daemon.example.com", token="***") as rpc:
|
|
211
|
+
rpc.version()
|
|
212
|
+
for _ in range(100):
|
|
213
|
+
rpc.ping()
|
|
214
|
+
print(rpc.latency_summary())
|
|
215
|
+
```
|
|
216
|
+
|
|
217
|
+
Localhost measurement (300 pings, same machine):
|
|
218
|
+
|
|
219
|
+
| metric | value |
|
|
220
|
+
|---|---|
|
|
221
|
+
| avg round trip | **0.94 ms** |
|
|
222
|
+
| p50 | 0.67 ms |
|
|
223
|
+
| p95 | 2.19 ms |
|
|
224
|
+
|
|
225
|
+
Deployable as-is: `deploy/Dockerfile`, `deploy/fly.toml`, `deploy/render.yaml`,
|
|
226
|
+
`deploy/README.md`. The daemon **refuses to bind a public host without a token**,
|
|
227
|
+
gzip-compresses responses above 512 bytes, and writes audit logs asynchronously
|
|
228
|
+
so disk I/O never sits on the request path.
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
## 11. Cloud mode: one machine, auto ports, one gateway
|
|
232
|
+
|
|
233
|
+
```bash
|
|
234
|
+
python -m pyicloudx cloud up # foreground; prints the registry
|
|
235
|
+
python -m pyicloudx cloud up --token t # daemon requires the token
|
|
236
|
+
python -m pyicloudx cloud status # health of every service
|
|
237
|
+
python -m pyicloudx cloud down
|
|
238
|
+
```
|
|
239
|
+
|
|
240
|
+
What it does:
|
|
241
|
+
|
|
242
|
+
- allocates **three free ports** (daemon / site / gateway) and records them in
|
|
243
|
+
`~/.pyicloud/cloud.json`;
|
|
244
|
+
- starts the daemon as a detached child and a static-site server in-process;
|
|
245
|
+
- binds **one gateway port**: `/health` and `/rpc` go to the daemon, everything
|
|
246
|
+
else to the site;
|
|
247
|
+
- restarts the daemon if it exits (`serve_forever` supervisor loop);
|
|
248
|
+
- `cloud status` reports per-service health by probing the real endpoints.
|
|
249
|
+
|
|
250
|
+
```
|
|
251
|
+
~/.pyicloud/cloud.json
|
|
252
|
+
{
|
|
253
|
+
"gateway": {"port": 64143, "url": "http://127.0.0.1:64143/"},
|
|
254
|
+
"services": {
|
|
255
|
+
"daemon": {"port": 64141, "pid": 20652},
|
|
256
|
+
"site": {"port": 64142}
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
```
|
|
260
|
+
|
|
261
|
+
Measured round trip on localhost: **0.94 ms** average (300 pings).
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
## 12. Storage and cloud sync
|
|
265
|
+
|
|
266
|
+
Content-addressed blob store (`sha256` is the address) plus daemon-to-daemon sync:
|
|
267
|
+
|
|
268
|
+
```bash
|
|
269
|
+
# store a file locally
|
|
270
|
+
python -m pyicloudx store put photo.jpg --dir ~/.pyicloud/blobs
|
|
271
|
+
python -m pyicloudx store list --dir ~/.pyicloud/blobs
|
|
272
|
+
|
|
273
|
+
# push/pull against another daemon
|
|
274
|
+
python -m pyicloudx sync status --remote https://daemon-b.example.com --token t
|
|
275
|
+
python -m pyicloudx sync push --remote https://daemon-b.example.com --token t
|
|
276
|
+
python -m pyicloudx sync pull --remote https://daemon-b.example.com --token t
|
|
277
|
+
```
|
|
278
|
+
|
|
279
|
+
Why sync is cheap:
|
|
280
|
+
|
|
281
|
+
- the sha256 of the bytes is the address, so identical uploads deduplicate;
|
|
282
|
+
- sync is a **manifest set comparison**: local-only → push, remote-only → pull;
|
|
283
|
+
- two different files with the same name are just two different blobs (no conflicts);
|
|
284
|
+
- reads re-hash and reject corrupted blobs.
|
|
285
|
+
|
|
286
|
+
Daemon endpoints: `GET /blobs` (manifest), `GET /blobs/<sha>`, `POST /blobs`
|
|
287
|
+
(raw upload), `GET /metrics` (JSON counters). RPC adds `store_list`,
|
|
288
|
+
`store_stat`, `store_delete`.
|
|
289
|
+
|
|
290
|
+
|
|
291
|
+
## 13. Tests
|
|
292
|
+
|
|
293
|
+
```bash
|
|
294
|
+
python -m unittest discover -s tests -v
|
|
295
|
+
```
|
|
296
|
+
|
|
297
|
+
208 offline tests across auth strategies, client flows, session storage, shield,
|
|
298
|
+
files, PCP protocol, routing, keyring and the extras module.
|
|
299
|
+
|
|
300
|
+
## 14. Build & publish
|
|
301
|
+
|
|
302
|
+
```bash
|
|
303
|
+
pip install build twine
|
|
304
|
+
python -m build --no-isolation --outdir dist
|
|
305
|
+
twine check dist/*
|
|
306
|
+
twine upload dist/* # needs TWINE_PASSWORD=__token__ / pypi-...
|
|
307
|
+
```
|
|
308
|
+
|
|
309
|
+
## 15. Limitations
|
|
310
|
+
|
|
311
|
+
1. **No vendor login is baked in**: the default `GENERIC` provider has no URLs, so
|
|
312
|
+
password auth requires you to pass a `Provider` (or use cookie/url auth).
|
|
313
|
+
2. The shield is in-process (no Redis); it is not a WAF.
|
|
314
|
+
3. PCP is a single-connection protocol; retransmission and congestion control
|
|
315
|
+
are delegated to TCP.
|
|
316
|
+
4. Benchmark numbers are local and machine-specific.
|