iclouddl 1.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- iclouddl-1.0.0/.gitignore +29 -0
- iclouddl-1.0.0/DISCLAIMER.md +13 -0
- iclouddl-1.0.0/LICENSE +21 -0
- iclouddl-1.0.0/PKG-INFO +109 -0
- iclouddl-1.0.0/README.md +78 -0
- iclouddl-1.0.0/iclouddl.py +786 -0
- iclouddl-1.0.0/pyproject.toml +52 -0
- iclouddl-1.0.0/requirements.txt +11 -0
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
# --- Secrets: never commit exported session cookies ---
|
|
2
|
+
*cookies*.json
|
|
3
|
+
cookies.json
|
|
4
|
+
*.cookies.json
|
|
5
|
+
|
|
6
|
+
# --- Downloaded data: your photos/videos don't belong in git ---
|
|
7
|
+
icloud-shared/
|
|
8
|
+
downloads/
|
|
9
|
+
*.iclouddl-index.json
|
|
10
|
+
|
|
11
|
+
# --- Python ---
|
|
12
|
+
__pycache__/
|
|
13
|
+
*.py[cod]
|
|
14
|
+
*.egg-info/
|
|
15
|
+
.eggs/
|
|
16
|
+
build/
|
|
17
|
+
dist/
|
|
18
|
+
.venv/
|
|
19
|
+
venv/
|
|
20
|
+
env/
|
|
21
|
+
.mypy_cache/
|
|
22
|
+
.pytest_cache/
|
|
23
|
+
.ruff_cache/
|
|
24
|
+
|
|
25
|
+
# --- Editor / OS ---
|
|
26
|
+
.vscode/
|
|
27
|
+
.idea/
|
|
28
|
+
.DS_Store
|
|
29
|
+
*.swp
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
# Disclaimer
|
|
2
|
+
|
|
3
|
+
ICloudDL is an **unofficial**, community tool. It is not affiliated with, authorized by, or
|
|
4
|
+
endorsed by Apple Inc. "iCloud" and "Apple" are trademarks of Apple Inc.
|
|
5
|
+
|
|
6
|
+
- It uses session cookies **you** export from your own signed-in `www.icloud.com` session.
|
|
7
|
+
- It only accesses **shared albums your own account already has access to** — it does not bypass
|
|
8
|
+
authentication, break encryption, or reach anything you couldn't open yourself in a browser.
|
|
9
|
+
- It is **read-only**: it downloads your data and never modifies, deletes, or uploads anything.
|
|
10
|
+
- You are responsible for how you use the photos and videos you download, including the rights of
|
|
11
|
+
whoever shared them with you.
|
|
12
|
+
|
|
13
|
+
Provided **as-is**, without warranty of any kind. AI-assisted. Use at your own risk.
|
iclouddl-1.0.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Amol Venkataraman
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
iclouddl-1.0.0/PKG-INFO
ADDED
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: iclouddl
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: Download photos and videos from an iCloud shared album, for those without an Apple device
|
|
5
|
+
Project-URL: Homepage, https://github.com/upstream-wontfix/ICloudDL
|
|
6
|
+
Project-URL: Repository, https://github.com/upstream-wontfix/ICloudDL
|
|
7
|
+
Project-URL: Issues, https://github.com/upstream-wontfix/ICloudDL/issues
|
|
8
|
+
Author: Amol Venkataraman
|
|
9
|
+
License-Expression: MIT
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Keywords: cli,download,exif,icloud,photos,shared-album
|
|
12
|
+
Classifier: Development Status :: 4 - Beta
|
|
13
|
+
Classifier: Environment :: Console
|
|
14
|
+
Classifier: Intended Audience :: End Users/Desktop
|
|
15
|
+
Classifier: Operating System :: OS Independent
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
23
|
+
Classifier: Topic :: Multimedia :: Graphics
|
|
24
|
+
Classifier: Topic :: Utilities
|
|
25
|
+
Requires-Python: >=3.9
|
|
26
|
+
Requires-Dist: requests>=2.28
|
|
27
|
+
Provides-Extra: heic
|
|
28
|
+
Requires-Dist: pillow-heif>=0.13; extra == 'heic'
|
|
29
|
+
Requires-Dist: pillow>=10.0; extra == 'heic'
|
|
30
|
+
Description-Content-Type: text/markdown
|
|
31
|
+
|
|
32
|
+
<!-- header:start -->
|
|
33
|
+
<div align="center">
|
|
34
|
+
|
|
35
|
+
<img alt="ICloudDL: The Download button Apple never shipped: save a whole iCloud Shared Album without a $999 glass rectangle." src=".github/assets/banner.svg" width="100%">
|
|
36
|
+
|
|
37
|
+

|
|
38
|
+
[](LICENSE)
|
|
39
|
+
|
|
40
|
+
[Install](#install) · [Use](#use) · [Heads-up on quality](#heads-up-on-quality)
|
|
41
|
+
|
|
42
|
+
</div>
|
|
43
|
+
|
|
44
|
+
<a href="https://haiscale.amolven.com/A1/"><img alt="HAI Scale A1, Mostly automated. Most of the code was written by AI coding agents. I directed the design, reviewed the changes and tested it on real shared albums. Unofficial and not affiliated with Apple. Only download albums you're allowed to save. Provided as is, with no warranty." src=".github/assets/hai.svg" width="100%"></a>
|
|
45
|
+
|
|
46
|
+
<sub>Full terms: [DISCLAIMER.md](DISCLAIMER.md)</sub>
|
|
47
|
+
<!-- header:end -->
|
|
48
|
+
|
|
49
|
+
Download photos and videos from an iCloud share for those without an Apple device.
|
|
50
|
+
|
|
51
|
+
Apple built "Shared Albums" so anyone can look at your photos — then made sure anyone *without* a $999 glass rectangle has to right-click and "Save As" one by one like it's a GeoCities page. ICloudDL is the **Download** button they never got around to shipping to the rest of us: point it at your exported cookies, run one command, and the whole shared album lands in a folder — proper file extensions, EXIF intact, JSON sidecars and all.
|
|
52
|
+
|
|
53
|
+
No Mac, no iPhone, no "just sign in with your Apple ID on your other Apple device."
|
|
54
|
+
|
|
55
|
+
## Install
|
|
56
|
+
|
|
57
|
+
From PyPI:
|
|
58
|
+
|
|
59
|
+
```bash
|
|
60
|
+
pipx install iclouddl # or: uv tool install iclouddl
|
|
61
|
+
pipx install 'iclouddl[heic]' # also transcode genuine HEIC files to JPEG (rarely needed)
|
|
62
|
+
pip install iclouddl # into an existing environment
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
From source:
|
|
66
|
+
|
|
67
|
+
```bash
|
|
68
|
+
pip install -r requirements.txt # needs `exiftool` on PATH for sidecars; pillow-heif only for true-HEIC transcode
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
JSON sidecars read embedded EXIF with the `exiftool` binary, which must be on PATH
|
|
72
|
+
(`apt install libimage-exiftool-perl` / `brew install exiftool`). Without it, sidecars carry the iCloud fields only.
|
|
73
|
+
|
|
74
|
+
## Use
|
|
75
|
+
|
|
76
|
+
Export your `www.icloud.com` cookies to a JSON file while signed in (any "cookies.json" browser extension works), then:
|
|
77
|
+
|
|
78
|
+
```bash
|
|
79
|
+
iclouddl -cf cookies.json -l # list the shared albums you can see
|
|
80
|
+
iclouddl -cf cookies.json -a "Lake Tahoe" -a 1 # download specific albums (name, index, or GUID)
|
|
81
|
+
iclouddl -cf cookies.json -A -o ~/Pictures/shared # download them all
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
From a source checkout, run `python3 iclouddl.py` in place of `iclouddl`.
|
|
85
|
+
|
|
86
|
+
| Flag | Meaning |
|
|
87
|
+
|------|---------|
|
|
88
|
+
| `-l, --list` | List shared albums |
|
|
89
|
+
| `-a, --album` | Download one album by index / name / GUID (repeatable) |
|
|
90
|
+
| `-A, --all` | Download every shared album |
|
|
91
|
+
| `-cf, --cookies` | Exported `www.icloud.com.cookies.json` (or `$ICLOUDDL_COOKIES`) |
|
|
92
|
+
| `-o, --output` | Output root (one folder per album) |
|
|
93
|
+
| `-nv, --novideo` | Skip videos |
|
|
94
|
+
| `-f, --force` | Re-download even if already present |
|
|
95
|
+
| `-j, --jobs` | Parallel downloads (default 4) |
|
|
96
|
+
| `--no-convert` | Keep bytes exactly as served (default: fix extensions, transcode true HEIC) |
|
|
97
|
+
| `--no-sidecar` | Don't write JSON sidecars |
|
|
98
|
+
| `--manifest` | Also write `album.json` with raw iCloud records |
|
|
99
|
+
| `--dry-run` | Show what would download; fetch nothing |
|
|
100
|
+
|
|
101
|
+
Runs are **idempotent**: each album keeps a `.iclouddl-index.json`, so re-running skips what's already downloaded and only fetches the rest. Drop the connection, run it again, it finishes the job.
|
|
102
|
+
|
|
103
|
+
## Heads-up on quality
|
|
104
|
+
|
|
105
|
+
iCloud serves shared-album media as **re-encoded ~1536px copies**, not full-resolution originals — that's a limit of iCloud Photo Sharing itself, not this tool. Capture date and camera/lens/exposure survive on every photo; GPS survives on some (Apple strips it from many shared copies, and videos carry none).
|
|
106
|
+
|
|
107
|
+
---
|
|
108
|
+
|
|
109
|
+
Unofficial, AI-assisted, provided as-is. Downloads only shared albums your own account already has access to. Not affiliated with or endorsed by Apple. See [DISCLAIMER.md](https://github.com/upstream-wontfix/ICloudDL/blob/main/DISCLAIMER.md).
|
iclouddl-1.0.0/README.md
ADDED
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
<!-- header:start -->
|
|
2
|
+
<div align="center">
|
|
3
|
+
|
|
4
|
+
<img alt="ICloudDL: The Download button Apple never shipped: save a whole iCloud Shared Album without a $999 glass rectangle." src=".github/assets/banner.svg" width="100%">
|
|
5
|
+
|
|
6
|
+

|
|
7
|
+
[](LICENSE)
|
|
8
|
+
|
|
9
|
+
[Install](#install) · [Use](#use) · [Heads-up on quality](#heads-up-on-quality)
|
|
10
|
+
|
|
11
|
+
</div>
|
|
12
|
+
|
|
13
|
+
<a href="https://haiscale.amolven.com/A1/"><img alt="HAI Scale A1, Mostly automated. Most of the code was written by AI coding agents. I directed the design, reviewed the changes and tested it on real shared albums. Unofficial and not affiliated with Apple. Only download albums you're allowed to save. Provided as is, with no warranty." src=".github/assets/hai.svg" width="100%"></a>
|
|
14
|
+
|
|
15
|
+
<sub>Full terms: [DISCLAIMER.md](DISCLAIMER.md)</sub>
|
|
16
|
+
<!-- header:end -->
|
|
17
|
+
|
|
18
|
+
Download photos and videos from an iCloud share for those without an Apple device.
|
|
19
|
+
|
|
20
|
+
Apple built "Shared Albums" so anyone can look at your photos — then made sure anyone *without* a $999 glass rectangle has to right-click and "Save As" one by one like it's a GeoCities page. ICloudDL is the **Download** button they never got around to shipping to the rest of us: point it at your exported cookies, run one command, and the whole shared album lands in a folder — proper file extensions, EXIF intact, JSON sidecars and all.
|
|
21
|
+
|
|
22
|
+
No Mac, no iPhone, no "just sign in with your Apple ID on your other Apple device."
|
|
23
|
+
|
|
24
|
+
## Install
|
|
25
|
+
|
|
26
|
+
From PyPI:
|
|
27
|
+
|
|
28
|
+
```bash
|
|
29
|
+
pipx install iclouddl # or: uv tool install iclouddl
|
|
30
|
+
pipx install 'iclouddl[heic]' # also transcode genuine HEIC files to JPEG (rarely needed)
|
|
31
|
+
pip install iclouddl # into an existing environment
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
From source:
|
|
35
|
+
|
|
36
|
+
```bash
|
|
37
|
+
pip install -r requirements.txt # needs `exiftool` on PATH for sidecars; pillow-heif only for true-HEIC transcode
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
JSON sidecars read embedded EXIF with the `exiftool` binary, which must be on PATH
|
|
41
|
+
(`apt install libimage-exiftool-perl` / `brew install exiftool`). Without it, sidecars carry the iCloud fields only.
|
|
42
|
+
|
|
43
|
+
## Use
|
|
44
|
+
|
|
45
|
+
Export your `www.icloud.com` cookies to a JSON file while signed in (any "cookies.json" browser extension works), then:
|
|
46
|
+
|
|
47
|
+
```bash
|
|
48
|
+
iclouddl -cf cookies.json -l # list the shared albums you can see
|
|
49
|
+
iclouddl -cf cookies.json -a "Lake Tahoe" -a 1 # download specific albums (name, index, or GUID)
|
|
50
|
+
iclouddl -cf cookies.json -A -o ~/Pictures/shared # download them all
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
From a source checkout, run `python3 iclouddl.py` in place of `iclouddl`.
|
|
54
|
+
|
|
55
|
+
| Flag | Meaning |
|
|
56
|
+
|------|---------|
|
|
57
|
+
| `-l, --list` | List shared albums |
|
|
58
|
+
| `-a, --album` | Download one album by index / name / GUID (repeatable) |
|
|
59
|
+
| `-A, --all` | Download every shared album |
|
|
60
|
+
| `-cf, --cookies` | Exported `www.icloud.com.cookies.json` (or `$ICLOUDDL_COOKIES`) |
|
|
61
|
+
| `-o, --output` | Output root (one folder per album) |
|
|
62
|
+
| `-nv, --novideo` | Skip videos |
|
|
63
|
+
| `-f, --force` | Re-download even if already present |
|
|
64
|
+
| `-j, --jobs` | Parallel downloads (default 4) |
|
|
65
|
+
| `--no-convert` | Keep bytes exactly as served (default: fix extensions, transcode true HEIC) |
|
|
66
|
+
| `--no-sidecar` | Don't write JSON sidecars |
|
|
67
|
+
| `--manifest` | Also write `album.json` with raw iCloud records |
|
|
68
|
+
| `--dry-run` | Show what would download; fetch nothing |
|
|
69
|
+
|
|
70
|
+
Runs are **idempotent**: each album keeps a `.iclouddl-index.json`, so re-running skips what's already downloaded and only fetches the rest. Drop the connection, run it again, it finishes the job.
|
|
71
|
+
|
|
72
|
+
## Heads-up on quality
|
|
73
|
+
|
|
74
|
+
iCloud serves shared-album media as **re-encoded ~1536px copies**, not full-resolution originals — that's a limit of iCloud Photo Sharing itself, not this tool. Capture date and camera/lens/exposure survive on every photo; GPS survives on some (Apple strips it from many shared copies, and videos carry none).
|
|
75
|
+
|
|
76
|
+
---
|
|
77
|
+
|
|
78
|
+
Unofficial, AI-assisted, provided as-is. Downloads only shared albums your own account already has access to. Not affiliated with or endorsed by Apple. See [DISCLAIMER.md](https://github.com/upstream-wontfix/ICloudDL/blob/main/DISCLAIMER.md).
|
|
@@ -0,0 +1,786 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""ICloudDL - Download photos and videos from an iCloud share for those without an Apple device.
|
|
3
|
+
|
|
4
|
+
Unofficial, AI-assisted, provided as-is. Only downloads shared albums your account
|
|
5
|
+
already has access to, using session cookies you export yourself.
|
|
6
|
+
"""
|
|
7
|
+
import argparse
|
|
8
|
+
import base64
|
|
9
|
+
import concurrent.futures as cf
|
|
10
|
+
import datetime as dt
|
|
11
|
+
import errno
|
|
12
|
+
import json
|
|
13
|
+
import os
|
|
14
|
+
import re
|
|
15
|
+
import subprocess
|
|
16
|
+
import sys
|
|
17
|
+
import threading
|
|
18
|
+
import time
|
|
19
|
+
import traceback
|
|
20
|
+
|
|
21
|
+
try:
|
|
22
|
+
import requests
|
|
23
|
+
except ImportError: # handled with a friendly message in main()
|
|
24
|
+
requests = None
|
|
25
|
+
|
|
26
|
+
UA = ("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
|
|
27
|
+
"(KHTML, like Gecko) Chrome/125.0 Safari/537.36")
|
|
28
|
+
CT_TEXT = {"Content-Type": "text/plain;charset=UTF-8"}
|
|
29
|
+
PAGE = 100 # assets fetched per request (URLs are short-lived, so we download each page promptly)
|
|
30
|
+
NET_TRIES = 5 # retries for any single network call
|
|
31
|
+
INDEX_NAME = ".iclouddl-index.json"
|
|
32
|
+
|
|
33
|
+
# resolution fields in the shared-stream records, best first
|
|
34
|
+
RES_PREF = ["resOriginalRes", "resVidMedRes", "resJPEGMedRes", "resVidSmallRes"]
|
|
35
|
+
|
|
36
|
+
_print_lock = threading.Lock()
|
|
37
|
+
VERBOSE = False
|
|
38
|
+
QUIET = False
|
|
39
|
+
_warned = set() # one-time warnings, keyed by tag
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
class ICloudDLError(Exception):
|
|
43
|
+
"""A clean, expected failure — printed as a message, not a traceback."""
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
class _AuthError(ICloudDLError):
|
|
47
|
+
def __init__(self, code):
|
|
48
|
+
self.code = code
|
|
49
|
+
super().__init__(f"auth failed: HTTP {code}")
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
class _FatalIO(ICloudDLError):
|
|
53
|
+
"""Unrecoverable local I/O problem (e.g. disk full) — don't retry."""
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def warn_once(tag, *a):
|
|
57
|
+
if tag not in _warned:
|
|
58
|
+
_warned.add(tag)
|
|
59
|
+
warn(*a)
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
# ----------------------------------------------------------------------------- logging
|
|
63
|
+
def _c(code, s):
|
|
64
|
+
return s if not sys.stdout.isatty() else f"\033[{code}m{s}\033[0m"
|
|
65
|
+
|
|
66
|
+
def info(*a):
|
|
67
|
+
if not QUIET:
|
|
68
|
+
with _print_lock:
|
|
69
|
+
print(*a, flush=True)
|
|
70
|
+
|
|
71
|
+
def good(*a):
|
|
72
|
+
if not QUIET:
|
|
73
|
+
with _print_lock:
|
|
74
|
+
print(_c("32", "✓"), *a, flush=True)
|
|
75
|
+
|
|
76
|
+
def warn(*a):
|
|
77
|
+
with _print_lock:
|
|
78
|
+
print(_c("33", "⚠"), *a, file=sys.stderr, flush=True)
|
|
79
|
+
|
|
80
|
+
def err(*a):
|
|
81
|
+
with _print_lock:
|
|
82
|
+
print(_c("31", "✗"), *a, file=sys.stderr, flush=True)
|
|
83
|
+
|
|
84
|
+
def dbg(*a):
|
|
85
|
+
if VERBOSE and not QUIET:
|
|
86
|
+
with _print_lock:
|
|
87
|
+
print(_c("90", " ·"), *a, flush=True)
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
# ----------------------------------------------------------------------------- helpers
|
|
91
|
+
def retry(fn, desc, tries=NET_TRIES):
|
|
92
|
+
"""Run fn() with exponential backoff; raise only after exhausting retries.
|
|
93
|
+
|
|
94
|
+
Fatal errors (stale cookies, disk full) are re-raised immediately — retrying
|
|
95
|
+
them just wastes time and buries the real message.
|
|
96
|
+
"""
|
|
97
|
+
last = None
|
|
98
|
+
for attempt in range(1, tries + 1):
|
|
99
|
+
try:
|
|
100
|
+
return fn()
|
|
101
|
+
except ICloudDLError:
|
|
102
|
+
raise # not transient; surface it now
|
|
103
|
+
except Exception as e: # noqa: BLE001 - the network layer raises many types
|
|
104
|
+
last = e
|
|
105
|
+
if attempt < tries:
|
|
106
|
+
wait = min(30, 2 ** attempt)
|
|
107
|
+
dbg(f"{desc}: attempt {attempt}/{tries} failed ({type(e).__name__}: {e}); "
|
|
108
|
+
f"retry in {wait}s")
|
|
109
|
+
time.sleep(wait)
|
|
110
|
+
else:
|
|
111
|
+
dbg(f"{desc}: giving up after {tries} attempts ({type(e).__name__}: {e})")
|
|
112
|
+
raise last
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def sanitize(name, fallback="album"):
|
|
116
|
+
name = (name or "").strip()
|
|
117
|
+
name = re.sub(r"[/\\\x00-\x1f]", "_", name)
|
|
118
|
+
name = re.sub(r'[:*?"<>|]', "_", name).rstrip(". ") # also unsafe on Windows/exFAT drives
|
|
119
|
+
# keep within the common 255-byte filename limit (emoji are multi-byte)
|
|
120
|
+
while len(name.encode("utf-8", "ignore")) > 200 and len(name) > 1:
|
|
121
|
+
name = name[:-1]
|
|
122
|
+
return name or fallback
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def b64txt(enc):
|
|
126
|
+
try:
|
|
127
|
+
return base64.b64decode(enc).decode("utf-8", "replace")
|
|
128
|
+
except Exception:
|
|
129
|
+
return None
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def fval(fields, key):
|
|
133
|
+
f = fields.get(key)
|
|
134
|
+
return f.get("value") if isinstance(f, dict) else None
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def ts_iso(ms):
|
|
138
|
+
if ms is None:
|
|
139
|
+
return None
|
|
140
|
+
try:
|
|
141
|
+
return dt.datetime.fromtimestamp(ms / 1000, dt.timezone.utc).isoformat()
|
|
142
|
+
except Exception:
|
|
143
|
+
return None
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def scalar_fields(fields):
|
|
147
|
+
"""Server-side record fields, minus the ASSETID blobs (which hold ephemeral URLs)."""
|
|
148
|
+
out = {}
|
|
149
|
+
for k, v in fields.items():
|
|
150
|
+
if not isinstance(v, dict):
|
|
151
|
+
continue
|
|
152
|
+
t, val = v.get("type"), v.get("value")
|
|
153
|
+
if t == "ASSETID":
|
|
154
|
+
continue
|
|
155
|
+
if t == "BYTES":
|
|
156
|
+
out[k] = b64txt(val)
|
|
157
|
+
elif t == "TIMESTAMP":
|
|
158
|
+
out[k] = {"epoch_ms": val, "iso": ts_iso(val)}
|
|
159
|
+
else:
|
|
160
|
+
out[k] = val
|
|
161
|
+
return out
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
# ----------------------------------------------------------------------------- session / API
|
|
165
|
+
def load_session(cookie_path):
|
|
166
|
+
if not cookie_path:
|
|
167
|
+
raise ICloudDLError(
|
|
168
|
+
"no cookie file given. Pass -cf/--cookies PATH, or set $ICLOUDDL_COOKIES, "
|
|
169
|
+
"pointing at cookies exported from www.icloud.com while signed in.")
|
|
170
|
+
if not os.path.exists(cookie_path):
|
|
171
|
+
raise ICloudDLError(f"cookie file not found: {cookie_path}")
|
|
172
|
+
try:
|
|
173
|
+
with open(cookie_path, encoding="utf-8") as fh:
|
|
174
|
+
raw = json.load(fh)
|
|
175
|
+
except json.JSONDecodeError as e:
|
|
176
|
+
raise ICloudDLError(
|
|
177
|
+
f"cookie file is not valid JSON ({e}). Export it with a 'cookies.json' browser "
|
|
178
|
+
"extension (not the Netscape cookies.txt format).")
|
|
179
|
+
except OSError as e:
|
|
180
|
+
raise ICloudDLError(f"could not read cookie file: {e}")
|
|
181
|
+
|
|
182
|
+
# Accept both a bare list and the common {\"cookies\": [...]} wrapper.
|
|
183
|
+
if isinstance(raw, dict):
|
|
184
|
+
raw = raw.get("cookies") or raw.get("Cookies") or []
|
|
185
|
+
if not isinstance(raw, list) or not raw:
|
|
186
|
+
raise ICloudDLError("cookie file contains no cookies (expected a JSON array of cookie "
|
|
187
|
+
"objects with 'name' and 'value').")
|
|
188
|
+
|
|
189
|
+
s = requests.Session()
|
|
190
|
+
loaded = 0
|
|
191
|
+
for c in raw:
|
|
192
|
+
if not isinstance(c, dict):
|
|
193
|
+
continue
|
|
194
|
+
nm, val = c.get("name"), c.get("value")
|
|
195
|
+
if not nm or val is None:
|
|
196
|
+
continue
|
|
197
|
+
dom = c.get("domain") or ".icloud.com"
|
|
198
|
+
try:
|
|
199
|
+
s.cookies.set(nm, val, domain=dom, path=c.get("path", "/"))
|
|
200
|
+
loaded += 1
|
|
201
|
+
except Exception as e: # noqa: BLE001 - one bad cookie shouldn't sink the rest
|
|
202
|
+
dbg(f"skipped cookie {nm!r}: {e}")
|
|
203
|
+
if loaded == 0:
|
|
204
|
+
raise ICloudDLError("no usable cookies found in the file.")
|
|
205
|
+
if not any(n.startswith(("X-APPLE-WEBAUTH", "X-APPLE-DS-WEB")) for n in s.cookies.keys()):
|
|
206
|
+
warn("cookie file has no X-APPLE-WEBAUTH* tokens — this may not be an iCloud session "
|
|
207
|
+
"export; authentication will probably fail.")
|
|
208
|
+
dbg(f"loaded {loaded} cookies from {cookie_path}")
|
|
209
|
+
s.headers.update({"Origin": "https://www.icloud.com", "Referer": "https://www.icloud.com/",
|
|
210
|
+
"User-Agent": UA})
|
|
211
|
+
return s
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def authenticate(cookie_path):
|
|
215
|
+
s = load_session(cookie_path)
|
|
216
|
+
|
|
217
|
+
def _val():
|
|
218
|
+
try:
|
|
219
|
+
r = s.post("https://setup.icloud.com/setup/ws/1/validate",
|
|
220
|
+
headers={"Content-Type": "application/json"}, data="{}", timeout=30)
|
|
221
|
+
except requests.exceptions.RequestException as e:
|
|
222
|
+
# Let retry() handle transient network blips; it re-raises after N tries.
|
|
223
|
+
raise ConnectionError(f"could not reach setup.icloud.com: {e}") from e
|
|
224
|
+
# Auth failures are not worth retrying: the cookies are stale, full stop.
|
|
225
|
+
if r.status_code in (401, 403, 421):
|
|
226
|
+
raise _AuthError(r.status_code)
|
|
227
|
+
r.raise_for_status()
|
|
228
|
+
try:
|
|
229
|
+
return r.json()
|
|
230
|
+
except ValueError as e:
|
|
231
|
+
raise ICloudDLError(f"validate returned non-JSON (HTTP {r.status_code}): {e}")
|
|
232
|
+
|
|
233
|
+
try:
|
|
234
|
+
data = retry(_val, "validate")
|
|
235
|
+
except _AuthError:
|
|
236
|
+
raise # main() prints the re-export-cookies guidance
|
|
237
|
+
except ConnectionError as e:
|
|
238
|
+
raise ICloudDLError(f"{e}\nCheck your internet connection and try again.")
|
|
239
|
+
|
|
240
|
+
ds = (data or {}).get("dsInfo") or {}
|
|
241
|
+
if not isinstance(data, dict) or data.get("success") is False or not ds:
|
|
242
|
+
raise _AuthError(data.get("error") if isinstance(data, dict) else "unknown")
|
|
243
|
+
dsid = ds.get("dsid")
|
|
244
|
+
ss = (data.get("webservices", {}).get("sharedstreams") or {}).get("url")
|
|
245
|
+
if not dsid or not ss:
|
|
246
|
+
raise ICloudDLError(
|
|
247
|
+
"signed in, but this account has no shared-streams service "
|
|
248
|
+
"(is iCloud Photos / Shared Albums enabled for it?).")
|
|
249
|
+
good(f"authenticated as {ds.get('appleId')} (dsid {dsid})")
|
|
250
|
+
return s, str(dsid), ss
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
def _post_json(s, url, body):
|
|
254
|
+
"""POST text/plain JSON, following a single X-Apple-MMe-Host partition hint, with retries."""
|
|
255
|
+
def _do():
|
|
256
|
+
r = s.post(url, headers=CT_TEXT, data=json.dumps(body), timeout=180)
|
|
257
|
+
if r.status_code == 330 and r.headers.get("x-apple-mme-host"):
|
|
258
|
+
host = r.headers["x-apple-mme-host"]
|
|
259
|
+
u2 = re.sub(r"https://[^/]+", f"https://{host}", url)
|
|
260
|
+
dbg(f"following partition hint -> {host}")
|
|
261
|
+
r = s.post(u2, headers=CT_TEXT, data=json.dumps(body), timeout=180)
|
|
262
|
+
if r.status_code in (401, 403, 421):
|
|
263
|
+
raise _AuthError(r.status_code) # session expired mid-run
|
|
264
|
+
r.raise_for_status()
|
|
265
|
+
return r
|
|
266
|
+
r = retry(_do, f"POST {url.rsplit('/', 1)[-1]}")
|
|
267
|
+
if not r.text.strip():
|
|
268
|
+
return {}
|
|
269
|
+
try:
|
|
270
|
+
return r.json()
|
|
271
|
+
except ValueError as e:
|
|
272
|
+
raise ICloudDLError(f"unexpected non-JSON response from {url.rsplit('/', 1)[-1]}: {e}")
|
|
273
|
+
|
|
274
|
+
|
|
275
|
+
def list_albums(s, dsid, ss):
|
|
276
|
+
data = _post_json(s, f"{ss}/{dsid}/sharedstreams/webgetalbumslist", {})
|
|
277
|
+
albums = []
|
|
278
|
+
for a in data.get("albums", []):
|
|
279
|
+
albums.append({
|
|
280
|
+
"name": (a.get("attributes") or {}).get("name"),
|
|
281
|
+
"guid": a.get("albumguid"),
|
|
282
|
+
"location": a.get("albumlocation"),
|
|
283
|
+
"ownerdsid": a.get("ownerdsid"),
|
|
284
|
+
"sharingtype": a.get("sharingtype"),
|
|
285
|
+
"attributes": a.get("attributes") or {},
|
|
286
|
+
})
|
|
287
|
+
return albums
|
|
288
|
+
|
|
289
|
+
|
|
290
|
+
def album_count(s, album):
|
|
291
|
+
"""Asset count for display only — never fatal; returns None if the summary call fails."""
|
|
292
|
+
try:
|
|
293
|
+
data = _post_json(s, album["location"] + "albumsummary", {"albumguid": album["guid"]})
|
|
294
|
+
return len(data.get("assets", []))
|
|
295
|
+
except _AuthError:
|
|
296
|
+
raise
|
|
297
|
+
except Exception as e: # noqa: BLE001
|
|
298
|
+
dbg(f"albumsummary failed for {album.get('name')!r}: {e}")
|
|
299
|
+
return None
|
|
300
|
+
|
|
301
|
+
|
|
302
|
+
def fetch_page(s, album, offset, page=PAGE):
|
|
303
|
+
# NB: the API's "limit" is the END rank (exclusive), not a count.
|
|
304
|
+
return _post_json(s, album["location"] + "webgetassets",
|
|
305
|
+
{"albumguid": album["guid"], "offset": str(offset),
|
|
306
|
+
"limit": str(offset + page)}).get("records", [])
|
|
307
|
+
|
|
308
|
+
|
|
309
|
+
# ----------------------------------------------------------------------------- download / convert
|
|
310
|
+
def pick_resource(fields):
|
|
311
|
+
for key in RES_PREF:
|
|
312
|
+
f = fields.get(key)
|
|
313
|
+
if isinstance(f, dict) and isinstance(f.get("value"), dict) and f["value"].get("downloadURL"):
|
|
314
|
+
stem = key[3:-3]
|
|
315
|
+
return key, f["value"]["downloadURL"], fval(fields, f"res{stem}FileType")
|
|
316
|
+
return None, None, None
|
|
317
|
+
|
|
318
|
+
|
|
319
|
+
def sniff_type(path):
|
|
320
|
+
"""Return (correct_ext, needs_transcode) from magic bytes."""
|
|
321
|
+
try:
|
|
322
|
+
head = open(path, "rb").read(32)
|
|
323
|
+
except Exception:
|
|
324
|
+
return None, False
|
|
325
|
+
if head[:3] == b"\xff\xd8\xff":
|
|
326
|
+
return ".jpg", False
|
|
327
|
+
if head[:4] == b"\x89PNG":
|
|
328
|
+
return ".png", False
|
|
329
|
+
if head[:6] in (b"GIF87a", b"GIF89a"):
|
|
330
|
+
return ".gif", False
|
|
331
|
+
if head[4:8] == b"ftyp":
|
|
332
|
+
brand = head[8:12]
|
|
333
|
+
if brand in (b"heic", b"heix", b"heif", b"mif1", b"hevc", b"hevx", b"msf1"):
|
|
334
|
+
return ".heic", True
|
|
335
|
+
if brand == b"qt ":
|
|
336
|
+
return ".mov", False
|
|
337
|
+
return ".mp4", False
|
|
338
|
+
return None, False
|
|
339
|
+
|
|
340
|
+
|
|
341
|
+
def transcode_heic(src, dst):
|
|
342
|
+
"""Convert a real HEIC to JPEG, preserving EXIF. Returns True on success."""
|
|
343
|
+
try:
|
|
344
|
+
import pillow_heif # noqa
|
|
345
|
+
from PIL import Image
|
|
346
|
+
except ImportError:
|
|
347
|
+
warn_once("heic-deps",
|
|
348
|
+
"found a real HEIC file but pillow-heif is not installed; keeping it as .heic. "
|
|
349
|
+
"Install extras to transcode: pip install pillow pillow-heif")
|
|
350
|
+
return False
|
|
351
|
+
try:
|
|
352
|
+
pillow_heif.register_heif_opener()
|
|
353
|
+
im = Image.open(src)
|
|
354
|
+
exif = im.info.get("exif")
|
|
355
|
+
im.convert("RGB").save(dst, "JPEG", quality=95, exif=exif if exif else b"")
|
|
356
|
+
return True
|
|
357
|
+
except Exception as e: # noqa: BLE001
|
|
358
|
+
warn(f"HEIC transcode failed for {os.path.basename(src)} ({e}); keeping original")
|
|
359
|
+
return False
|
|
360
|
+
|
|
361
|
+
|
|
362
|
+
def download_url(s, url, dest):
|
|
363
|
+
tmp = dest + ".part"
|
|
364
|
+
|
|
365
|
+
def _do():
|
|
366
|
+
try:
|
|
367
|
+
with s.get(url, stream=True, timeout=300) as resp:
|
|
368
|
+
if resp.status_code in (401, 403, 410):
|
|
369
|
+
# A signed asset URL went stale (rare: only on very slow/huge runs).
|
|
370
|
+
raise ICloudDLError(
|
|
371
|
+
f"asset URL expired (HTTP {resp.status_code}); re-run to resume.")
|
|
372
|
+
resp.raise_for_status()
|
|
373
|
+
expected = resp.headers.get("Content-Length")
|
|
374
|
+
written = 0
|
|
375
|
+
with open(tmp, "wb") as fh:
|
|
376
|
+
for chunk in resp.iter_content(1 << 16):
|
|
377
|
+
if chunk:
|
|
378
|
+
fh.write(chunk)
|
|
379
|
+
written += len(chunk)
|
|
380
|
+
if expected is not None and written != int(expected):
|
|
381
|
+
raise IOError(f"short read: got {written} of {expected} bytes")
|
|
382
|
+
os.replace(tmp, dest)
|
|
383
|
+
except OSError as e:
|
|
384
|
+
if getattr(e, "errno", None) in (errno.ENOSPC, errno.EDQUOT, errno.EROFS):
|
|
385
|
+
raise _FatalIO(f"cannot write {dest}: {os.strerror(e.errno)} — "
|
|
386
|
+
"free up disk space (or pick another --output) and re-run.")
|
|
387
|
+
raise
|
|
388
|
+
finally:
|
|
389
|
+
if os.path.exists(tmp):
|
|
390
|
+
try:
|
|
391
|
+
os.remove(tmp)
|
|
392
|
+
except OSError:
|
|
393
|
+
pass
|
|
394
|
+
retry(_do, f"download {os.path.basename(dest)}")
|
|
395
|
+
|
|
396
|
+
|
|
397
|
+
# ----------------------------------------------------------------------------- EXIF / sidecar
|
|
398
|
+
def run_exiftool(paths):
|
|
399
|
+
"""Read EXIF for many files, in chunks so we never blow the arg limit or stall forever."""
|
|
400
|
+
if not paths:
|
|
401
|
+
return {}
|
|
402
|
+
base = ["exiftool", "-json", "-q", "-api", "largefilesupport=1",
|
|
403
|
+
"-c", "%+.8f", "-d", "%Y-%m-%dT%H:%M:%S"]
|
|
404
|
+
out = {}
|
|
405
|
+
CHUNK = 200
|
|
406
|
+
for i in range(0, len(paths), CHUNK):
|
|
407
|
+
batch = paths[i:i + CHUNK]
|
|
408
|
+
try:
|
|
409
|
+
p = subprocess.run(base + batch, capture_output=True, text=True, timeout=1800)
|
|
410
|
+
except FileNotFoundError:
|
|
411
|
+
warn_once("no-exiftool",
|
|
412
|
+
"exiftool not found on PATH; sidecars will carry iCloud fields only "
|
|
413
|
+
"(no embedded EXIF/GPS). Install: apt install libimage-exiftool-perl "
|
|
414
|
+
"or brew install exiftool")
|
|
415
|
+
return out
|
|
416
|
+
except subprocess.TimeoutExpired:
|
|
417
|
+
warn(f"exiftool timed out on {len(batch)} files; those sidecars lack embedded EXIF")
|
|
418
|
+
continue
|
|
419
|
+
except Exception as e: # noqa: BLE001
|
|
420
|
+
warn(f"exiftool failed: {e}")
|
|
421
|
+
continue
|
|
422
|
+
if not p.stdout.strip():
|
|
423
|
+
if p.stderr.strip():
|
|
424
|
+
dbg(f"exiftool: {p.stderr.strip()[:200]}")
|
|
425
|
+
continue
|
|
426
|
+
try:
|
|
427
|
+
for e in json.loads(p.stdout):
|
|
428
|
+
src = e.pop("SourceFile", None)
|
|
429
|
+
if src:
|
|
430
|
+
out[os.path.abspath(src)] = e
|
|
431
|
+
except ValueError as e:
|
|
432
|
+
warn(f"could not parse exiftool output: {e}")
|
|
433
|
+
return out
|
|
434
|
+
|
|
435
|
+
|
|
436
|
+
def _num(v):
|
|
437
|
+
try:
|
|
438
|
+
return float(str(v).split()[0])
|
|
439
|
+
except Exception:
|
|
440
|
+
return None
|
|
441
|
+
|
|
442
|
+
|
|
443
|
+
def build_sidecar(album, meta, exif):
|
|
444
|
+
gps = None
|
|
445
|
+
lat, lon = exif.get("GPSLatitude"), exif.get("GPSLongitude")
|
|
446
|
+
if lat is not None and lon is not None:
|
|
447
|
+
gps = {"lat": _num(lat), "lon": _num(lon), "alt": _num(exif.get("GPSAltitude"))}
|
|
448
|
+
return {
|
|
449
|
+
"source": "icloud-shared-album",
|
|
450
|
+
"album": album["name"],
|
|
451
|
+
"filename": meta["savedAs"],
|
|
452
|
+
"original_filename": meta["originalFilename"],
|
|
453
|
+
"capture_time": exif.get("DateTimeOriginal") or exif.get("CreateDate")
|
|
454
|
+
or (meta["master_fields"].get("originalCreationDate") or {}).get("iso"),
|
|
455
|
+
"gps": gps,
|
|
456
|
+
"camera": {"make": exif.get("Make"), "model": exif.get("Model"),
|
|
457
|
+
"lens": exif.get("LensModel")},
|
|
458
|
+
"exposure": {"f_number": exif.get("FNumber"), "exposure_time": exif.get("ExposureTime"),
|
|
459
|
+
"iso": exif.get("ISO"), "focal_length_mm": _num(exif.get("FocalLength"))},
|
|
460
|
+
"image": {"width": exif.get("ImageWidth"), "height": exif.get("ImageHeight"),
|
|
461
|
+
"content_type": exif.get("MIMEType")},
|
|
462
|
+
"icloud": {
|
|
463
|
+
"asset_guid": meta.get("assetguid"),
|
|
464
|
+
"master_record": meta.get("masterRecordName"),
|
|
465
|
+
"owner_dsid": album.get("ownerdsid"),
|
|
466
|
+
"sharing_type": album.get("sharingtype"),
|
|
467
|
+
"contributed_by": meta["asset_fields"].get("contributedBy"),
|
|
468
|
+
"added_date": (meta["asset_fields"].get("addedDate") or {}).get("iso"),
|
|
469
|
+
"batch_create_date": (meta["asset_fields"].get("batchCreateDate") or {}).get("iso"),
|
|
470
|
+
"duration_ms": meta["asset_fields"].get("duration"),
|
|
471
|
+
},
|
|
472
|
+
"icloud_fields": {"master": meta["master_fields"], "asset": meta["asset_fields"]},
|
|
473
|
+
"exif_raw": exif,
|
|
474
|
+
"notes": ["iCloud serves shared-album media as re-encoded (~1536px) files, "
|
|
475
|
+
"not full-resolution originals."],
|
|
476
|
+
}
|
|
477
|
+
|
|
478
|
+
|
|
479
|
+
# ----------------------------------------------------------------------------- album driver
|
|
480
|
+
def load_index(folder):
|
|
481
|
+
path = os.path.join(folder, INDEX_NAME)
|
|
482
|
+
if os.path.exists(path):
|
|
483
|
+
try:
|
|
484
|
+
return json.load(open(path))
|
|
485
|
+
except Exception:
|
|
486
|
+
return {}
|
|
487
|
+
return {}
|
|
488
|
+
|
|
489
|
+
|
|
490
|
+
def save_index(folder, index):
|
|
491
|
+
path = os.path.join(folder, INDEX_NAME)
|
|
492
|
+
tmp = path + ".tmp"
|
|
493
|
+
with open(tmp, "w") as fh:
|
|
494
|
+
json.dump(index, fh, indent=2, ensure_ascii=False)
|
|
495
|
+
os.replace(tmp, path)
|
|
496
|
+
|
|
497
|
+
|
|
498
|
+
def deterministic_name(original, assetguid, taken):
|
|
499
|
+
"""Stable filename: original stem, disambiguated by asset GUID (never by order)."""
|
|
500
|
+
base = sanitize(os.path.basename(original or ""), assetguid)
|
|
501
|
+
stem, dot, ext = base.rpartition(".")
|
|
502
|
+
if not dot:
|
|
503
|
+
stem, ext = base, ""
|
|
504
|
+
cand = base
|
|
505
|
+
owner = taken.get(cand.lower())
|
|
506
|
+
if owner and owner != assetguid:
|
|
507
|
+
short = assetguid.split("-")[0][:8]
|
|
508
|
+
cand = f"{stem}_{short}{('.' + ext) if ext else ''}"
|
|
509
|
+
taken[cand.lower()] = assetguid
|
|
510
|
+
return cand
|
|
511
|
+
|
|
512
|
+
|
|
513
|
+
def download_album(s, album, args):
|
|
514
|
+
name = album["name"] or album["guid"]
|
|
515
|
+
folder = os.path.join(args.output, sanitize(album["name"], album["guid"]))
|
|
516
|
+
try:
|
|
517
|
+
os.makedirs(folder, exist_ok=True)
|
|
518
|
+
except OSError as e:
|
|
519
|
+
raise ICloudDLError(f"cannot create output folder {folder!r}: {e}")
|
|
520
|
+
index = {} if args.force else load_index(folder)
|
|
521
|
+
taken = {v["savedAs"].lower(): k for k, v in index.items()
|
|
522
|
+
if isinstance(v, dict) and v.get("savedAs")}
|
|
523
|
+
|
|
524
|
+
total = album_count(s, album)
|
|
525
|
+
info(f"\n=== {name} ({'?' if total is None else total} items) -> {folder}")
|
|
526
|
+
|
|
527
|
+
new_items = [] # (dest_path, meta) downloaded this run (need sidecar/exif)
|
|
528
|
+
index_lock = threading.Lock()
|
|
529
|
+
done = skipped = failed = 0
|
|
530
|
+
|
|
531
|
+
offset = 0
|
|
532
|
+
while True:
|
|
533
|
+
records = retry(lambda: fetch_page(s, album, offset), f"page@{offset}")
|
|
534
|
+
masters = [r for r in records if r.get("recordType") == "CPLMaster"]
|
|
535
|
+
assets_by_master = {}
|
|
536
|
+
for r in records:
|
|
537
|
+
if r.get("recordType") == "CPLAsset":
|
|
538
|
+
mref = (r.get("fields", {}).get("masterRef", {}) or {}).get("value", {})
|
|
539
|
+
mrn = mref.get("recordName") if isinstance(mref, dict) else None
|
|
540
|
+
if mrn:
|
|
541
|
+
assets_by_master[mrn] = r
|
|
542
|
+
if not masters:
|
|
543
|
+
break
|
|
544
|
+
|
|
545
|
+
jobs = []
|
|
546
|
+
for m in masters:
|
|
547
|
+
mf = m.get("fields", {})
|
|
548
|
+
asset = assets_by_master.get(m["recordName"])
|
|
549
|
+
af = asset.get("fields", {}) if asset else {}
|
|
550
|
+
assetguid = asset["recordName"] if asset else m["recordName"]
|
|
551
|
+
key, url, ftype = pick_resource(mf)
|
|
552
|
+
is_video = (ftype or "").startswith(("public.mpeg", "com.apple.quicktime")) \
|
|
553
|
+
or (key or "").startswith(("resVid",))
|
|
554
|
+
if args.novideo and is_video:
|
|
555
|
+
continue
|
|
556
|
+
filename = b64txt(fval(mf, "filenameEnc")) or b64txt(fval(af, "filenameEnc")) \
|
|
557
|
+
or (assetguid + ".bin")
|
|
558
|
+
|
|
559
|
+
# idempotency: already have this asset on disk?
|
|
560
|
+
prev = index.get(assetguid)
|
|
561
|
+
if prev and not args.force and os.path.exists(os.path.join(folder, prev["savedAs"])):
|
|
562
|
+
skipped += 1
|
|
563
|
+
continue
|
|
564
|
+
|
|
565
|
+
with index_lock:
|
|
566
|
+
saved = prev["savedAs"] if (prev and not args.force) \
|
|
567
|
+
else deterministic_name(filename, assetguid, taken)
|
|
568
|
+
meta = {
|
|
569
|
+
"assetguid": assetguid, "masterRecordName": m["recordName"],
|
|
570
|
+
"originalFilename": filename, "savedAs": saved,
|
|
571
|
+
"resourceUsed": key, "fileType": ftype,
|
|
572
|
+
"master_fields": scalar_fields(mf), "asset_fields": scalar_fields(af),
|
|
573
|
+
}
|
|
574
|
+
if url is None:
|
|
575
|
+
failed += 1
|
|
576
|
+
warn(f"no download URL for {filename}")
|
|
577
|
+
continue
|
|
578
|
+
jobs.append((meta, url))
|
|
579
|
+
|
|
580
|
+
def work(job):
|
|
581
|
+
meta, url = job
|
|
582
|
+
dest = os.path.join(folder, meta["savedAs"])
|
|
583
|
+
if args.dry_run:
|
|
584
|
+
return ("dry", meta, dest)
|
|
585
|
+
try:
|
|
586
|
+
download_url(s, url, dest)
|
|
587
|
+
if not args.no_convert:
|
|
588
|
+
ext, needs = sniff_type(dest)
|
|
589
|
+
if needs: # genuine HEIC -> transcode to JPEG
|
|
590
|
+
newdest = os.path.splitext(dest)[0] + ".jpg"
|
|
591
|
+
if transcode_heic(dest, newdest):
|
|
592
|
+
if newdest != dest:
|
|
593
|
+
os.remove(dest)
|
|
594
|
+
dest = newdest
|
|
595
|
+
elif ext: # already a normal format: just correct the extension (lossless)
|
|
596
|
+
if os.path.splitext(dest)[1].lower() != ext:
|
|
597
|
+
newdest = os.path.splitext(dest)[0] + ext
|
|
598
|
+
if os.path.abspath(newdest) != os.path.abspath(dest):
|
|
599
|
+
os.replace(dest, newdest)
|
|
600
|
+
dest = newdest
|
|
601
|
+
meta["savedAs"] = os.path.basename(dest)
|
|
602
|
+
return ("ok", meta, dest)
|
|
603
|
+
except (_FatalIO, _AuthError):
|
|
604
|
+
raise # abort the whole run — retrying every remaining asset is pointless
|
|
605
|
+
except Exception as e: # noqa: BLE001 - never let one asset kill the run
|
|
606
|
+
return ("fail", meta, str(e))
|
|
607
|
+
|
|
608
|
+
with cf.ThreadPoolExecutor(max_workers=max(1, args.jobs)) as pool:
|
|
609
|
+
for status, meta, extra in pool.map(work, jobs):
|
|
610
|
+
if status == "ok":
|
|
611
|
+
done += 1
|
|
612
|
+
new_items.append((extra, meta))
|
|
613
|
+
with index_lock:
|
|
614
|
+
index[meta["assetguid"]] = {"savedAs": meta["savedAs"]}
|
|
615
|
+
if done % 25 == 0:
|
|
616
|
+
info(f" downloaded {done} (skipped {skipped})...")
|
|
617
|
+
elif status == "dry":
|
|
618
|
+
done += 1
|
|
619
|
+
elif status == "fail":
|
|
620
|
+
failed += 1
|
|
621
|
+
err(f" failed: {meta['originalFilename']}: {extra}")
|
|
622
|
+
|
|
623
|
+
if not args.dry_run:
|
|
624
|
+
save_index(folder, index)
|
|
625
|
+
offset += len(masters)
|
|
626
|
+
if len(masters) < PAGE:
|
|
627
|
+
break
|
|
628
|
+
|
|
629
|
+
if args.dry_run:
|
|
630
|
+
info(f" [dry-run] would download {done}, skip {skipped}")
|
|
631
|
+
return
|
|
632
|
+
|
|
633
|
+
# sidecars (idempotent: regenerated deterministically for items fetched this run)
|
|
634
|
+
if not args.no_sidecar and new_items:
|
|
635
|
+
info(f" writing sidecars + EXIF for {len(new_items)} files...")
|
|
636
|
+
exif = run_exiftool([p for p, _ in new_items])
|
|
637
|
+
for dest, meta in new_items:
|
|
638
|
+
side = build_sidecar(album, meta, exif.get(os.path.abspath(dest), {}))
|
|
639
|
+
with open(dest + ".json", "w") as fh:
|
|
640
|
+
json.dump(side, fh, indent=2, ensure_ascii=False)
|
|
641
|
+
|
|
642
|
+
if args.manifest:
|
|
643
|
+
with open(os.path.join(folder, "album.json"), "w") as fh:
|
|
644
|
+
json.dump({"album": album, "count": total, "index": index}, fh,
|
|
645
|
+
indent=2, ensure_ascii=False)
|
|
646
|
+
|
|
647
|
+
media = len([f for f in os.listdir(folder)
|
|
648
|
+
if not f.endswith(".json") and f != INDEX_NAME])
|
|
649
|
+
good(f"{name}: {done} new, {skipped} already present, {failed} failed "
|
|
650
|
+
f"({media} files on disk)")
|
|
651
|
+
if failed:
|
|
652
|
+
warn(f"{failed} asset(s) in {name!r} did not download — re-run the same command to "
|
|
653
|
+
"retry just those (finished files are skipped).")
|
|
654
|
+
|
|
655
|
+
|
|
656
|
+
# ----------------------------------------------------------------------------- resolve / CLI
|
|
657
|
+
def resolve_albums(selectors, albums):
|
|
658
|
+
chosen, seen = [], set()
|
|
659
|
+
for sel in selectors:
|
|
660
|
+
s = sel.strip()
|
|
661
|
+
match = None
|
|
662
|
+
if s.isdigit() and int(s) < len(albums):
|
|
663
|
+
match = albums[int(s)]
|
|
664
|
+
if match is None:
|
|
665
|
+
for a in albums:
|
|
666
|
+
if s.lower() in ((a.get("guid") or "").lower(), (a.get("name") or "").lower()):
|
|
667
|
+
match = a
|
|
668
|
+
break
|
|
669
|
+
if match is None:
|
|
670
|
+
warn(f"no album matches {sel!r}; skipping "
|
|
671
|
+
f"(use -l to see available names/indexes)")
|
|
672
|
+
continue
|
|
673
|
+
if match["guid"] not in seen:
|
|
674
|
+
seen.add(match["guid"])
|
|
675
|
+
chosen.append(match)
|
|
676
|
+
return chosen
|
|
677
|
+
|
|
678
|
+
|
|
679
|
+
__version__ = "1.0.0"
|
|
680
|
+
|
|
681
|
+
EXAMPLES = """\
|
|
682
|
+
examples:
|
|
683
|
+
iclouddl -cf cookies.json -l list shared albums you can access
|
|
684
|
+
iclouddl -cf cookies.json -a "Lake Tahoe" -a 1 download specific albums (name / index / GUID)
|
|
685
|
+
iclouddl -cf cookies.json -A -o ~/Pictures/shared download every album
|
|
686
|
+
iclouddl -cf cookies.json -A --dry-run preview without downloading
|
|
687
|
+
iclouddl -cf cookies.json -A -v verbose: show retries + full tracebacks
|
|
688
|
+
|
|
689
|
+
Re-runs are idempotent: already-downloaded assets are skipped. Add -v (or --debug) when
|
|
690
|
+
reporting a bug so the output shows exactly where it failed.
|
|
691
|
+
"""
|
|
692
|
+
|
|
693
|
+
|
|
694
|
+
def build_parser():
|
|
695
|
+
p = argparse.ArgumentParser(
|
|
696
|
+
prog="iclouddl",
|
|
697
|
+
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
698
|
+
description="Download photos and videos from an iCloud share for those without an Apple device.",
|
|
699
|
+
epilog=EXAMPLES)
|
|
700
|
+
p.add_argument("--version", action="version", version=f"%(prog)s {__version__}")
|
|
701
|
+
# actions (mutually exclusive)
|
|
702
|
+
g = p.add_mutually_exclusive_group()
|
|
703
|
+
g.add_argument("-l", "--list", action="store_true", help="List shared albums you can access")
|
|
704
|
+
g.add_argument("-a", "--album", action="append", metavar="ALBUM",
|
|
705
|
+
help="Download one album by index, name, or GUID (repeatable)")
|
|
706
|
+
g.add_argument("-A", "--all", action="store_true", help="Download every shared album")
|
|
707
|
+
# inputs / options
|
|
708
|
+
p.add_argument("-cf", "--cookies", default=os.environ.get("ICLOUDDL_COOKIES"),
|
|
709
|
+
help="Exported www.icloud.com.cookies.json (or $ICLOUDDL_COOKIES)")
|
|
710
|
+
p.add_argument("-o", "--output", default="icloud-shared", help="Output root directory")
|
|
711
|
+
p.add_argument("-nv", "--novideo", action="store_true", help="Skip videos")
|
|
712
|
+
p.add_argument("-f", "--force", action="store_true", help="Re-download even if already present")
|
|
713
|
+
p.add_argument("-j", "--jobs", type=int, default=4, help="Parallel downloads (default 4)")
|
|
714
|
+
p.add_argument("--no-convert", action="store_true",
|
|
715
|
+
help="Keep bytes exactly as served (default: fix extension / transcode true HEIC)")
|
|
716
|
+
p.add_argument("--no-sidecar", action="store_true", help="Do not write JSON sidecars")
|
|
717
|
+
p.add_argument("--manifest", action="store_true", help="Also write album.json (raw iCloud records)")
|
|
718
|
+
p.add_argument("--dry-run", action="store_true", help="Show what would download; fetch nothing")
|
|
719
|
+
p.add_argument("-q", "--quiet", action="store_true", help="Only print warnings and errors")
|
|
720
|
+
p.add_argument("-v", "--verbose", "--debug", dest="verbose", action="store_true",
|
|
721
|
+
help="Verbose: show retries and full tracebacks (use when reporting a bug)")
|
|
722
|
+
return p
|
|
723
|
+
|
|
724
|
+
|
|
725
|
+
def _run(args):
|
|
726
|
+
"""The actual work; raises ICloudDLError / other exceptions for main() to format."""
|
|
727
|
+
if requests is None:
|
|
728
|
+
raise ICloudDLError("the 'requests' package is required but not installed. "
|
|
729
|
+
"Install dependencies: pip install -r requirements.txt")
|
|
730
|
+
if args.jobs < 1:
|
|
731
|
+
warn(f"--jobs {args.jobs} is invalid; using 1")
|
|
732
|
+
args.jobs = 1
|
|
733
|
+
|
|
734
|
+
s, dsid, ss = authenticate(args.cookies)
|
|
735
|
+
albums = retry(lambda: list_albums(s, dsid, ss), "list albums")
|
|
736
|
+
|
|
737
|
+
if args.list:
|
|
738
|
+
if not albums:
|
|
739
|
+
info("no shared albums found on this account")
|
|
740
|
+
for i, a in enumerate(albums):
|
|
741
|
+
n = album_count(s, a)
|
|
742
|
+
info(f"[{i}] {a['name']} ({'?' if n is None else n} items) {a['guid']}")
|
|
743
|
+
return 0
|
|
744
|
+
|
|
745
|
+
targets = albums if args.all else resolve_albums(args.album, albums)
|
|
746
|
+
if not targets:
|
|
747
|
+
raise ICloudDLError("no matching albums to download (run with -l to list them)")
|
|
748
|
+
for a in targets:
|
|
749
|
+
download_album(s, a, args)
|
|
750
|
+
good(f"\nall done: {len(targets)} album(s) under {os.path.abspath(args.output)}")
|
|
751
|
+
return 0
|
|
752
|
+
|
|
753
|
+
|
|
754
|
+
def main(argv=None):
|
|
755
|
+
global VERBOSE, QUIET
|
|
756
|
+
args = build_parser().parse_args(argv)
|
|
757
|
+
VERBOSE, QUIET = args.verbose, args.quiet
|
|
758
|
+
|
|
759
|
+
if not (args.list or args.all or args.album):
|
|
760
|
+
build_parser().print_help()
|
|
761
|
+
return 0
|
|
762
|
+
|
|
763
|
+
try:
|
|
764
|
+
return _run(args)
|
|
765
|
+
except KeyboardInterrupt:
|
|
766
|
+
err("\ninterrupted. Re-run the same command to resume where it left off.")
|
|
767
|
+
return 130
|
|
768
|
+
except _AuthError as e:
|
|
769
|
+
err(f"iCloud rejected the session (HTTP {e.code}). Your cookies are expired or were "
|
|
770
|
+
"invalidated by signing out.")
|
|
771
|
+
err("Fix: sign in at www.icloud.com, re-export the cookies, overwrite the file, re-run.")
|
|
772
|
+
return 3
|
|
773
|
+
except ICloudDLError as e:
|
|
774
|
+
err(str(e))
|
|
775
|
+
return 1
|
|
776
|
+
except Exception as e: # noqa: BLE001 - last line of defense; keep it legible
|
|
777
|
+
err(f"unexpected error: {type(e).__name__}: {e}")
|
|
778
|
+
if VERBOSE:
|
|
779
|
+
traceback.print_exc()
|
|
780
|
+
else:
|
|
781
|
+
err("re-run with -v (or --debug) to see the full traceback.")
|
|
782
|
+
return 1
|
|
783
|
+
|
|
784
|
+
|
|
785
|
+
if __name__ == "__main__":
|
|
786
|
+
sys.exit(main())
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "iclouddl"
|
|
3
|
+
dynamic = ["version"]
|
|
4
|
+
description = "Download photos and videos from an iCloud shared album, for those without an Apple device"
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
license = "MIT"
|
|
7
|
+
license-files = ["LICENSE"]
|
|
8
|
+
authors = [{ name = "Amol Venkataraman" }]
|
|
9
|
+
requires-python = ">=3.9"
|
|
10
|
+
dependencies = ["requests>=2.28"]
|
|
11
|
+
keywords = ["icloud", "shared-album", "photos", "download", "cli", "exif"]
|
|
12
|
+
classifiers = [
|
|
13
|
+
"Development Status :: 4 - Beta",
|
|
14
|
+
"Environment :: Console",
|
|
15
|
+
"Intended Audience :: End Users/Desktop",
|
|
16
|
+
"Operating System :: OS Independent",
|
|
17
|
+
"Programming Language :: Python :: 3",
|
|
18
|
+
"Programming Language :: Python :: 3 :: Only",
|
|
19
|
+
"Programming Language :: Python :: 3.9",
|
|
20
|
+
"Programming Language :: Python :: 3.10",
|
|
21
|
+
"Programming Language :: Python :: 3.11",
|
|
22
|
+
"Programming Language :: Python :: 3.12",
|
|
23
|
+
"Programming Language :: Python :: 3.13",
|
|
24
|
+
"Topic :: Multimedia :: Graphics",
|
|
25
|
+
"Topic :: Utilities",
|
|
26
|
+
]
|
|
27
|
+
|
|
28
|
+
[project.optional-dependencies]
|
|
29
|
+
# only needed to transcode genuine HEIC files to JPEG (iCloud usually serves JPEG already)
|
|
30
|
+
heic = ["pillow>=10.0", "pillow-heif>=0.13"]
|
|
31
|
+
|
|
32
|
+
[project.scripts]
|
|
33
|
+
iclouddl = "iclouddl:main"
|
|
34
|
+
|
|
35
|
+
[project.urls]
|
|
36
|
+
Homepage = "https://github.com/upstream-wontfix/ICloudDL"
|
|
37
|
+
Repository = "https://github.com/upstream-wontfix/ICloudDL"
|
|
38
|
+
Issues = "https://github.com/upstream-wontfix/ICloudDL/issues"
|
|
39
|
+
|
|
40
|
+
[build-system]
|
|
41
|
+
requires = ["hatchling>=1.27"]
|
|
42
|
+
build-backend = "hatchling.build"
|
|
43
|
+
|
|
44
|
+
[tool.hatch.version]
|
|
45
|
+
path = "iclouddl.py"
|
|
46
|
+
|
|
47
|
+
# explicit allow-lists: the working tree holds exported cookies and downloaded photos
|
|
48
|
+
[tool.hatch.build.targets.sdist]
|
|
49
|
+
only-include = ["iclouddl.py", "README.md", "LICENSE", "DISCLAIMER.md", "requirements.txt", "pyproject.toml"]
|
|
50
|
+
|
|
51
|
+
[tool.hatch.build.targets.wheel]
|
|
52
|
+
only-include = ["iclouddl.py"]
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
# Core
|
|
2
|
+
requests>=2.28
|
|
3
|
+
|
|
4
|
+
# Optional: only needed to transcode genuine HEIC files to JPEG.
|
|
5
|
+
# (iCloud shared albums usually serve JPEG already, so this is rarely exercised.)
|
|
6
|
+
pillow>=10.0
|
|
7
|
+
pillow-heif>=0.13
|
|
8
|
+
|
|
9
|
+
# Note: JSON sidecars read EXIF via the `exiftool` binary, which must be on PATH
|
|
10
|
+
# (e.g. `apt install libimage-exiftool-perl` / `brew install exiftool`).
|
|
11
|
+
# The `pyexiftool` package is NOT required — the tool shells out to `exiftool` directly.
|