devart-dl 4.3.3__tar.gz → 4.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {devart_dl-4.3.3/devart_dl.egg-info → devart_dl-4.4.0}/PKG-INFO +3 -1
- {devart_dl-4.3.3 → devart_dl-4.4.0}/README.en.md +2 -0
- {devart_dl-4.3.3 → devart_dl-4.4.0}/README.md +2 -0
- {devart_dl-4.3.3 → devart_dl-4.4.0}/da_downloader/manager.py +11 -3
- {devart_dl-4.3.3 → devart_dl-4.4.0}/da_downloader/models.py +79 -3
- {devart_dl-4.3.3 → devart_dl-4.4.0}/da_downloader/official_api.py +21 -1
- {devart_dl-4.3.3 → devart_dl-4.4.0}/da_downloader/provider.py +81 -11
- {devart_dl-4.3.3 → devart_dl-4.4.0/devart_dl.egg-info}/PKG-INFO +3 -1
- {devart_dl-4.3.3 → devart_dl-4.4.0}/pyproject.toml +1 -1
- {devart_dl-4.3.3 → devart_dl-4.4.0}/tests/test_manager.py +15 -0
- {devart_dl-4.3.3 → devart_dl-4.4.0}/tests/test_official_provider.py +72 -0
- {devart_dl-4.3.3 → devart_dl-4.4.0}/tests/test_provider.py +43 -0
- {devart_dl-4.3.3 → devart_dl-4.4.0}/.env.example +0 -0
- {devart_dl-4.3.3 → devart_dl-4.4.0}/LICENSE +0 -0
- {devart_dl-4.3.3 → devart_dl-4.4.0}/MANIFEST.in +0 -0
- {devart_dl-4.3.3 → devart_dl-4.4.0}/bin/devart-dl +0 -0
- {devart_dl-4.3.3 → devart_dl-4.4.0}/bin/install.sh +0 -0
- {devart_dl-4.3.3 → devart_dl-4.4.0}/da_downloader/__init__.py +0 -0
- {devart_dl-4.3.3 → devart_dl-4.4.0}/da_downloader/api.py +0 -0
- {devart_dl-4.3.3 → devart_dl-4.4.0}/da_downloader/archive.py +0 -0
- {devart_dl-4.3.3 → devart_dl-4.4.0}/da_downloader/auth.py +0 -0
- {devart_dl-4.3.3 → devart_dl-4.4.0}/da_downloader/cli.py +0 -0
- {devart_dl-4.3.3 → devart_dl-4.4.0}/da_downloader/da_cookie_login.mjs +0 -0
- {devart_dl-4.3.3 → devart_dl-4.4.0}/da_downloader/download.py +0 -0
- {devart_dl-4.3.3 → devart_dl-4.4.0}/da_downloader/errors.py +0 -0
- {devart_dl-4.3.3 → devart_dl-4.4.0}/da_downloader/http.py +0 -0
- {devart_dl-4.3.3 → devart_dl-4.4.0}/da_downloader/oauth.py +0 -0
- {devart_dl-4.3.3 → devart_dl-4.4.0}/da_downloader/path.py +0 -0
- {devart_dl-4.3.3 → devart_dl-4.4.0}/da_downloader/targets.py +0 -0
- {devart_dl-4.3.3 → devart_dl-4.4.0}/devart_dl.egg-info/SOURCES.txt +0 -0
- {devart_dl-4.3.3 → devart_dl-4.4.0}/devart_dl.egg-info/dependency_links.txt +0 -0
- {devart_dl-4.3.3 → devart_dl-4.4.0}/devart_dl.egg-info/entry_points.txt +0 -0
- {devart_dl-4.3.3 → devart_dl-4.4.0}/devart_dl.egg-info/requires.txt +0 -0
- {devart_dl-4.3.3 → devart_dl-4.4.0}/devart_dl.egg-info/top_level.txt +0 -0
- {devart_dl-4.3.3 → devart_dl-4.4.0}/requirements.txt +0 -0
- {devart_dl-4.3.3 → devart_dl-4.4.0}/setup.cfg +0 -0
- {devart_dl-4.3.3 → devart_dl-4.4.0}/setup.py +0 -0
- {devart_dl-4.3.3 → devart_dl-4.4.0}/tests/test_api.py +0 -0
- {devart_dl-4.3.3 → devart_dl-4.4.0}/tests/test_archive.py +0 -0
- {devart_dl-4.3.3 → devart_dl-4.4.0}/tests/test_cli.py +0 -0
- {devart_dl-4.3.3 → devart_dl-4.4.0}/tests/test_conformance.py +0 -0
- {devart_dl-4.3.3 → devart_dl-4.4.0}/tests/test_download.py +0 -0
- {devart_dl-4.3.3 → devart_dl-4.4.0}/tests/test_http_downloader.py +0 -0
- {devart_dl-4.3.3 → devart_dl-4.4.0}/tests/test_path.py +0 -0
- {devart_dl-4.3.3 → devart_dl-4.4.0}/tests/test_targets.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: devart-dl
|
|
3
|
-
Version: 4.
|
|
3
|
+
Version: 4.4.0
|
|
4
4
|
Summary: 专业的 DeviantArt 作品批量下载工具 - Professional DeviantArt batch downloader with anti-ban features
|
|
5
5
|
Author: DeviantArt Downloader Team
|
|
6
6
|
License-Expression: MIT
|
|
@@ -89,11 +89,13 @@ devart-dl login browser # 打开你的 Chrome 登录,自动保存 Cookie,
|
|
|
89
89
|
|
|
90
90
|
- **官方 API(OAuth)优先**:登录一次即可下载原图,无需导出 Cookie,也无需防封延时。
|
|
91
91
|
- **URL 优先**:直接粘贴作品 / 画廊 / 收藏夹 / 标签链接即可下载。
|
|
92
|
+
- **文本作品**:文学 / 日志型 deviation 在响应内直接提取正文,保存为 UTF-8 `.txt`;媒体与文本自动分流。
|
|
92
93
|
- **下载可靠**:HTTP Range 断点续传、自动重试、429 退避、原子落盘。
|
|
93
94
|
- **下载档案**:`--archive` 用 SQLite 记住已下载,跨会话自动跳过。
|
|
94
95
|
- **路径模板 + 元数据 sidecar**:`--directory`/`--filename` 模板与 `--write-info-json`。
|
|
95
96
|
- **代理与降级**:支持 `--proxy`;未登录 OAuth 时自动回退 Cookie。
|
|
96
97
|
- **Resolve 模式**:`devart-dl resolve URL` 输出 `ResolvedDeviation`/`MediaAsset` 的 JSON 契约,不写文件。
|
|
98
|
+
- **付费/订阅锁定识别**:官方 API 返回 `premium_folder_data` / `tier_access`,或网页端非成熟作品的模糊主图(`blur_`),都会识别为「需要订阅/购买」:网页通道跳过,官方通道明确报错,不再把打码图当原图保存。
|
|
97
99
|
|
|
98
100
|
## 文档
|
|
99
101
|
|
|
@@ -20,11 +20,13 @@ See [CHANGELOG.md](CHANGELOG.md) for release notes.
|
|
|
20
20
|
|
|
21
21
|
- **Official API first (OAuth)** — log in once to download originals; no cookie export, no anti-ban delays.
|
|
22
22
|
- **URL-first** — `devart-dl URL` downloads artworks / galleries / favourites / tags / fav.me links directly.
|
|
23
|
+
- **Literature support** — extracts inline literature/journal bodies and saves them as UTF-8 `.txt`, with media and text handled by the same pipeline.
|
|
23
24
|
- **Reliable downloads** — HTTP Range resume, automatic retry, 429 backoff, HTML/empty-file validation, atomic finalize.
|
|
24
25
|
- **Download archive** — `--archive` remembers what you've downloaded in SQLite, skipping it across sessions.
|
|
25
26
|
- **Path templates** — `--directory` / `--filename` support `{id}` `{title}` `{author}` `{published}` `{ext}` and more, with safe sanitization.
|
|
26
27
|
- **Metadata sidecars** — `--write-info-json` writes a `.json` next to each file.
|
|
27
28
|
- **Proxy & fallback** — `--proxy` or proxy environment variables; falls back to cookies when not logged in via OAuth.
|
|
29
|
+
- **Paid / subscription-lock detection** — official API `premium_folder_data` / `tier_access`, or a blurred (`blur_`) non-mature main on the web path, is recognised as subscription/purchase-gated: the web path skips it and the official path fails with a clear message instead of saving a censored placeholder.
|
|
28
30
|
|
|
29
31
|
---
|
|
30
32
|
|
|
@@ -55,11 +55,13 @@ devart-dl login browser # 打开你的 Chrome 登录,自动保存 Cookie,
|
|
|
55
55
|
|
|
56
56
|
- **官方 API(OAuth)优先**:登录一次即可下载原图,无需导出 Cookie,也无需防封延时。
|
|
57
57
|
- **URL 优先**:直接粘贴作品 / 画廊 / 收藏夹 / 标签链接即可下载。
|
|
58
|
+
- **文本作品**:文学 / 日志型 deviation 在响应内直接提取正文,保存为 UTF-8 `.txt`;媒体与文本自动分流。
|
|
58
59
|
- **下载可靠**:HTTP Range 断点续传、自动重试、429 退避、原子落盘。
|
|
59
60
|
- **下载档案**:`--archive` 用 SQLite 记住已下载,跨会话自动跳过。
|
|
60
61
|
- **路径模板 + 元数据 sidecar**:`--directory`/`--filename` 模板与 `--write-info-json`。
|
|
61
62
|
- **代理与降级**:支持 `--proxy`;未登录 OAuth 时自动回退 Cookie。
|
|
62
63
|
- **Resolve 模式**:`devart-dl resolve URL` 输出 `ResolvedDeviation`/`MediaAsset` 的 JSON 契约,不写文件。
|
|
64
|
+
- **付费/订阅锁定识别**:官方 API 返回 `premium_folder_data` / `tier_access`,或网页端非成熟作品的模糊主图(`blur_`),都会识别为「需要订阅/购买」:网页通道跳过,官方通道明确报错,不再把打码图当原图保存。
|
|
63
65
|
|
|
64
66
|
## 文档
|
|
65
67
|
|
|
@@ -73,9 +73,17 @@ class DownloadManager:
|
|
|
73
73
|
return DownloadOutcome(item.artwork_id, "planned", path=path)
|
|
74
74
|
|
|
75
75
|
try:
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
76
|
+
if item.content is not None:
|
|
77
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
78
|
+
part = path.with_name(path.name + ".part")
|
|
79
|
+
with part.open("w", encoding="utf-8", newline="\n") as stream:
|
|
80
|
+
stream.write(item.content)
|
|
81
|
+
part.replace(path)
|
|
82
|
+
result = TransferResult(path, len(item.content.encode("utf-8")))
|
|
83
|
+
else:
|
|
84
|
+
result = self.downloader.download(
|
|
85
|
+
item.media_url, path, overwrite=self.overwrite
|
|
86
|
+
)
|
|
79
87
|
except DeviantArtError as exc:
|
|
80
88
|
return DownloadOutcome(item.artwork_id, "failed", path=path, reason=str(exc))
|
|
81
89
|
|
|
@@ -3,9 +3,12 @@ DownloadItem contract that crosses the provider/downloader boundary."""
|
|
|
3
3
|
|
|
4
4
|
from __future__ import annotations
|
|
5
5
|
|
|
6
|
+
import json
|
|
7
|
+
import re
|
|
6
8
|
from dataclasses import dataclass, field
|
|
7
9
|
from datetime import datetime
|
|
8
10
|
from enum import Enum
|
|
11
|
+
from html import unescape
|
|
9
12
|
from typing import Any, Dict, Mapping, Optional
|
|
10
13
|
|
|
11
14
|
|
|
@@ -29,6 +32,8 @@ class Deviation:
|
|
|
29
32
|
is_downloadable: bool
|
|
30
33
|
is_mature: bool
|
|
31
34
|
deviation_type: str
|
|
35
|
+
premium: bool = False
|
|
36
|
+
text_content: Dict[str, Any] = field(default_factory=dict)
|
|
32
37
|
|
|
33
38
|
@classmethod
|
|
34
39
|
def from_api_response(cls, data: Dict[str, Any]) -> "Deviation":
|
|
@@ -45,6 +50,9 @@ class Deviation:
|
|
|
45
50
|
media = data.get("media") or {}
|
|
46
51
|
if not isinstance(media, dict):
|
|
47
52
|
media = {}
|
|
53
|
+
text_content = data.get("textContent") or data.get("text_content") or {}
|
|
54
|
+
if not isinstance(text_content, dict):
|
|
55
|
+
text_content = {}
|
|
48
56
|
|
|
49
57
|
deviation_type = data.get("type", "unknown")
|
|
50
58
|
if data.get("isVideo"):
|
|
@@ -63,6 +71,8 @@ class Deviation:
|
|
|
63
71
|
),
|
|
64
72
|
is_mature=bool(data.get("isMature", data.get("is_mature", False))),
|
|
65
73
|
deviation_type=str(deviation_type),
|
|
74
|
+
premium=_is_locked(data),
|
|
75
|
+
text_content=text_content,
|
|
66
76
|
)
|
|
67
77
|
|
|
68
78
|
def _extract_extension_from_media(self, media: Dict[str, Any]) -> str:
|
|
@@ -91,12 +101,73 @@ class Deviation:
|
|
|
91
101
|
flags = []
|
|
92
102
|
if self.is_mature:
|
|
93
103
|
flags.append("MATURE")
|
|
104
|
+
if self.premium:
|
|
105
|
+
flags.append("PREMIUM")
|
|
94
106
|
if self.is_downloadable:
|
|
95
107
|
flags.append("DOWNLOADABLE")
|
|
96
108
|
flag_str = f" [{', '.join(flags)}]" if flags else ""
|
|
97
109
|
return f"{self.title} by {self.author}{flag_str}"
|
|
98
110
|
|
|
99
111
|
|
|
112
|
+
def literature_text(payload: Mapping[str, Any]) -> str:
|
|
113
|
+
"""Extract plain UTF-8 text from a DeviantArt literature DTO.
|
|
114
|
+
|
|
115
|
+
DeviantArt currently stores tiptap JSON in ``textContent.html.markup``;
|
|
116
|
+
older journal payloads use HTML. Both are accepted, while non-literature
|
|
117
|
+
DTOs return an empty string.
|
|
118
|
+
"""
|
|
119
|
+
html = payload.get("textContent") or payload.get("text_content") or {}
|
|
120
|
+
if not isinstance(html, dict):
|
|
121
|
+
return ""
|
|
122
|
+
markup = (html.get("html") or {}).get("markup")
|
|
123
|
+
if markup is None:
|
|
124
|
+
return ""
|
|
125
|
+
if isinstance(markup, dict):
|
|
126
|
+
return _tiptap_text(markup).strip()
|
|
127
|
+
if not isinstance(markup, str) or not markup.strip():
|
|
128
|
+
return ""
|
|
129
|
+
try:
|
|
130
|
+
return _tiptap_text(json.loads(markup)).strip()
|
|
131
|
+
except (TypeError, ValueError):
|
|
132
|
+
text = re.sub(r"<br\s*/?>", "\n", markup, flags=re.IGNORECASE)
|
|
133
|
+
text = re.sub(r"</p\s*>", "\n", text, flags=re.IGNORECASE)
|
|
134
|
+
text = re.sub(r"<[^>]+>", "", text)
|
|
135
|
+
return re.sub(r"\n{3,}", "\n\n", unescape(text)).strip()
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def _tiptap_text(node: Any) -> str:
|
|
139
|
+
if not isinstance(node, dict):
|
|
140
|
+
return ""
|
|
141
|
+
parts: list[str] = []
|
|
142
|
+
if isinstance(node.get("text"), str):
|
|
143
|
+
parts.append(node["text"])
|
|
144
|
+
if node.get("type") == "hardBreak":
|
|
145
|
+
parts.append("\n")
|
|
146
|
+
content = node.get("content")
|
|
147
|
+
if isinstance(content, list):
|
|
148
|
+
parts.extend(_tiptap_text(child) for child in content)
|
|
149
|
+
if node.get("type") in {
|
|
150
|
+
"paragraph",
|
|
151
|
+
"bulletList",
|
|
152
|
+
"orderedList",
|
|
153
|
+
"blockquote",
|
|
154
|
+
"codeBlock",
|
|
155
|
+
}:
|
|
156
|
+
parts.append("\n")
|
|
157
|
+
return "".join(parts)
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def _is_locked(data: Dict[str, Any]) -> bool:
|
|
161
|
+
"""True for premium/subscription-locked works (official/web DTO shapes)."""
|
|
162
|
+
premium = data.get("premium_folder_data") or data.get("premiumFolderData")
|
|
163
|
+
if isinstance(premium, dict):
|
|
164
|
+
access = premium.get("has_access", premium.get("hasAccess"))
|
|
165
|
+
if access is False:
|
|
166
|
+
return True
|
|
167
|
+
tier = data.get("tier_access") or data.get("tierAccess")
|
|
168
|
+
return tier in ("locked", "locked-subscribed")
|
|
169
|
+
|
|
170
|
+
|
|
100
171
|
@dataclass(frozen=True)
|
|
101
172
|
class DownloadItem:
|
|
102
173
|
"""A single resolved media file, produced by a provider and consumed by the
|
|
@@ -108,14 +179,16 @@ class DownloadItem:
|
|
|
108
179
|
url: str # deviation page URL (for metadata / archive identity)
|
|
109
180
|
title: str
|
|
110
181
|
author: str
|
|
111
|
-
media_url: str # the actual file to download
|
|
182
|
+
media_url: str # the actual file to download (empty for inline literature)
|
|
112
183
|
extension: Optional[str] = None # without leading dot, e.g. "jpg"
|
|
184
|
+
content: Optional[str] = None # inline UTF-8 payload (literature); no HTTP download
|
|
113
185
|
published_at: Optional[datetime] = None
|
|
114
186
|
mature: bool = False
|
|
115
187
|
metadata: Mapping[str, Any] = field(default_factory=dict)
|
|
116
188
|
|
|
117
189
|
|
|
118
190
|
_MIME_BY_EXT = {
|
|
191
|
+
"txt": "text/plain",
|
|
119
192
|
"jpg": "image/jpeg",
|
|
120
193
|
"jpeg": "image/jpeg",
|
|
121
194
|
"png": "image/png",
|
|
@@ -130,7 +203,7 @@ _MIME_BY_EXT = {
|
|
|
130
203
|
|
|
131
204
|
_VIDEO_EXTS = frozenset({"mp4", "webm", "mov", "m4v"})
|
|
132
205
|
|
|
133
|
-
_DEFAULT_MIME = {"image": "image/jpeg", "video": "video/mp4"}
|
|
206
|
+
_DEFAULT_MIME = {"image": "image/jpeg", "video": "video/mp4", "document": "text/plain"}
|
|
134
207
|
|
|
135
208
|
|
|
136
209
|
@dataclass(frozen=True)
|
|
@@ -199,7 +272,10 @@ def make_media_asset(
|
|
|
199
272
|
mature: bool = False,
|
|
200
273
|
) -> MediaAsset:
|
|
201
274
|
ext = (extension or "").lstrip(".").lower() or None
|
|
202
|
-
|
|
275
|
+
if ext == "txt":
|
|
276
|
+
kind = "document"
|
|
277
|
+
else:
|
|
278
|
+
kind = "video" if ext in _VIDEO_EXTS else "image"
|
|
203
279
|
return MediaAsset(
|
|
204
280
|
id=f"deviantart:{deviation_id}:p{index}",
|
|
205
281
|
deviation_id=deviation_id,
|
|
@@ -316,6 +316,26 @@ def deviation_uuid(init_data: dict) -> str:
|
|
|
316
316
|
return uuid
|
|
317
317
|
|
|
318
318
|
|
|
319
|
+
def is_locked(deviation: dict) -> bool:
|
|
320
|
+
"""Premium/paid works the current account cannot access.
|
|
321
|
+
|
|
322
|
+
The official DTO marks these with ``premium_folder_data.has_access ==
|
|
323
|
+
False`` or ``tier_access`` locked states (matching DAKit); the web DTO may
|
|
324
|
+
use the camelCase pair ``premiumFolderData.hasAccess`` / ``tierAccess``.
|
|
325
|
+
"""
|
|
326
|
+
premium = deviation.get("premium_folder_data") or deviation.get(
|
|
327
|
+
"premiumFolderData"
|
|
328
|
+
)
|
|
329
|
+
if isinstance(premium, dict) and premium.get("has_access", premium.get("hasAccess")) is False:
|
|
330
|
+
return True
|
|
331
|
+
tier = deviation.get("tier_access") or deviation.get("tierAccess")
|
|
332
|
+
return tier in ("locked", "locked-subscribed")
|
|
333
|
+
|
|
334
|
+
|
|
335
|
+
def is_blurred(url: str) -> bool:
|
|
336
|
+
"""True when the URL is a censored ``blur_`` Wix transform (mature/paid)."""
|
|
337
|
+
return "blur_" in url
|
|
338
|
+
|
|
319
339
|
def additional_media_urls(init_data: dict) -> list:
|
|
320
340
|
"""Original-file URLs of a multimedia deviation's extra pages
|
|
321
341
|
(``deviation.extended.additionalMedia``, each entry nests its Wix
|
|
@@ -336,7 +356,7 @@ def additional_media_urls(init_data: dict) -> list:
|
|
|
336
356
|
if not isinstance(media, dict):
|
|
337
357
|
continue
|
|
338
358
|
url = _descriptor_media_url(media)
|
|
339
|
-
if url and
|
|
359
|
+
if url and not is_blurred(url):
|
|
340
360
|
urls.append(url)
|
|
341
361
|
return urls
|
|
342
362
|
|
|
@@ -14,12 +14,14 @@ from typing import Any, Callable, Iterator, Optional, Protocol
|
|
|
14
14
|
from .api import ActionType, DeviantArtAPI
|
|
15
15
|
from .errors import MediaUnavailableError, ParseError
|
|
16
16
|
from .errors import AuthenticationError, NetworkError
|
|
17
|
-
from .models import Deviation, DownloadItem
|
|
17
|
+
from .models import Deviation, DownloadItem, literature_text
|
|
18
18
|
from .official_api import (
|
|
19
19
|
OfficialApiClient,
|
|
20
20
|
additional_media_urls,
|
|
21
21
|
deviation_init,
|
|
22
22
|
deviation_uuid,
|
|
23
|
+
is_blurred,
|
|
24
|
+
is_locked,
|
|
23
25
|
)
|
|
24
26
|
from .targets import DownloadTarget, TargetKind
|
|
25
27
|
|
|
@@ -114,12 +116,34 @@ class WebProvider:
|
|
|
114
116
|
break
|
|
115
117
|
|
|
116
118
|
def _to_item(self, deviation: Deviation) -> Optional[DownloadItem]:
|
|
119
|
+
if deviation.deviation_type == "literature":
|
|
120
|
+
text = literature_text({"textContent": deviation.text_content})
|
|
121
|
+
if text:
|
|
122
|
+
return DownloadItem(
|
|
123
|
+
artwork_id=deviation.deviation_id,
|
|
124
|
+
url=deviation.url,
|
|
125
|
+
title=deviation.title,
|
|
126
|
+
author=deviation.author,
|
|
127
|
+
media_url="",
|
|
128
|
+
extension="txt",
|
|
129
|
+
content=text,
|
|
130
|
+
mature=deviation.is_mature,
|
|
131
|
+
metadata={"kind": "literature"},
|
|
132
|
+
)
|
|
133
|
+
logger.warning("empty literature payload for %r; skipping", deviation.title)
|
|
134
|
+
return None
|
|
135
|
+
|
|
117
136
|
media_url = self.api.get_download_url(deviation, self.quality)
|
|
118
137
|
if media_url is None and self.quality == "o":
|
|
119
138
|
media_url = self.api.get_download_url(deviation, "f")
|
|
120
139
|
if media_url is None:
|
|
121
140
|
logger.warning("no media URL for %r; skipping", deviation.title)
|
|
122
141
|
return None
|
|
142
|
+
# 付费/订阅锁定:非成熟作品的模糊主图 = 当前账号拿不到原图,跳过而不是把
|
|
143
|
+
# 打码图当成作品保存(成熟打码由 official provider / additional pages 处理)。
|
|
144
|
+
if deviation.premium or (not deviation.is_mature and is_blurred(media_url)):
|
|
145
|
+
logger.warning("premium/subscription-locked %r; skipping", deviation.title)
|
|
146
|
+
return None
|
|
123
147
|
return DownloadItem(
|
|
124
148
|
artwork_id=deviation.deviation_id,
|
|
125
149
|
url=deviation.url,
|
|
@@ -207,6 +231,24 @@ class OfficialProvider:
|
|
|
207
231
|
identifier = target.identifier or ""
|
|
208
232
|
init_data = deviation_init(identifier, target.username)
|
|
209
233
|
uuid = deviation_uuid(init_data)
|
|
234
|
+
init_deviation = init_data.get("deviation") if isinstance(init_data, dict) else {}
|
|
235
|
+
if isinstance(init_deviation, dict) and init_deviation.get("type") == "literature":
|
|
236
|
+
text = literature_text(init_deviation)
|
|
237
|
+
if text:
|
|
238
|
+
author = init_deviation.get("author") or {}
|
|
239
|
+
yield DownloadItem(
|
|
240
|
+
artwork_id=uuid,
|
|
241
|
+
url=str(init_deviation.get("url") or ""),
|
|
242
|
+
title=str(init_deviation.get("title") or "Untitled"),
|
|
243
|
+
author=str(author.get("username") or "unknown"),
|
|
244
|
+
media_url="",
|
|
245
|
+
extension="txt",
|
|
246
|
+
content=text,
|
|
247
|
+
published_at=_parse_published(init_deviation.get("published_time")),
|
|
248
|
+
mature=bool(init_deviation.get("is_mature", False)),
|
|
249
|
+
metadata={"kind": "literature", "quality": self.quality},
|
|
250
|
+
)
|
|
251
|
+
return
|
|
210
252
|
main = self._item(self.client.deviation(uuid))
|
|
211
253
|
yield main
|
|
212
254
|
for index, url in enumerate(additional_media_urls(init_data), start=1):
|
|
@@ -244,6 +286,21 @@ class OfficialProvider:
|
|
|
244
286
|
if not uuid:
|
|
245
287
|
raise ParseError("a deviation is missing its id")
|
|
246
288
|
author = deviation.get("author") or {}
|
|
289
|
+
if deviation.get("type") == "literature":
|
|
290
|
+
text = literature_text(deviation)
|
|
291
|
+
if text:
|
|
292
|
+
return DownloadItem(
|
|
293
|
+
artwork_id=uuid,
|
|
294
|
+
url=str(deviation.get("url") or ""),
|
|
295
|
+
title=str(deviation.get("title") or "Untitled"),
|
|
296
|
+
author=str(author.get("username") or "unknown"),
|
|
297
|
+
media_url="",
|
|
298
|
+
extension="txt",
|
|
299
|
+
content=text,
|
|
300
|
+
published_at=_parse_published(deviation.get("published_time")),
|
|
301
|
+
mature=bool(deviation.get("is_mature", False)),
|
|
302
|
+
metadata={"kind": "literature", "quality": self.quality},
|
|
303
|
+
)
|
|
247
304
|
media_url, ext = self._media(deviation, uuid)
|
|
248
305
|
return DownloadItem(
|
|
249
306
|
artwork_id=uuid,
|
|
@@ -262,6 +319,8 @@ class OfficialProvider:
|
|
|
262
319
|
original = self.client.original_download(uuid)
|
|
263
320
|
ext = _ext_from_url(original.filename)
|
|
264
321
|
return original.url, ext
|
|
322
|
+
url: Optional[str] = None
|
|
323
|
+
ext: Optional[str] = None
|
|
265
324
|
content = deviation.get("content") or {}
|
|
266
325
|
# `content.src` is a poster for video works; only videos[].src is playable.
|
|
267
326
|
videos = deviation.get("videos") or []
|
|
@@ -269,16 +328,27 @@ class OfficialProvider:
|
|
|
269
328
|
for video in videos:
|
|
270
329
|
src = video.get("src") if isinstance(video, dict) else None
|
|
271
330
|
if src:
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
if
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
331
|
+
url, ext = str(src), _ext_from_url(str(src))
|
|
332
|
+
break
|
|
333
|
+
if url is None and content.get("src"):
|
|
334
|
+
url, ext = str(content["src"]), _ext_from_url(str(content["src"]))
|
|
335
|
+
if url is None:
|
|
336
|
+
preview = deviation.get("preview") or {}
|
|
337
|
+
if preview.get("src"):
|
|
338
|
+
url, ext = str(preview["src"]), _ext_from_url(str(preview["src"]))
|
|
339
|
+
if url is None:
|
|
340
|
+
thumbs = deviation.get("thumbs") or []
|
|
341
|
+
if thumbs and isinstance(thumbs[0], dict) and thumbs[0].get("src"):
|
|
342
|
+
url, ext = str(thumbs[0]["src"]), _ext_from_url(str(thumbs[0]["src"]))
|
|
343
|
+
if url is None:
|
|
344
|
+
raise MediaUnavailableError(f"no media available for deviation {uuid}")
|
|
345
|
+
# 付费/订阅锁定:非成熟作品的模糊主图(或显式 premium 字段)=拿不到原图,
|
|
346
|
+
# 明确报错而不是保存一张打码占位图。成熟作品由 mature_content=true 提供未打码源。
|
|
347
|
+
if is_locked(deviation) or (not deviation.get("is_mature") and is_blurred(url)):
|
|
348
|
+
raise MediaUnavailableError(
|
|
349
|
+
f"deviation {uuid} requires a subscription/purchase (premium)"
|
|
350
|
+
)
|
|
351
|
+
return url, ext
|
|
282
352
|
|
|
283
353
|
|
|
284
354
|
def _ext_from_url(url: str) -> Optional[str]:
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: devart-dl
|
|
3
|
-
Version: 4.
|
|
3
|
+
Version: 4.4.0
|
|
4
4
|
Summary: 专业的 DeviantArt 作品批量下载工具 - Professional DeviantArt batch downloader with anti-ban features
|
|
5
5
|
Author: DeviantArt Downloader Team
|
|
6
6
|
License-Expression: MIT
|
|
@@ -89,11 +89,13 @@ devart-dl login browser # 打开你的 Chrome 登录,自动保存 Cookie,
|
|
|
89
89
|
|
|
90
90
|
- **官方 API(OAuth)优先**:登录一次即可下载原图,无需导出 Cookie,也无需防封延时。
|
|
91
91
|
- **URL 优先**:直接粘贴作品 / 画廊 / 收藏夹 / 标签链接即可下载。
|
|
92
|
+
- **文本作品**:文学 / 日志型 deviation 在响应内直接提取正文,保存为 UTF-8 `.txt`;媒体与文本自动分流。
|
|
92
93
|
- **下载可靠**:HTTP Range 断点续传、自动重试、429 退避、原子落盘。
|
|
93
94
|
- **下载档案**:`--archive` 用 SQLite 记住已下载,跨会话自动跳过。
|
|
94
95
|
- **路径模板 + 元数据 sidecar**:`--directory`/`--filename` 模板与 `--write-info-json`。
|
|
95
96
|
- **代理与降级**:支持 `--proxy`;未登录 OAuth 时自动回退 Cookie。
|
|
96
97
|
- **Resolve 模式**:`devart-dl resolve URL` 输出 `ResolvedDeviation`/`MediaAsset` 的 JSON 契约,不写文件。
|
|
98
|
+
- **付费/订阅锁定识别**:官方 API 返回 `premium_folder_data` / `tier_access`,或网页端非成熟作品的模糊主图(`blur_`),都会识别为「需要订阅/购买」:网页通道跳过,官方通道明确报错,不再把打码图当原图保存。
|
|
97
99
|
|
|
98
100
|
## 文档
|
|
99
101
|
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "devart-dl"
|
|
7
|
-
version = "4.
|
|
7
|
+
version = "4.4.0"
|
|
8
8
|
description = "专业的 DeviantArt 作品批量下载工具 - Professional DeviantArt batch downloader with anti-ban features"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.10"
|
|
@@ -156,3 +156,18 @@ def test_dry_run_plans_without_writing(tmp_path: Path) -> None:
|
|
|
156
156
|
assert not outcome.path.exists()
|
|
157
157
|
assert downloader.calls == [] # never touched the network or disk
|
|
158
158
|
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def test_inline_literature_writes_text_without_http(tmp_path: Path) -> None:
|
|
162
|
+
downloader = FakeDownloader()
|
|
163
|
+
manager = make_manager(tmp_path, downloader=downloader)
|
|
164
|
+
item = make_item(media_url="", extension="txt", content="Chapter\n\nText")
|
|
165
|
+
|
|
166
|
+
outcome = manager.run(item)
|
|
167
|
+
|
|
168
|
+
assert outcome.status == "downloaded"
|
|
169
|
+
assert outcome.path is not None
|
|
170
|
+
assert outcome.path.read_bytes() == "Chapter\n\nText".encode("utf-8")
|
|
171
|
+
assert outcome.size == len("Chapter\n\nText".encode("utf-8"))
|
|
172
|
+
assert downloader.calls == []
|
|
173
|
+
assert not outcome.path.with_name(outcome.path.name + ".part").exists()
|
|
@@ -274,6 +274,33 @@ def test_no_media_raises(monkeypatch) -> None:
|
|
|
274
274
|
list(provider.resolve(TargetParser.parse("123456")))
|
|
275
275
|
|
|
276
276
|
|
|
277
|
+
def test_blurred_non_mature_main_raises_premium(monkeypatch) -> None:
|
|
278
|
+
monkeypatch.setattr(provider_mod, "deviation_init", lambda identifier, username=None, **_: {"deviation": {"extended": {"deviationUuid": "uuid-1", "additionalMedia": []}}})
|
|
279
|
+
client = FakeClient()
|
|
280
|
+
client.deviations["uuid-1"] = make_deviation(
|
|
281
|
+
uuid="uuid-1",
|
|
282
|
+
content_src="https://images.test/blur_40/full.jpg",
|
|
283
|
+
is_downloadable=False,
|
|
284
|
+
mature=False,
|
|
285
|
+
)
|
|
286
|
+
provider = OfficialProvider(client, quality="f")
|
|
287
|
+
|
|
288
|
+
with pytest.raises(MediaUnavailableError, match="subscription/purchase"):
|
|
289
|
+
list(provider.resolve(TargetParser.parse("123456")))
|
|
290
|
+
|
|
291
|
+
|
|
292
|
+
def test_premium_folder_without_access_raises_premium(monkeypatch) -> None:
|
|
293
|
+
monkeypatch.setattr(provider_mod, "deviation_init", lambda identifier, username=None, **_: {"deviation": {"extended": {"deviationUuid": "uuid-1", "additionalMedia": []}}})
|
|
294
|
+
client = FakeClient()
|
|
295
|
+
deviation = make_deviation(uuid="uuid-1", is_downloadable=False)
|
|
296
|
+
deviation["premium_folder_data"] = {"has_access": False}
|
|
297
|
+
client.deviations["uuid-1"] = deviation
|
|
298
|
+
provider = OfficialProvider(client, quality="f")
|
|
299
|
+
|
|
300
|
+
with pytest.raises(MediaUnavailableError, match="subscription/purchase"):
|
|
301
|
+
list(provider.resolve(TargetParser.parse("123456")))
|
|
302
|
+
|
|
303
|
+
|
|
277
304
|
def test_tag_target() -> None:
|
|
278
305
|
client = FakeClient()
|
|
279
306
|
client.pages[("browse_tags", "landscape", 0)] = {
|
|
@@ -306,3 +333,48 @@ def test_resolve_short_id_follows_redirect() -> None:
|
|
|
306
333
|
return FakeResponse()
|
|
307
334
|
|
|
308
335
|
assert _resolve_short_id("abc123", FakeHttp(), 30) == "123456"
|
|
336
|
+
|
|
337
|
+
|
|
338
|
+
def test_artwork_literature_uses_init_text_and_skips_media_api(monkeypatch) -> None:
|
|
339
|
+
markup = "{\"type\":\"doc\",\"content\":[{\"type\":\"paragraph\",\"content\":[{\"type\":\"text\",\"text\":\"Once upon a time.\"}]}]}"
|
|
340
|
+
init = {
|
|
341
|
+
"deviation": {
|
|
342
|
+
"extended": {"deviationUuid": "uuid-1", "additionalMedia": []},
|
|
343
|
+
"type": "literature",
|
|
344
|
+
"title": "Story",
|
|
345
|
+
"author": {"username": "alice"},
|
|
346
|
+
"url": "https://www.deviantart.com/alice/art/story-123456",
|
|
347
|
+
"textContent": {"html": {"markup": markup}},
|
|
348
|
+
}
|
|
349
|
+
}
|
|
350
|
+
monkeypatch.setattr(provider_mod, "deviation_init", lambda identifier, username=None, **_: init)
|
|
351
|
+
provider = OfficialProvider(FakeClient(), quality="o")
|
|
352
|
+
|
|
353
|
+
items = list(provider.resolve(TargetParser.parse("123456")))
|
|
354
|
+
|
|
355
|
+
assert items[0].content == "Once upon a time."
|
|
356
|
+
assert items[0].media_url == ""
|
|
357
|
+
assert items[0].extension == "txt"
|
|
358
|
+
assert items[0].metadata == {"kind": "literature", "quality": "o"}
|
|
359
|
+
|
|
360
|
+
|
|
361
|
+
def test_gallery_literature_is_inline_text() -> None:
|
|
362
|
+
client = FakeClient()
|
|
363
|
+
client.pages[("gallery_all", "alice", 0)] = {
|
|
364
|
+
"results": [{
|
|
365
|
+
"deviationid": "uuid-1",
|
|
366
|
+
"title": "Story",
|
|
367
|
+
"author": {"username": "alice"},
|
|
368
|
+
"url": "https://www.deviantart.com/alice/art/story-1",
|
|
369
|
+
"type": "literature",
|
|
370
|
+
"textContent": {"html": {"markup": "{\"type\":\"doc\",\"content\":[{\"type\":\"text\",\"text\":\"Text\"}]}"}},
|
|
371
|
+
}],
|
|
372
|
+
"has_more": False,
|
|
373
|
+
}
|
|
374
|
+
provider = OfficialProvider(client, quality="o")
|
|
375
|
+
|
|
376
|
+
items = list(provider.resolve(TargetParser.parse("https://www.deviantart.com/alice/gallery")))
|
|
377
|
+
|
|
378
|
+
assert items[0].content == "Text"
|
|
379
|
+
assert items[0].media_url == ""
|
|
380
|
+
assert items[0].extension == "txt"
|
|
@@ -74,6 +74,31 @@ def test_gallery_yields_items_with_media_url_and_extension() -> None:
|
|
|
74
74
|
assert items[0].extension == "png"
|
|
75
75
|
|
|
76
76
|
|
|
77
|
+
def test_blurred_non_mature_item_is_skipped() -> None:
|
|
78
|
+
dev = make_deviation()
|
|
79
|
+
provider = make_provider(
|
|
80
|
+
pages=[([dev], False, 1, "")],
|
|
81
|
+
media_urls={"123": "https://images.test/blur_40/123.jpg"},
|
|
82
|
+
)
|
|
83
|
+
|
|
84
|
+
items = list(provider.resolve(TargetParser.parse("https://www.deviantart.com/alice/gallery")))
|
|
85
|
+
|
|
86
|
+
assert items == []
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def test_premium_locked_item_is_skipped() -> None:
|
|
90
|
+
dev = make_deviation()
|
|
91
|
+
dev.premium = True
|
|
92
|
+
provider = make_provider(
|
|
93
|
+
pages=[([dev], False, 1, "")],
|
|
94
|
+
media_urls={"123": "https://images.test/123.jpg"},
|
|
95
|
+
)
|
|
96
|
+
|
|
97
|
+
items = list(provider.resolve(TargetParser.parse("https://www.deviantart.com/alice/gallery")))
|
|
98
|
+
|
|
99
|
+
assert items == []
|
|
100
|
+
|
|
101
|
+
|
|
77
102
|
def test_pagination_across_two_pages() -> None:
|
|
78
103
|
first = make_deviation("1")
|
|
79
104
|
second = make_deviation("2")
|
|
@@ -224,3 +249,21 @@ def test_composite_tag_without_official_requires_official() -> None:
|
|
|
224
249
|
router = CompositeProvider(official=None, web=web)
|
|
225
250
|
with pytest.raises(MediaUnavailableError):
|
|
226
251
|
list(router.resolve(TargetParser.parse("https://www.deviantart.com/tag/landscape")))
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
def test_literature_item_is_delivered_as_inline_text() -> None:
|
|
255
|
+
dev = make_deviation("123", title="Story")
|
|
256
|
+
dev.deviation_type = "literature"
|
|
257
|
+
dev.media = {}
|
|
258
|
+
dev.text_content = {
|
|
259
|
+
"html": {"markup": "{\"type\":\"doc\",\"content\":[{\"type\":\"paragraph\",\"content\":[{\"type\":\"text\",\"text\":\"Chapter one\"}]},{\"type\":\"paragraph\",\"content\":[{\"type\":\"text\",\"text\":\"Chapter two\"}]}]}"}
|
|
260
|
+
}
|
|
261
|
+
provider = make_provider(pages=[([dev], False, 1, "")])
|
|
262
|
+
|
|
263
|
+
items = list(provider.resolve(TargetParser.parse("https://www.deviantart.com/alice/gallery")))
|
|
264
|
+
|
|
265
|
+
assert len(items) == 1
|
|
266
|
+
assert items[0].content == "Chapter one\nChapter two"
|
|
267
|
+
assert items[0].media_url == ""
|
|
268
|
+
assert items[0].extension == "txt"
|
|
269
|
+
assert items[0].metadata == {"kind": "literature"}
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|