devart-dl 4.3.4__tar.gz → 4.4.0__tar.gz

{devart_dl-4.3.4/devart_dl.egg-info → devart_dl-4.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: devart-dl
-Version: 4.3.4
+Version: 4.4.0
 Summary: 专业的 DeviantArt 作品批量下载工具 - Professional DeviantArt batch downloader with anti-ban features
 Author: DeviantArt Downloader Team
 License-Expression: MIT
@@ -89,6 +89,7 @@ devart-dl login browser   # 打开你的 Chrome 登录，自动保存 Cookie，
 
 - **官方 API（OAuth）优先**：登录一次即可下载原图，无需导出 Cookie，也无需防封延时。
 - **URL 优先**：直接粘贴作品 / 画廊 / 收藏夹 / 标签链接即可下载。
+- **文本作品**：文学 / 日志型 deviation 在响应内直接提取正文，保存为 UTF-8 `.txt`；媒体与文本自动分流。
 - **下载可靠**：HTTP Range 断点续传、自动重试、429 退避、原子落盘。
 - **下载档案**：`--archive` 用 SQLite 记住已下载，跨会话自动跳过。
 - **路径模板 + 元数据 sidecar**：`--directory`/`--filename` 模板与 `--write-info-json`。

{devart_dl-4.3.4 → devart_dl-4.4.0}/README.en.md

@@ -20,6 +20,7 @@ See [CHANGELOG.md](CHANGELOG.md) for release notes.
 
 - **Official API first (OAuth)** — log in once to download originals; no cookie export, no anti-ban delays.
 - **URL-first** — `devart-dl URL` downloads artworks / galleries / favourites / tags / fav.me links directly.
+- **Literature support** — extracts inline literature/journal bodies and saves them as UTF-8 `.txt`, with media and text handled by the same pipeline.
 - **Reliable downloads** — HTTP Range resume, automatic retry, 429 backoff, HTML/empty-file validation, atomic finalize.
 - **Download archive** — `--archive` remembers what you've downloaded in SQLite, skipping it across sessions.
 - **Path templates** — `--directory` / `--filename` support `{id}` `{title}` `{author}` `{published}` `{ext}` and more, with safe sanitization.

{devart_dl-4.3.4 → devart_dl-4.4.0}/README.md

@@ -55,6 +55,7 @@ devart-dl login browser   # 打开你的 Chrome 登录，自动保存 Cookie，
 
 - **官方 API（OAuth）优先**：登录一次即可下载原图，无需导出 Cookie，也无需防封延时。
 - **URL 优先**：直接粘贴作品 / 画廊 / 收藏夹 / 标签链接即可下载。
+- **文本作品**：文学 / 日志型 deviation 在响应内直接提取正文，保存为 UTF-8 `.txt`；媒体与文本自动分流。
 - **下载可靠**：HTTP Range 断点续传、自动重试、429 退避、原子落盘。
 - **下载档案**：`--archive` 用 SQLite 记住已下载，跨会话自动跳过。
 - **路径模板 + 元数据 sidecar**：`--directory`/`--filename` 模板与 `--write-info-json`。

{devart_dl-4.3.4 → devart_dl-4.4.0}/da_downloader/manager.py

@@ -73,9 +73,17 @@ class DownloadManager:
             return DownloadOutcome(item.artwork_id, "planned", path=path)
 
         try:
-            result: TransferResult = self.downloader.download(
-                item.media_url, path, overwrite=self.overwrite
-            )
+            if item.content is not None:
+                path.parent.mkdir(parents=True, exist_ok=True)
+                part = path.with_name(path.name + ".part")
+                with part.open("w", encoding="utf-8", newline="\n") as stream:
+                    stream.write(item.content)
+                part.replace(path)
+                result = TransferResult(path, len(item.content.encode("utf-8")))
+            else:
+                result = self.downloader.download(
+                    item.media_url, path, overwrite=self.overwrite
+                )
         except DeviantArtError as exc:
             return DownloadOutcome(item.artwork_id, "failed", path=path, reason=str(exc))
 

{devart_dl-4.3.4 → devart_dl-4.4.0}/da_downloader/models.py

@@ -3,9 +3,12 @@ DownloadItem contract that crosses the provider/downloader boundary."""
 
 from __future__ import annotations
 
+import json
+import re
 from dataclasses import dataclass, field
 from datetime import datetime
 from enum import Enum
+from html import unescape
 from typing import Any, Dict, Mapping, Optional
 
 
@@ -30,6 +33,7 @@ class Deviation:
     is_mature: bool
     deviation_type: str
     premium: bool = False
+    text_content: Dict[str, Any] = field(default_factory=dict)
 
     @classmethod
     def from_api_response(cls, data: Dict[str, Any]) -> "Deviation":
@@ -46,6 +50,9 @@ class Deviation:
         media = data.get("media") or {}
         if not isinstance(media, dict):
             media = {}
+        text_content = data.get("textContent") or data.get("text_content") or {}
+        if not isinstance(text_content, dict):
+            text_content = {}
 
         deviation_type = data.get("type", "unknown")
         if data.get("isVideo"):
@@ -65,6 +72,7 @@ class Deviation:
             is_mature=bool(data.get("isMature", data.get("is_mature", False))),
             deviation_type=str(deviation_type),
             premium=_is_locked(data),
+            text_content=text_content,
         )
 
     def _extract_extension_from_media(self, media: Dict[str, Any]) -> str:
@@ -101,6 +109,54 @@ class Deviation:
         return f"{self.title} by {self.author}{flag_str}"
 
 
+def literature_text(payload: Mapping[str, Any]) -> str:
+    """Extract plain UTF-8 text from a DeviantArt literature DTO.
+
+    DeviantArt currently stores tiptap JSON in ``textContent.html.markup``;
+    older journal payloads use HTML. Both are accepted, while non-literature
+    DTOs return an empty string.
+    """
+    html = payload.get("textContent") or payload.get("text_content") or {}
+    if not isinstance(html, dict):
+        return ""
+    markup = (html.get("html") or {}).get("markup")
+    if markup is None:
+        return ""
+    if isinstance(markup, dict):
+        return _tiptap_text(markup).strip()
+    if not isinstance(markup, str) or not markup.strip():
+        return ""
+    try:
+        return _tiptap_text(json.loads(markup)).strip()
+    except (TypeError, ValueError):
+        text = re.sub(r"<br\s*/?>", "\n", markup, flags=re.IGNORECASE)
+        text = re.sub(r"</p\s*>", "\n", text, flags=re.IGNORECASE)
+        text = re.sub(r"<[^>]+>", "", text)
+        return re.sub(r"\n{3,}", "\n\n", unescape(text)).strip()
+
+
+def _tiptap_text(node: Any) -> str:
+    if not isinstance(node, dict):
+        return ""
+    parts: list[str] = []
+    if isinstance(node.get("text"), str):
+        parts.append(node["text"])
+    if node.get("type") == "hardBreak":
+        parts.append("\n")
+    content = node.get("content")
+    if isinstance(content, list):
+        parts.extend(_tiptap_text(child) for child in content)
+    if node.get("type") in {
+        "paragraph",
+        "bulletList",
+        "orderedList",
+        "blockquote",
+        "codeBlock",
+    }:
+        parts.append("\n")
+    return "".join(parts)
+
+
 def _is_locked(data: Dict[str, Any]) -> bool:
     """True for premium/subscription-locked works (official/web DTO shapes)."""
     premium = data.get("premium_folder_data") or data.get("premiumFolderData")
@@ -123,14 +179,16 @@ class DownloadItem:
     url: str  # deviation page URL (for metadata / archive identity)
     title: str
     author: str
-    media_url: str  # the actual file to download
+    media_url: str  # the actual file to download (empty for inline literature)
     extension: Optional[str] = None  # without leading dot, e.g. "jpg"
+    content: Optional[str] = None  # inline UTF-8 payload (literature); no HTTP download
     published_at: Optional[datetime] = None
     mature: bool = False
     metadata: Mapping[str, Any] = field(default_factory=dict)
 
 
 _MIME_BY_EXT = {
+    "txt": "text/plain",
     "jpg": "image/jpeg",
     "jpeg": "image/jpeg",
     "png": "image/png",
@@ -145,7 +203,7 @@ _MIME_BY_EXT = {
 
 _VIDEO_EXTS = frozenset({"mp4", "webm", "mov", "m4v"})
 
-_DEFAULT_MIME = {"image": "image/jpeg", "video": "video/mp4"}
+_DEFAULT_MIME = {"image": "image/jpeg", "video": "video/mp4", "document": "text/plain"}
 
 
 @dataclass(frozen=True)
@@ -214,7 +272,10 @@ def make_media_asset(
     mature: bool = False,
 ) -> MediaAsset:
     ext = (extension or "").lstrip(".").lower() or None
-    kind = "video" if ext in _VIDEO_EXTS else "image"
+    if ext == "txt":
+        kind = "document"
+    else:
+        kind = "video" if ext in _VIDEO_EXTS else "image"
     return MediaAsset(
         id=f"deviantart:{deviation_id}:p{index}",
         deviation_id=deviation_id,

{devart_dl-4.3.4 → devart_dl-4.4.0}/da_downloader/provider.py

@@ -14,7 +14,7 @@ from typing import Any, Callable, Iterator, Optional, Protocol
 from .api import ActionType, DeviantArtAPI
 from .errors import MediaUnavailableError, ParseError
 from .errors import AuthenticationError, NetworkError
-from .models import Deviation, DownloadItem
+from .models import Deviation, DownloadItem, literature_text
 from .official_api import (
     OfficialApiClient,
     additional_media_urls,
@@ -116,6 +116,23 @@ class WebProvider:
                 break
 
     def _to_item(self, deviation: Deviation) -> Optional[DownloadItem]:
+        if deviation.deviation_type == "literature":
+            text = literature_text({"textContent": deviation.text_content})
+            if text:
+                return DownloadItem(
+                    artwork_id=deviation.deviation_id,
+                    url=deviation.url,
+                    title=deviation.title,
+                    author=deviation.author,
+                    media_url="",
+                    extension="txt",
+                    content=text,
+                    mature=deviation.is_mature,
+                    metadata={"kind": "literature"},
+                )
+            logger.warning("empty literature payload for %r; skipping", deviation.title)
+            return None
+
         media_url = self.api.get_download_url(deviation, self.quality)
         if media_url is None and self.quality == "o":
             media_url = self.api.get_download_url(deviation, "f")
@@ -214,6 +231,24 @@ class OfficialProvider:
         identifier = target.identifier or ""
         init_data = deviation_init(identifier, target.username)
         uuid = deviation_uuid(init_data)
+        init_deviation = init_data.get("deviation") if isinstance(init_data, dict) else {}
+        if isinstance(init_deviation, dict) and init_deviation.get("type") == "literature":
+            text = literature_text(init_deviation)
+            if text:
+                author = init_deviation.get("author") or {}
+                yield DownloadItem(
+                    artwork_id=uuid,
+                    url=str(init_deviation.get("url") or ""),
+                    title=str(init_deviation.get("title") or "Untitled"),
+                    author=str(author.get("username") or "unknown"),
+                    media_url="",
+                    extension="txt",
+                    content=text,
+                    published_at=_parse_published(init_deviation.get("published_time")),
+                    mature=bool(init_deviation.get("is_mature", False)),
+                    metadata={"kind": "literature", "quality": self.quality},
+                )
+                return
         main = self._item(self.client.deviation(uuid))
         yield main
         for index, url in enumerate(additional_media_urls(init_data), start=1):
@@ -251,6 +286,21 @@ class OfficialProvider:
         if not uuid:
             raise ParseError("a deviation is missing its id")
         author = deviation.get("author") or {}
+        if deviation.get("type") == "literature":
+            text = literature_text(deviation)
+            if text:
+                return DownloadItem(
+                    artwork_id=uuid,
+                    url=str(deviation.get("url") or ""),
+                    title=str(deviation.get("title") or "Untitled"),
+                    author=str(author.get("username") or "unknown"),
+                    media_url="",
+                    extension="txt",
+                    content=text,
+                    published_at=_parse_published(deviation.get("published_time")),
+                    mature=bool(deviation.get("is_mature", False)),
+                    metadata={"kind": "literature", "quality": self.quality},
+                )
         media_url, ext = self._media(deviation, uuid)
         return DownloadItem(
             artwork_id=uuid,

{devart_dl-4.3.4 → devart_dl-4.4.0/devart_dl.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: devart-dl
-Version: 4.3.4
+Version: 4.4.0
 Summary: 专业的 DeviantArt 作品批量下载工具 - Professional DeviantArt batch downloader with anti-ban features
 Author: DeviantArt Downloader Team
 License-Expression: MIT
@@ -89,6 +89,7 @@ devart-dl login browser   # 打开你的 Chrome 登录，自动保存 Cookie，
 
 - **官方 API（OAuth）优先**：登录一次即可下载原图，无需导出 Cookie，也无需防封延时。
 - **URL 优先**：直接粘贴作品 / 画廊 / 收藏夹 / 标签链接即可下载。
+- **文本作品**：文学 / 日志型 deviation 在响应内直接提取正文，保存为 UTF-8 `.txt`；媒体与文本自动分流。
 - **下载可靠**：HTTP Range 断点续传、自动重试、429 退避、原子落盘。
 - **下载档案**：`--archive` 用 SQLite 记住已下载，跨会话自动跳过。
 - **路径模板 + 元数据 sidecar**：`--directory`/`--filename` 模板与 `--write-info-json`。

{devart_dl-4.3.4 → devart_dl-4.4.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "devart-dl"
-version = "4.3.4"
+version = "4.4.0"
 description = "专业的 DeviantArt 作品批量下载工具 - Professional DeviantArt batch downloader with anti-ban features"
 readme = "README.md"
 requires-python = ">=3.10"

{devart_dl-4.3.4 → devart_dl-4.4.0}/tests/test_manager.py

@@ -156,3 +156,18 @@ def test_dry_run_plans_without_writing(tmp_path: Path) -> None:
     assert not outcome.path.exists()
     assert downloader.calls == []  # never touched the network or disk
 
+
+
+def test_inline_literature_writes_text_without_http(tmp_path: Path) -> None:
+    downloader = FakeDownloader()
+    manager = make_manager(tmp_path, downloader=downloader)
+    item = make_item(media_url="", extension="txt", content="Chapter\n\nText")
+
+    outcome = manager.run(item)
+
+    assert outcome.status == "downloaded"
+    assert outcome.path is not None
+    assert outcome.path.read_bytes() == "Chapter\n\nText".encode("utf-8")
+    assert outcome.size == len("Chapter\n\nText".encode("utf-8"))
+    assert downloader.calls == []
+    assert not outcome.path.with_name(outcome.path.name + ".part").exists()

{devart_dl-4.3.4 → devart_dl-4.4.0}/tests/test_official_provider.py

@@ -333,3 +333,48 @@ def test_resolve_short_id_follows_redirect() -> None:
             return FakeResponse()
 
     assert _resolve_short_id("abc123", FakeHttp(), 30) == "123456"
+
+
+def test_artwork_literature_uses_init_text_and_skips_media_api(monkeypatch) -> None:
+    markup = "{\"type\":\"doc\",\"content\":[{\"type\":\"paragraph\",\"content\":[{\"type\":\"text\",\"text\":\"Once upon a time.\"}]}]}"
+    init = {
+        "deviation": {
+            "extended": {"deviationUuid": "uuid-1", "additionalMedia": []},
+            "type": "literature",
+            "title": "Story",
+            "author": {"username": "alice"},
+            "url": "https://www.deviantart.com/alice/art/story-123456",
+            "textContent": {"html": {"markup": markup}},
+        }
+    }
+    monkeypatch.setattr(provider_mod, "deviation_init", lambda identifier, username=None, **_: init)
+    provider = OfficialProvider(FakeClient(), quality="o")
+
+    items = list(provider.resolve(TargetParser.parse("123456")))
+
+    assert items[0].content == "Once upon a time."
+    assert items[0].media_url == ""
+    assert items[0].extension == "txt"
+    assert items[0].metadata == {"kind": "literature", "quality": "o"}
+
+
+def test_gallery_literature_is_inline_text() -> None:
+    client = FakeClient()
+    client.pages[("gallery_all", "alice", 0)] = {
+        "results": [{
+            "deviationid": "uuid-1",
+            "title": "Story",
+            "author": {"username": "alice"},
+            "url": "https://www.deviantart.com/alice/art/story-1",
+            "type": "literature",
+            "textContent": {"html": {"markup": "{\"type\":\"doc\",\"content\":[{\"type\":\"text\",\"text\":\"Text\"}]}"}},
+        }],
+        "has_more": False,
+    }
+    provider = OfficialProvider(client, quality="o")
+
+    items = list(provider.resolve(TargetParser.parse("https://www.deviantart.com/alice/gallery")))
+
+    assert items[0].content == "Text"
+    assert items[0].media_url == ""
+    assert items[0].extension == "txt"

{devart_dl-4.3.4 → devart_dl-4.4.0}/tests/test_provider.py

@@ -249,3 +249,21 @@ def test_composite_tag_without_official_requires_official() -> None:
     router = CompositeProvider(official=None, web=web)
     with pytest.raises(MediaUnavailableError):
         list(router.resolve(TargetParser.parse("https://www.deviantart.com/tag/landscape")))
+
+
+def test_literature_item_is_delivered_as_inline_text() -> None:
+    dev = make_deviation("123", title="Story")
+    dev.deviation_type = "literature"
+    dev.media = {}
+    dev.text_content = {
+        "html": {"markup": "{\"type\":\"doc\",\"content\":[{\"type\":\"paragraph\",\"content\":[{\"type\":\"text\",\"text\":\"Chapter one\"}]},{\"type\":\"paragraph\",\"content\":[{\"type\":\"text\",\"text\":\"Chapter two\"}]}]}"}
+    }
+    provider = make_provider(pages=[([dev], False, 1, "")])
+
+    items = list(provider.resolve(TargetParser.parse("https://www.deviantart.com/alice/gallery")))
+
+    assert len(items) == 1
+    assert items[0].content == "Chapter one\nChapter two"
+    assert items[0].media_url == ""
+    assert items[0].extension == "txt"
+    assert items[0].metadata == {"kind": "literature"}