meltify 0.2.1__tar.gz → 0.2.2__tar.gz

{meltify-0.2.1 → meltify-0.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: meltify
-Version: 0.2.1
+Version: 0.2.2
 Summary: Melt files of any format into prompt-ready text where every line cites its source
 Keywords: llm,prompt,ocr,pdf,transcription,claude-code,agent-skills
 Author: jeon-jihyeon
@@ -17,7 +17,7 @@ Requires-Dist: ocrmac>=1.0.1 ; sys_platform == 'darwin'
 Requires-Dist: trafilatura>=2
 Requires-Dist: python-hwpx>=1
 Requires-Dist: striprtf>=0.0.29
-Requires-Dist: meltify[office,media,asr-mlx,archive,crypto] ; extra == 'all'
+Requires-Dist: meltify[office,media,asr-mlx,archive,parquet,crypto] ; extra == 'all'
 Requires-Dist: libarchive-c>=5 ; extra == 'archive'
 Requires-Dist: mlx-whisper>=0.4 ; platform_machine == 'arm64' and sys_platform == 'darwin' and extra == 'asr-mlx'
 Requires-Dist: msoffcrypto-tool>=6 ; extra == 'crypto'
@@ -32,6 +32,7 @@ Requires-Dist: python-calamine>=0.4 ; extra == 'office'
 Requires-Dist: legacy-doc>=0.2 ; extra == 'office'
 Requires-Dist: olefile>=0.47 ; extra == 'office'
 Requires-Dist: metafile-render>=0.3 ; extra == 'office'
+Requires-Dist: pyarrow>=17 ; extra == 'parquet'
 Requires-Dist: playwright>=1.50 ; extra == 'render'
 Requires-Python: >=3.11.9
 Project-URL: Repository, https://github.com/jeon-jihyeon/meltify
@@ -43,6 +44,7 @@ Provides-Extra: iwork
 Provides-Extra: media
 Provides-Extra: ocr-paddle
 Provides-Extra: office
+Provides-Extra: parquet
 Provides-Extra: render
 Description-Content-Type: text/markdown
 
@@ -99,8 +101,8 @@ Each command has a skill of the same name, such as `meltify-ocr`, that tells the
 
 | Group | Formats | Needs |
 |---|---|---|
-| Documents | PDF, Word `.docx` `.docm` `.dotx` `.dotm` `.doc` `.dot`, HWP and HWPX, RTF, ODT, Pages, WordPerfect, EPUB, XPS, FictionBook, MOBI, HTML, Markdown and plain text | `office` for Word and EPUB |
-| Spreadsheets | Excel `.xlsx` `.xlsm` `.xltx` `.xltm` `.xlsb` `.xls` `.xlt`, Hancom `.cell`, ODS, Numbers, CSV and TSV | `office` for `.xlsb`, `.xls` and `.xlt`, `iwork` for Numbers |
+| Documents | PDF, Word `.docx` `.docm` `.dotx` `.dotm` `.doc` `.dot`, HWP and HWPX, RTF, ODT, Pages, WordPerfect, EPUB, XPS, FictionBook, MOBI, HTML and Safari `.webarchive`, Markdown and plain text | `office` for Word and EPUB |
+| Spreadsheets | Excel `.xlsx` `.xlsm` `.xltx` `.xltm` `.xlsb` `.xls` `.xlt`, Hancom `.cell`, ODS, Numbers, Parquet, CSV and TSV | `office` for `.xlsb`, `.xls` and `.xlt`, `iwork` for Numbers, `parquet` for Parquet |
 | Slides | PowerPoint `.pptx` `.pptm` `.potx` `.potm` `.ppsx` `.ppsm` `.ppt` `.pps` `.pot`, Hancom `.show`, ODP, Keynote | `office` for PowerPoint |
 | Drawings | ODG, SVG and `.svgz`, EMF and WMF with `.emz` and `.wmz`, comic book `.cbz` | |
 | Mail and chat | `.eml`, `.msg`, mbox, KakaoTalk exports, Slack export zips | `office` for `.msg` |
@@ -186,6 +188,7 @@ Optional parts install on request with `meltify doctor --install NAME`:
 | `office` | Word, PowerPoint, Outlook `.msg` and EPUB, legacy `.doc`, `.ppt`, `.xls` and `.xlsb`, faster `.xlsx` reading through calamine, and drawing EMF pictures without LibreOffice |
 | `archive` | 7z and rar through the system libarchive, plus a pinned 7-Zip download for encrypted ones |
 | `iwork` | Numbers tables. Pages and Keynote need nothing extra |
+| `parquet` | Parquet files, the first 200 rows and per-column min, max and null counts |
 | `crypto` | encrypted Office files, AES zips, and password-protected HWP and iWork files |
 | `render` | `read --render` for JavaScript-heavy pages, using your Chrome or a downloaded Chromium |
 | `media` | video URLs through yt-dlp |
@@ -208,6 +211,7 @@ See [examples/meltify.toml](examples/meltify.toml) for a sample. A few keys wort
 
 - `lang` is the language OCR expects (`ko` by default)
 - `asr.lang` is the spoken language for speech engines. It's `auto` by default, separate from `lang`, so Whisper detects each recording's language instead of translating it
+- `read.parquet_rows` sets how many Parquet rows melt per file
 - `render.quicklook = false` keeps Quick Look and Spotlight out of every render
 
 ## Limits
@@ -219,6 +223,7 @@ See [examples/meltify.toml](examples/meltify.toml) for a sample. A few keys wort
 - Rendered rows, EMF bitmaps and Quick Look previews are only as good as the OCR that reads them, and Quick Look exists only on macOS
 - Pages and Keynote charts are listed in `needs` instead of read, and a Pages or Keynote file with no text records falls back to its embedded preview image
 - DRM-protected HWP files, password-protected WordPerfect files, and Hancom `.cell` and `.show` files from before 2014 and Hanshow `.hpt` files, which are binary, aren't read. Save the Hancom ones again as `.xlsx` or `.pptx`
+- Parquet files show their first 200 rows, and the rest are counted in `needs`
 - Encrypted 7z and rar need 7-Zip, which `meltify doctor --install archive` downloads. Plain ones open with the system libarchive, a separate package on Linux, with 7-Zip, or with `bsdtar` on macOS
 - The plugin launcher is a POSIX shell script and `doctor --install` only downloads macOS and Linux builds, so Windows isn't supported
 

{meltify-0.2.1 → meltify-0.2.2}/README.md

@@ -51,8 +51,8 @@ Each command has a skill of the same name, such as `meltify-ocr`, that tells the
 
 | Group | Formats | Needs |
 |---|---|---|
-| Documents | PDF, Word `.docx` `.docm` `.dotx` `.dotm` `.doc` `.dot`, HWP and HWPX, RTF, ODT, Pages, WordPerfect, EPUB, XPS, FictionBook, MOBI, HTML, Markdown and plain text | `office` for Word and EPUB |
-| Spreadsheets | Excel `.xlsx` `.xlsm` `.xltx` `.xltm` `.xlsb` `.xls` `.xlt`, Hancom `.cell`, ODS, Numbers, CSV and TSV | `office` for `.xlsb`, `.xls` and `.xlt`, `iwork` for Numbers |
+| Documents | PDF, Word `.docx` `.docm` `.dotx` `.dotm` `.doc` `.dot`, HWP and HWPX, RTF, ODT, Pages, WordPerfect, EPUB, XPS, FictionBook, MOBI, HTML and Safari `.webarchive`, Markdown and plain text | `office` for Word and EPUB |
+| Spreadsheets | Excel `.xlsx` `.xlsm` `.xltx` `.xltm` `.xlsb` `.xls` `.xlt`, Hancom `.cell`, ODS, Numbers, Parquet, CSV and TSV | `office` for `.xlsb`, `.xls` and `.xlt`, `iwork` for Numbers, `parquet` for Parquet |
 | Slides | PowerPoint `.pptx` `.pptm` `.potx` `.potm` `.ppsx` `.ppsm` `.ppt` `.pps` `.pot`, Hancom `.show`, ODP, Keynote | `office` for PowerPoint |
 | Drawings | ODG, SVG and `.svgz`, EMF and WMF with `.emz` and `.wmz`, comic book `.cbz` | |
 | Mail and chat | `.eml`, `.msg`, mbox, KakaoTalk exports, Slack export zips | `office` for `.msg` |
@@ -138,6 +138,7 @@ Optional parts install on request with `meltify doctor --install NAME`:
 | `office` | Word, PowerPoint, Outlook `.msg` and EPUB, legacy `.doc`, `.ppt`, `.xls` and `.xlsb`, faster `.xlsx` reading through calamine, and drawing EMF pictures without LibreOffice |
 | `archive` | 7z and rar through the system libarchive, plus a pinned 7-Zip download for encrypted ones |
 | `iwork` | Numbers tables. Pages and Keynote need nothing extra |
+| `parquet` | Parquet files, the first 200 rows and per-column min, max and null counts |
 | `crypto` | encrypted Office files, AES zips, and password-protected HWP and iWork files |
 | `render` | `read --render` for JavaScript-heavy pages, using your Chrome or a downloaded Chromium |
 | `media` | video URLs through yt-dlp |
@@ -160,6 +161,7 @@ See [examples/meltify.toml](examples/meltify.toml) for a sample. A few keys wort
 
 - `lang` is the language OCR expects (`ko` by default)
 - `asr.lang` is the spoken language for speech engines. It's `auto` by default, separate from `lang`, so Whisper detects each recording's language instead of translating it
+- `read.parquet_rows` sets how many Parquet rows melt per file
 - `render.quicklook = false` keeps Quick Look and Spotlight out of every render
 
 ## Limits
@@ -171,6 +173,7 @@ See [examples/meltify.toml](examples/meltify.toml) for a sample. A few keys wort
 - Rendered rows, EMF bitmaps and Quick Look previews are only as good as the OCR that reads them, and Quick Look exists only on macOS
 - Pages and Keynote charts are listed in `needs` instead of read, and a Pages or Keynote file with no text records falls back to its embedded preview image
 - DRM-protected HWP files, password-protected WordPerfect files, and Hancom `.cell` and `.show` files from before 2014 and Hanshow `.hpt` files, which are binary, aren't read. Save the Hancom ones again as `.xlsx` or `.pptx`
+- Parquet files show their first 200 rows, and the rest are counted in `needs`
 - Encrypted 7z and rar need 7-Zip, which `meltify doctor --install archive` downloads. Plain ones open with the system libarchive, a separate package on Linux, with 7-Zip, or with `bsdtar` on macOS
 - The plugin launcher is a POSIX shell script and `doctor --install` only downloads macOS and Linux builds, so Windows isn't supported
 

{meltify-0.2.1 → meltify-0.2.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "meltify"
-version = "0.2.1"
+version = "0.2.2"
 description = "Melt files of any format into prompt-ready text where every line cites its source"
 readme = "README.md"
 requires-python = ">=3.11.9"
@@ -47,6 +47,7 @@ crypto = [
 ]
 archive = ["libarchive-c>=5"]
 iwork = ["numbers-parser>=4"]
+parquet = ["pyarrow>=17"]
 render = ["playwright>=1.50"]
 media = ["yt-dlp>=2026.8.19"]
 asr-mlx = ["mlx-whisper>=0.4; sys_platform == 'darwin' and platform_machine == 'arm64'"]
@@ -54,7 +55,7 @@ ocr-paddle = [
     "paddleocr>=3.2",
     "paddlepaddle>=3.2",
 ]
-all = ["meltify[office,media,asr-mlx,archive,crypto]"]
+all = ["meltify[office,media,asr-mlx,archive,parquet,crypto]"]
 
 [project.urls]
 Repository = "https://github.com/jeon-jihyeon/meltify"

{meltify-0.2.1 → meltify-0.2.2}/pyproject.toml.orig

@@ -1,6 +1,6 @@
 [project]
 name = "meltify"
-version = "0.2.1"
+version = "0.2.2"
 description = "Melt files of any format into prompt-ready text where every line cites its source"
 readme = "README.md"
 requires-python = ">=3.11.9"
@@ -35,11 +35,12 @@ crypto = ["msoffcrypto-tool>=6", "pyzipper>=0.4", "cryptography>=42"]
 archive = ["libarchive-c>=5"]
 # Only Numbers needs it, since Pages and Keynote are read natively
 iwork = ["numbers-parser>=4"]
+parquet = ["pyarrow>=17"]
 render = ["playwright>=1.50"]
 media = ["yt-dlp>=2026.8.19"]
 asr-mlx = ["mlx-whisper>=0.4; sys_platform == 'darwin' and platform_machine == 'arm64'"]
 ocr-paddle = ["paddleocr>=3.2", "paddlepaddle>=3.2"]
-all = ["meltify[office,media,asr-mlx,archive,crypto]"]
+all = ["meltify[office,media,asr-mlx,archive,parquet,crypto]"]
 
 [project.urls]
 Repository = "https://github.com/jeon-jihyeon/meltify"

{meltify-0.2.1 → meltify-0.2.2}/src/meltify/commands/doctor.py

@@ -47,6 +47,7 @@ MODULES = [
     ("pyzipper", "read aes zip", "meltify doctor --install crypto", False),
     ("cryptography", "read encrypted hwp iwork", "meltify doctor --install crypto", False),
     ("numbers_parser", "read numbers", "meltify doctor --install iwork", False),
+    ("pyarrow", "read parquet", "meltify doctor --install parquet", False),
     ("playwright", "read --render", "meltify doctor --install render", False),
     ("yt_dlp", "media urls", "meltify doctor --install media", False),
     ("paddleocr", "ocr engine paddle", "meltify doctor --install ocr-paddle", False),
@@ -78,6 +79,7 @@ EXTRAS = [
     "office",
     "archive",
     "iwork",
+    "parquet",
     "crypto",
     "render",
     "media",

{meltify-0.2.1 → meltify-0.2.2}/src/meltify/commands/read.py

@@ -628,6 +628,7 @@ def _run(args: argparse.Namespace, settings: dict[str, Any], password: str | Non
             password=given_password(args, env.warnings.append, password),
             fallback=bool(conf.get("fallback", True)),
             shallow=args.shallow,
+            parquet_rows=int(conf.get("parquet_rows", 200)),
             quicklook=looks,
             temps=temps,
         )

{meltify-0.2.1 → meltify-0.2.2}/src/meltify/converters/__init__.py

@@ -346,6 +346,7 @@ register("media", f"{_HERE}:audio", *AUDIO)
 register("media", f"{_HERE}:video", *VIDEO)
 register("office", f"{_HERE}.office:convert", *OFFICE)
 register("web", f"{_HERE}.web:convert", ".html", ".htm", ".xhtml")
+register("webarchive", f"{_HERE}.webarchive:convert", ".webarchive")
 register("epub", f"{_HERE}.epub:convert", ".epub")
 register(
     "archive",
@@ -372,6 +373,7 @@ register("wordperfect", f"{_HERE}.wordperfect:convert", ".wpd", ".wp", ".wp5", "
 register("notebook", f"{_HERE}.notebook:convert", ".ipynb")
 register("mbox", f"{_HERE}.mbox:convert", ".mbox")
 register("sqlite", f"{_HERE}.sqlite:convert", ".db", ".sqlite", ".sqlite3")
+register("parquet", f"{_HERE}.parquet:convert", ".parquet")
 # Registered after TEXT, so these take over .srt and .vtt from the plain text entry
 register("subtitle", f"{_HERE}.subtitle:convert", ".srt", ".vtt", ".ass")
 register("text", f"{_HERE}.text:convert_card", ".vcf", ".ics")
@@ -383,6 +385,7 @@ register_sniff("slack", f"{_HERE}.slack:convert", f"{_HERE}.slack:sniff", {".zip
 register_sniff("pdf", f"{_HERE}.pdf:convert", _magic(b"%PDF-"))
 register_sniff("sqlite", f"{_HERE}.sqlite:convert", _magic(b"SQLite format 3\x00"))
 register_sniff("rtf", f"{_HERE}.rtf:convert", _magic(b"{\\rtf"))
+register_sniff("parquet", f"{_HERE}.parquet:convert", _magic(b"PAR1"))
 register_sniff("wordperfect", f"{_HERE}.wordperfect:convert", _wordperfect)
 # Document zips before the archive sniff, so an unnamed one isn't unpacked as a plain zip
 for kind, target, families in (

meltify-0.2.2/src/meltify/converters/parquet.py

@@ -0,0 +1,103 @@
+"""Parquet tables: the schema, a per-column profile, then the first rows cited by row number
+
+A cite like `a.parquet:7` points at the seventh data row, counted from 1
+"""
+
+from __future__ import annotations
+
+from dataclasses import replace
+from pathlib import Path
+
+from meltify.converters import Block, Converted, run
+from meltify.converters.limits import MAX_MEMBER_BYTES, human_bytes
+from meltify.converters.tables import sample_cell
+from meltify.evidence import Src
+from meltify.needs import not_read
+from meltify.safe import MissingTool
+
+
+def _groups(meta, limit: int) -> tuple[list[int], str | None]:
+    """Leading row groups that hold the first `limit` rows, and why they stop short if they do
+
+    Arrow unpacks a whole page at a time, and a few KB of zstd can declare one value of
+    hundreds of MB, so the sizes the footer declares are checked before anything is read
+    """
+    groups: list[int] = []
+    rows = size = 0
+    for i in range(meta.num_row_groups):
+        if rows >= limit:
+            break
+        group = meta.row_group(i)
+        columns = sum(group.column(j).total_uncompressed_size for j in range(group.num_columns))
+        size += max(group.total_byte_size, columns)
+        if size > MAX_MEMBER_BYTES:
+            past = human_bytes(MAX_MEMBER_BYTES)
+            return groups, f"row group {i + 1} not read, it unpacks past {past}"
+        groups.append(i)
+        rows += group.num_rows
+    return groups, None
+
+
+def _head(f, limit: int, groups: list[int]) -> list[tuple]:
+    rows: list[tuple] = []
+    # Batches end at row group boundaries, so one batch may hold fewer rows than asked
+    for batch in f.iter_batches(batch_size=limit, row_groups=groups):
+        # Tuples, not dicts, so two columns with the same name both survive
+        rows += list(zip(*(c.to_pylist() for c in batch.columns), strict=True))
+        if len(rows) >= limit:
+            break
+    return rows[:limit]
+
+
+def _profile(meta) -> str:
+    """Min, max and nulls per column from the footer statistics, without reading any data
+
+    A row group that wrote no statistics leaves that column's cells blank, since the rest
+    can't speak for it
+    """
+    lines = ["| column | min | max | nulls |", "|---|---|---|---|"]
+    for j in range(meta.num_columns):
+        stats = [meta.row_group(i).column(j).statistics for i in range(meta.num_row_groups)]
+        low = high = nulls = ""
+        if stats and all(s is not None and s.has_min_max for s in stats):
+            low, high = min(s.min for s in stats), max(s.max for s in stats)
+        if stats and all(s is not None and s.has_null_count for s in stats):
+            nulls = f"{sum(s.null_count for s in stats):,}"
+        name = meta.schema.column(j).path
+        cells = " | ".join(map(sample_cell, (name, low, high)))
+        lines.append(f"| {cells} | {nulls} |")
+    return "\n".join(lines)
+
+
+def convert(path: Path, src: Src) -> Converted:
+    try:
+        import pyarrow.parquet as pq
+    except ImportError as e:
+        raise MissingTool("pyarrow", "meltify doctor --install parquet") from e
+
+    out = Converted("parquet")
+    with pq.ParquetFile(path) as f:
+        total = f.metadata.num_rows
+        fields = list(f.schema_arrow)
+        schema = "\n".join(f"{field.name}: {field.type}" for field in fields)
+        out.blocks.append(Block(src, f"{total:,} rows\n{schema}"))
+        if f.metadata.num_row_groups:
+            out.blocks.append(Block(src, _profile(f.metadata)))
+        limit = run.current().parquet_rows or total
+        groups, cut = _groups(f.metadata, limit)
+        rows = _head(f, limit, groups) if groups else []
+
+    if rows:
+        lines = [
+            "| row | " + " | ".join(sample_cell(field.name) for field in fields) + " |",
+            "|---" * (len(fields) + 1) + "|",
+        ]
+        for n, row in enumerate(rows, start=1):
+            lines.append(f"| {n} | " + " | ".join(sample_cell(v) for v in row) + " |")
+        out.blocks.append(Block(replace(src, line=1), "\n".join(lines)))
+    if (rest := total - len(rows)) > 0:
+        shown = f"first {len(rows):,} shown, read.parquet_rows sets how many"
+        out.needs.append(not_read(rest, "more row", shown))
+    if cut:
+        out.needs.append(cut)
+    return out

{meltify-0.2.1 → meltify-0.2.2}/src/meltify/converters/run.py

@@ -71,6 +71,9 @@ class RunContext:
     fallback: bool = True  # read.fallback
     # read --shallow, which starts no renderer or subprocess that only feeds OCR
     shallow: bool = False
+    # read.parquet_rows, 0 for every row. A data file can hold billions of rows, so only the
+    # head is melted and the rest counted
+    parquet_rows: int = 200
     quicklook: bool = True  # render.quicklook
     # State the run's workers share, which two runs with the same settings still don't
     budget: Budget = field(default_factory=Budget, compare=False)

meltify-0.2.2/src/meltify/converters/webarchive.py

@@ -0,0 +1,160 @@
+"""Safari web archives: the saved page through the web extractor, its images through OCR
+
+A webarchive is a property list holding the page bytes, every subresource the page
+loaded, and one nested archive per frame
+"""
+
+from __future__ import annotations
+
+import hashlib
+import mimetypes
+import plistlib
+import posixpath
+import tempfile
+from pathlib import Path
+from urllib.parse import urldefrag, urljoin, urlparse
+
+from meltify.converters import Child, Converted
+from meltify.converters.embeds import Embeds
+from meltify.evidence import Src
+from meltify.needs import not_read
+
+HTML = {"text/html", "application/xhtml+xml"}
+# Distinct frames melted per archive, the rest listed in needs
+MAX_FRAMES = 100
+# Frame bytes per byte of the archive. Frames are slices of the archive, so unless they
+# share bytes they can't add up to more than it
+FRAME_RATIO = 2
+
+
+def _name(resource: dict, fallback: str) -> str:
+    """A file name that keeps a suffix, so the registry can melt the resource as a child"""
+    name = posixpath.basename(urlparse(resource.get("WebResourceURL") or "").path) or fallback
+    if not Path(name).suffix:
+        name += mimetypes.guess_extension(resource.get("WebResourceMIMEType") or "") or ""
+    return name
+
+
+def _image(resource: object) -> bytes | None:
+    # Stylesheets, scripts and fonts carry no text worth citing
+    if not isinstance(resource, dict):
+        return None
+    data = resource.get("WebResourceData")
+    image = str(resource.get("WebResourceMIMEType")).startswith("image/")
+    return data if image and isinstance(data, bytes) else None
+
+
+def _utf8(data: bytes, encoding: str | None) -> bytes:
+    # Safari records the encoding the page was served with, which may disagree with its meta
+    # tag, and a UTF-8 byte order mark outranks the meta tag in the HTML parser
+    if not encoding:
+        return data
+    try:
+        text = data.decode(encoding, errors="replace")
+    except LookupError:
+        return data
+    return b"\xef\xbb\xbf" + text.removeprefix("\ufeff").encode("utf-8")
+
+
+def _page(resource: dict, src: Src, images: dict[str, tuple[str, bytes]], used: set[str]):
+    """The page, with the pictures it references loaded from the archive"""
+    from meltify.converters.web import convert_page
+
+    base = resource.get("WebResourceURL") or ""
+
+    def load(ref: str, embeds: Embeds) -> tuple[str, bytes] | str:
+        url = urldefrag(urljoin(base, ref)).url
+        if url not in images:
+            return "remote image"
+        used.add(url)
+        return images[url]
+
+    data = _utf8(resource["WebResourceData"], resource.get("WebResourceTextEncodingName"))
+    with tempfile.TemporaryDirectory(prefix="meltify-webarchive-") as tmp:
+        page = Path(tmp) / "page.html"
+        page.write_bytes(data)
+        # A saved page keeps navigation and footers, as a saved .html does
+        out = convert_page(page, src, whole=True, load=load)
+    out.kind = "webarchive"
+    return out
+
+
+def convert(path: Path, src: Src) -> Converted:
+    raw = path.read_bytes()
+    try:
+        archive = plistlib.loads(raw)
+    except ValueError as e:
+        return Converted("webarchive", needs=[f"unreadable webarchive: {e}"[:200]])
+    main = archive.get("WebMainResource") if isinstance(archive, dict) else None
+    if not isinstance(main, dict) or not isinstance(main.get("WebResourceData"), bytes):
+        return Converted("webarchive", needs=["webarchive has no main resource"])
+
+    embeds = Embeds()
+    images: dict[str, tuple[str, bytes]] = {}
+    for i, res in enumerate(archive.get("WebSubresources") or [], start=1):
+        data = _image(res)
+        if data is None:
+            continue
+        # Every picture is already in memory with the plist, so only the size limit applies
+        # here, and the page's references share each one under the picture total
+        if why := embeds.refusal(len(data)):
+            embeds.skipped[why] += 1
+            continue
+        images[urldefrag(res.get("WebResourceURL") or f"#{i}").url] = (
+            _name(res, f"image{i}"),
+            data,
+        )
+
+    used: set[str] = set()
+    if (main.get("WebResourceMIMEType") or "text/html") in HTML:
+        out = _page(main, src, images, used)
+    else:
+        # Safari also saves a bare image or PDF this way, so it melts like the file itself
+        out = Converted("webarchive")
+        out.children.append(Child(_name(main, "main"), src, data=main["WebResourceData"]))
+    # Pictures only a stylesheet or script loads, cited by their name in the archive
+    for url, (name, data) in images.items():
+        if url not in used:
+            # An SVG melts as a child under its own name, so it hangs off the archive itself
+            at = src if Path(name).suffix.lower() == ".svg" else src.inside(name)
+            embeds.picture(at, data, name)
+    embeds.into(out)
+
+    out.needs += _frames(archive.get("WebSubframeArchives") or [], src, len(raw), out)
+    return out
+
+
+def _frames(frames: list, src: Src, size: int, out: Converted) -> list[str]:
+    """Each frame is a whole webarchive of its own, melted again as a child
+
+    A binary plist can reference one frame object many times, and dumping each reference
+    again would multiply the archive at every nesting level. Identical frames go out once,
+    and the frames together never get more bytes than the archive itself holds
+    """
+    seen: set[int] = set()
+    dumped: set[bytes] = set()
+    left = FRAME_RATIO * size
+    many = large = 0
+    for i, frame in enumerate(frames, start=1):
+        if id(frame) in seen:
+            continue
+        seen.add(id(frame))
+        if len(dumped) >= MAX_FRAMES:
+            many += 1
+            continue
+        data = plistlib.dumps(frame, fmt=plistlib.FMT_BINARY)
+        digest = hashlib.sha256(data).digest()
+        if digest in dumped:
+            continue
+        if len(data) > left:
+            large += 1
+            continue
+        left -= len(data)
+        dumped.add(digest)
+        out.children.append(Child(f"frame{i}.webarchive", src, data=data))
+    needs = []
+    if many:
+        needs.append(not_read(many, "frame", f"over {MAX_FRAMES} frames"))
+    if large:
+        needs.append(not_read(large, "frame", "more frame data than the archive holds"))
+    return needs

{meltify-0.2.1 → meltify-0.2.2}/src/meltify/data/defaults.toml

@@ -74,6 +74,8 @@ max_bytes = 20000000
 # Binary files no converter reads get one more try: LibreOffice for the formats it imports,
 # Pillow, ffprobe, then on macOS Quick Look and Spotlight. Off keeps them as an unsupported need
 fallback = true
+# Parquet rows melted per file, 0 for all. The rest stays counted as a need
+parquet_rows = 200
 
 [render]
 # macOS only. Quick Look previews draw what LibreOffice can't or isn't installed for, like