agent-folder-workspace 0.1.0__py3-none-any.whl

agent_folder_workspace/adapters/legacy_office.py

@@ -0,0 +1,552 @@
+"""Virtual hierarchy for DOC/XLS/PPT backed by the in-package CFB reader."""
+
+from __future__ import annotations
+
+import math
+from pathlib import Path
+from typing import Any, Literal
+
+from agent_folder_workspace.adapters.base import BaseAdapter, ReadResult, VirtualNodeSpec
+from agent_folder_workspace.cfb import CfbDirectoryEntry, CfbFile
+from agent_folder_workspace.config import WorkspaceConfig
+from agent_folder_workspace.errors import InputValidationError, ResourceLimitError
+from agent_folder_workspace.legacy import (
+    BiffParseResult,
+    BinaryRecord,
+    DocParseResult,
+    PptParseResult,
+    parse_biff,
+    parse_doc,
+    parse_ppt,
+)
+from agent_folder_workspace.models import NodeKind, NodeV1
+
+LegacyFamily = Literal["doc", "xls", "ppt"]
+
+
+class LegacyOfficeAdapter(BaseAdapter):
+    """Expose semantics and a lossless container/record tree lazily."""
+
+    _ROOTS = ("semantic", "storages", "streams", "records", "directory_entries")
+
+    def __init__(self, family: LegacyFamily) -> None:
+        self.family = family
+
+    def children(
+        self, node: NodeV1, source: Path, config: WorkspaceConfig
+    ) -> list[VirtualNodeSpec]:
+        logical = node.logical_path
+        if logical == "/":
+            return [
+                VirtualNodeSpec(name, NodeKind.CONTAINER, f"/legacy/{name}") for name in self._ROOTS
+            ]
+        with CfbFile.open(source, max_directory_entries=config.max_index_nodes_per_file) as cfb:
+            if logical == "/legacy/streams":
+                return [self._stream_spec(entry) for entry in cfb.streams]
+            if logical == "/legacy/storages":
+                return [
+                    VirtualNodeSpec(
+                        entry.name,
+                        NodeKind.CONTAINER,
+                        f"/legacy/storages/{entry.index}",
+                        metadata=self._entry_metadata(entry),
+                    )
+                    for entry in cfb.storages
+                ]
+            if logical == "/legacy/directory_entries":
+                return [
+                    VirtualNodeSpec(
+                        entry.name or f"Empty {entry.index}",
+                        NodeKind.STREAM if entry.is_stream else NodeKind.CONTAINER,
+                        f"/legacy/directory_entries/{entry.index}",
+                        size=entry.size,
+                        metadata=self._entry_metadata(entry),
+                        source_part=f"cfb:{entry.path}" if entry.is_stream else None,
+                        source_offset=0 if entry.is_stream else None,
+                        source_length=entry.size if entry.is_stream else None,
+                    )
+                    for entry in cfb.entries
+                    if entry.object_type
+                ]
+            if logical == "/legacy/records":
+                return [
+                    VirtualNodeSpec(
+                        entry.name,
+                        NodeKind.CONTAINER,
+                        f"/legacy/records/{entry.index}",
+                        metadata={"stream_path": entry.path, "stream_index": entry.index},
+                        source_part=f"cfb:{entry.path}",
+                        source_offset=0,
+                        source_length=entry.size,
+                    )
+                    for entry in self._record_streams(cfb)
+                ]
+            if logical.startswith("/legacy/records/"):
+                return self._record_children(node, cfb, config)
+            if logical == "/legacy/semantic":
+                return self._semantic_children(cfb, config)
+            if logical.startswith("/legacy/semantic/sheet/") and node.kind == NodeKind.SHEET:
+                return self._sheet_blocks(node, cfb, config)
+        return []
+
+    def read(
+        self,
+        node: NodeV1,
+        source: Path,
+        offset: int,
+        limit: int,
+        config: WorkspaceConfig,
+    ) -> ReadResult:
+        with CfbFile.open(source, max_directory_entries=config.max_index_nodes_per_file) as cfb:
+            logical = node.logical_path
+            if "/legacy/semantic/sheet/" in logical and "/rows/" in logical:
+                parsed = self._parse_xls(cfb, config)
+                sheet_cells = self._sheet_cells(parsed, int(node.metadata.get("sheet_index", 0)))
+                start = int(node.metadata.get("cell_start", 0)) + offset
+                end = min(start + limit, int(node.metadata.get("cell_end", len(sheet_cells))))
+                items = sheet_cells[start:end]
+                return ReadResult(
+                    items=items,
+                    total=int(node.metadata.get("cell_end", 0))
+                    - int(node.metadata.get("cell_start", 0)),
+                )
+            if logical.startswith("/legacy/semantic/paragraph/"):
+                item = {
+                    "text": node.metadata.get("text", ""),
+                    "cp_start": node.metadata.get("cp_start"),
+                    "cp_end": node.metadata.get("cp_end"),
+                    "source_ranges": node.metadata.get("source_ranges", []),
+                }
+                return ReadResult(
+                    items=[item] if offset == 0 else [], total=1, text=str(item["text"])
+                )
+            if logical.startswith("/legacy/semantic/slide/"):
+                item = {
+                    "index": node.metadata.get("slide_index"),
+                    "text": node.metadata.get("text", ""),
+                    "source_ref": node.metadata.get("source_ref"),
+                }
+                return ReadResult(
+                    items=[item] if offset == 0 else [], total=1, text=str(item["text"])
+                )
+            if logical.startswith("/legacy/records/") and node.kind in {
+                NodeKind.RECORD,
+                NodeKind.UNKNOWN_RECORD,
+            }:
+                fields = dict(node.metadata.get("fields", {}))
+                item = {
+                    "record_type": node.metadata.get("record_type"),
+                    "name": node.name,
+                    "offset": node.metadata.get("record_offset"),
+                    "length": node.metadata.get("record_length"),
+                    "fields": fields,
+                }
+                return ReadResult(items=[item] if offset == 0 else [], total=1)
+            if logical.startswith("/legacy/streams/"):
+                stream_path = str(node.metadata.get("stream_path", ""))
+                entry = cfb.find(stream_path)
+                item = self._entry_metadata(entry)
+                return ReadResult(items=[item] if offset == 0 else [], total=1)
+        return ReadResult(items=[], total=0)
+
+    def read_binary_part(
+        self,
+        node: NodeV1,
+        source: Path,
+        offset: int,
+        length: int,
+        config: WorkspaceConfig,
+    ) -> tuple[bytes, int]:
+        part = node.source_ref.part if node.source_ref is not None else None
+        if not part:
+            return super().read_binary_part(node, source, offset, length, config)
+        with CfbFile.open(source, max_directory_entries=config.max_index_nodes_per_file) as cfb:
+            if part.startswith("cfb:"):
+                return cfb.read_stream_range(part[4:], offset, length)
+            if part.startswith("record:"):
+                stream_path, record_offset, record_length = self._split_record_part(part)
+                if offset >= record_length:
+                    return b"", record_length
+                requested = min(length, record_length - offset)
+                payload, _stream_length = cfb.read_stream_range(
+                    stream_path,
+                    record_offset + offset,
+                    requested,
+                )
+                return payload, record_length
+        return super().read_binary_part(node, source, offset, length, config)
+
+    @staticmethod
+    def _entry_metadata(entry: CfbDirectoryEntry) -> dict[str, Any]:
+        return {
+            "directory_index": entry.index,
+            "path": entry.path,
+            "object_type": entry.object_type,
+            "start_sector": entry.start_sector,
+            "size": entry.size,
+        }
+
+    def _stream_spec(self, entry: CfbDirectoryEntry) -> VirtualNodeSpec:
+        return VirtualNodeSpec(
+            entry.name,
+            NodeKind.STREAM,
+            f"/legacy/streams/{entry.index}",
+            size=entry.size,
+            metadata={**self._entry_metadata(entry), "stream_path": entry.path},
+            source_part=f"cfb:{entry.path}",
+            source_offset=0,
+            source_length=entry.size,
+        )
+
+    def _record_streams(self, cfb: CfbFile) -> list[CfbDirectoryEntry]:
+        names = {
+            "xls": {"workbook", "book"},
+            "doc": {"worddocument", "0table", "1table", "data"},
+            "ppt": {"powerpoint document", "current user"},
+        }[self.family]
+        return [entry for entry in cfb.streams if entry.name.casefold() in names]
+
+    def _record_children(
+        self, node: NodeV1, cfb: CfbFile, config: WorkspaceConfig
+    ) -> list[VirtualNodeSpec]:
+        stream_path = str(
+            node.metadata.get("record_stream") or node.metadata.get("stream_path") or ""
+        )
+        if not stream_path:
+            return []
+        data = self._bounded_stream(cfb, stream_path, config)
+        records: list[BinaryRecord]
+        if self.family == "xls":
+            records = parse_biff(data, max_records=config.max_index_nodes_per_file).records
+        elif self.family == "ppt":
+            parsed = parse_ppt(data, max_records=config.max_index_nodes_per_file)
+            parent_offset = node.metadata.get("record_offset")
+            if parent_offset is None:
+                records = parsed.records
+            else:
+                parent = self._find_ppt_record(parsed, int(parent_offset))
+                records = parent.children if parent is not None else []
+        else:
+            if node.metadata.get("record_offset") is not None:
+                return []
+            return self._doc_record_specs(stream_path, data, node.logical_path)
+        return [self._record_spec(stream_path, record, node.logical_path) for record in records]
+
+    def _record_spec(
+        self, stream_path: str, record: BinaryRecord, parent_logical: str
+    ) -> VirtualNodeSpec:
+        return VirtualNodeSpec(
+            f"{record.name} @ {record.offset}",
+            NodeKind.UNKNOWN_RECORD if record.kind == "unknown_record" else NodeKind.RECORD,
+            f"{parent_logical}/{record.offset}",
+            child_count=len(record.children) or None,
+            size=len(record.raw),
+            metadata={
+                "record_stream": stream_path,
+                "record_offset": record.offset,
+                "record_length": len(record.raw),
+                "record_type": record.record_type,
+                "fields": record.fields,
+            },
+            source_part=f"record:{stream_path}:{record.offset}:{len(record.raw)}",
+            source_offset=record.offset,
+            source_length=len(record.raw),
+        )
+
+    @staticmethod
+    def _doc_record_specs(
+        stream_path: str, data: bytes, parent_logical: str
+    ) -> list[VirtualNodeSpec]:
+        if stream_path.casefold() != "worddocument" or len(data) < 32:
+            return [
+                VirtualNodeSpec(
+                    "unknown stream bytes",
+                    NodeKind.UNKNOWN_RECORD,
+                    f"{parent_logical}/0",
+                    size=len(data),
+                    metadata={
+                        "record_stream": stream_path,
+                        "record_offset": 0,
+                        "record_length": len(data),
+                        "record_type": "unknown_stream_bytes",
+                        "fields": {},
+                    },
+                    source_part=f"record:{stream_path}:0:{len(data)}",
+                    source_offset=0,
+                    source_length=len(data),
+                )
+            ]
+        fc_min = int.from_bytes(data[24:28], "little")
+        fib_length = min(max(fc_min, 32), len(data))
+        specs = [
+            VirtualNodeSpec(
+                "FIB and pre-text structures",
+                NodeKind.RECORD,
+                f"{parent_logical}/0",
+                size=fib_length,
+                metadata={
+                    "record_stream": stream_path,
+                    "record_offset": 0,
+                    "record_length": fib_length,
+                    "record_type": "FIB",
+                    "fields": {"fcMin": fc_min},
+                },
+                source_part=f"record:{stream_path}:0:{fib_length}",
+                source_offset=0,
+                source_length=fib_length,
+            )
+        ]
+        block_size = 64 * 1024
+        for start in range(fib_length, len(data), block_size):
+            end = min(start + block_size, len(data))
+            specs.append(
+                VirtualNodeSpec(
+                    f"unknown_record @ {start}",
+                    NodeKind.UNKNOWN_RECORD,
+                    f"{parent_logical}/{start}",
+                    size=end - start,
+                    metadata={
+                        "record_stream": stream_path,
+                        "record_offset": start,
+                        "record_length": end - start,
+                        "record_type": "unknown_record",
+                        "fields": {},
+                    },
+                    source_part=f"record:{stream_path}:{start}:{end - start}",
+                    source_offset=start,
+                    source_length=end - start,
+                )
+            )
+        return specs
+
+    @staticmethod
+    def _find_ppt_record(parsed: PptParseResult, offset: int) -> BinaryRecord | None:
+        pending = list(parsed.records)
+        while pending:
+            record = pending.pop()
+            if record.offset == offset:
+                return record
+            pending.extend(record.children)
+        return None
+
+    @staticmethod
+    def _split_record_part(part: str) -> tuple[str, int, int]:
+        body = part[len("record:") :]
+        stream_path, offset_text, length_text = body.rsplit(":", 2)
+        return stream_path, int(offset_text), int(length_text)
+
+    def _semantic_children(self, cfb: CfbFile, config: WorkspaceConfig) -> list[VirtualNodeSpec]:
+        if self.family == "xls":
+            parsed_xls = self._parse_xls(cfb, config)
+            if any(record.record_type == 0x002F for record in parsed_xls.records):
+                raise InputValidationError("encrypted legacy XLS content requires a credential")
+            sheets = parsed_xls.sheets or [
+                {
+                    "name": "Workbook",
+                    "stream_offset": 0,
+                    "stream_end": sum(len(record.raw) for record in parsed_xls.records),
+                    "stream_length": sum(len(record.raw) for record in parsed_xls.records),
+                    "substream_valid": True,
+                    "substream_complete": False,
+                    "cells": parsed_xls.cells,
+                }
+            ]
+            return [
+                VirtualNodeSpec(
+                    str(sheet.get("name") or f"Sheet {index + 1}"),
+                    NodeKind.SHEET,
+                    f"/legacy/semantic/sheet/{index}",
+                    child_count=max(
+                        1,
+                        math.ceil(
+                            len(self._sheet_cells(parsed_xls, index))
+                            / max(config.max_page_items, 1)
+                        ),
+                    ),
+                    metadata={
+                        "sheet_index": index,
+                        **{key: value for key, value in sheet.items() if key != "cells"},
+                    },
+                    source_part=self._sheet_source_part(parsed_xls, sheet),
+                    source_offset=int(sheet.get("stream_offset", 0)),
+                    source_length=int(sheet.get("stream_length", 0)),
+                )
+                for index, sheet in enumerate(sheets)
+            ]
+        if self.family == "doc":
+            parsed_doc = self._parse_doc(cfb, config)
+            if parsed_doc.fib_fields.get("encrypted") is True:
+                raise InputValidationError("encrypted legacy DOC content requires a credential")
+            return self._paragraph_specs(parsed_doc)
+        parsed_ppt = self._parse_ppt(cfb, config)
+        return [
+            VirtualNodeSpec(
+                f"Slide {slide.index + 1}",
+                NodeKind.SLIDE,
+                f"/legacy/semantic/slide/{slide.index}",
+                metadata={
+                    "slide_index": slide.index,
+                    "text": slide.text,
+                    "source_ref": {
+                        "stream": "PowerPoint Document",
+                        "offset": slide.record_offset,
+                        "length": slide.record_length,
+                    },
+                },
+                source_part=(
+                    f"record:PowerPoint Document:{slide.record_offset}:{slide.record_length}"
+                ),
+                source_offset=slide.record_offset,
+                source_length=slide.record_length,
+            )
+            for slide in parsed_ppt.slides
+        ]
+
+    def _sheet_blocks(
+        self, node: NodeV1, cfb: CfbFile, config: WorkspaceConfig
+    ) -> list[VirtualNodeSpec]:
+        parsed = self._parse_xls(cfb, config)
+        sheet_index = int(node.metadata.get("sheet_index", 0))
+        cells = self._sheet_cells(parsed, sheet_index)
+        block_size = max(1, config.max_page_items)
+        if not cells:
+            ranges = [(0, 0)]
+        else:
+            ranges = [
+                (start, min(start + block_size, len(cells)))
+                for start in range(0, len(cells), block_size)
+            ]
+        return [
+            VirtualNodeSpec(
+                f"Cells {start}:{end}",
+                NodeKind.ROW_BLOCK,
+                f"{node.logical_path}/rows/{start}:{end}",
+                child_count=0,
+                metadata={
+                    "sheet_index": sheet_index,
+                    "cell_start": start,
+                    "cell_end": end,
+                },
+                source_part=node.source_ref.part if node.source_ref is not None else None,
+                source_offset=node.source_ref.offset if node.source_ref is not None else None,
+                source_length=node.source_ref.length if node.source_ref is not None else None,
+            )
+            for start, end in ranges
+        ]
+
+    @staticmethod
+    def _sheet_cells(parsed: BiffParseResult, index: int) -> list[dict[str, Any]]:
+        if 0 <= index < len(parsed.sheets):
+            cells = parsed.sheets[index].get("cells")
+            if isinstance(cells, list):
+                return cells
+        return parsed.cells if not parsed.sheets and index == 0 else []
+
+    @staticmethod
+    def _sheet_source_part(parsed: BiffParseResult, sheet: dict[str, Any]) -> str | None:
+        if not sheet.get("substream_valid"):
+            return None
+        start = int(sheet.get("stream_offset", 0))
+        length = int(sheet.get("stream_length", 0))
+        return f"record:{parsed.stream_name}:{start}:{length}"
+
+    @staticmethod
+    def _parse_xls(cfb: CfbFile, config: WorkspaceConfig) -> BiffParseResult:
+        for name in ("Workbook", "Book"):
+            try:
+                return parse_biff(
+                    LegacyOfficeAdapter._bounded_stream(cfb, name, config),
+                    stream_name=name,
+                    max_records=config.max_index_nodes_per_file,
+                )
+            except KeyError:
+                continue
+        return parse_biff(
+            b"",
+            stream_name="Workbook",
+            max_records=config.max_index_nodes_per_file,
+        )
+
+    @staticmethod
+    def _parse_doc(cfb: CfbFile, config: WorkspaceConfig) -> DocParseResult:
+        word = LegacyOfficeAdapter._bounded_stream(cfb, "WordDocument", config)
+        table: bytes | None = None
+        if len(word) >= 12:
+            flags = int.from_bytes(word[10:12], "little")
+            table_name = "1Table" if flags & 0x0200 else "0Table"
+            try:
+                table = LegacyOfficeAdapter._bounded_stream(cfb, table_name, config)
+            except KeyError:
+                table = None
+        return parse_doc(
+            word,
+            table,
+            max_pieces=config.max_index_nodes_per_file,
+            max_paragraphs=config.max_index_nodes_per_file,
+        )
+
+    @staticmethod
+    def _parse_ppt(cfb: CfbFile, config: WorkspaceConfig) -> PptParseResult:
+        return parse_ppt(
+            LegacyOfficeAdapter._bounded_stream(cfb, "PowerPoint Document", config),
+            max_records=config.max_index_nodes_per_file,
+        )
+
+    @staticmethod
+    def _bounded_stream(cfb: CfbFile, stream_path: str, config: WorkspaceConfig) -> bytes:
+        entry = cfb.find(stream_path)
+        if entry.size > config.max_legacy_stream_bytes:
+            raise ResourceLimitError(
+                f"legacy stream exceeds {config.max_legacy_stream_bytes} bytes"
+            )
+        return cfb.read_stream(entry)
+
+    @staticmethod
+    def _paragraph_specs(parsed: DocParseResult) -> list[VirtualNodeSpec]:
+        specs: list[VirtualNodeSpec] = []
+        for index, paragraph in enumerate(parsed.paragraphs):
+            ranges: list[dict[str, Any]] = []
+            for piece in parsed.pieces:
+                overlap_start = max(paragraph.cp_start, piece.cp_start)
+                overlap_end = min(paragraph.cp_end, piece.cp_end)
+                if overlap_start >= overlap_end:
+                    continue
+                width = 2 if piece.encoding == "utf-16le" else 1
+                ranges.append(
+                    {
+                        "stream": "WordDocument",
+                        "offset": piece.stream_offset + (overlap_start - piece.cp_start) * width,
+                        "length": (overlap_end - overlap_start) * width,
+                    }
+                )
+            source_start = min((int(item["offset"]) for item in ranges), default=None)
+            source_end = max(
+                (int(item["offset"]) + int(item["length"]) for item in ranges),
+                default=None,
+            )
+            source_length = (
+                source_end - source_start
+                if source_start is not None and source_end is not None
+                else None
+            )
+            specs.append(
+                VirtualNodeSpec(
+                    f"Paragraph {index + 1}",
+                    NodeKind.PARAGRAPH,
+                    f"/legacy/semantic/paragraph/{index}",
+                    metadata={
+                        "text": paragraph.text,
+                        "cp_start": paragraph.cp_start,
+                        "cp_end": paragraph.cp_end,
+                        "source_ranges": ranges,
+                    },
+                    source_part=(
+                        f"record:WordDocument:{source_start}:{source_length}"
+                        if source_start is not None and source_length is not None
+                        else "cfb:WordDocument"
+                    ),
+                    source_offset=source_start,
+                    source_length=source_length,
+                )
+            )
+        return specs