smart-slice 0.3.0__py3-none-any.whl

smart_slice/_config.py

@@ -0,0 +1,60 @@
+# coding=utf-8
+"""Environment-backed settings helpers.
+
+There is no framework settings object: every tunable is read from the process
+environment.  The defensive parser caps live on
+:class:`smart_slice._validation.ParserLimits`, which resolves each field through
+:func:`_load_limits_from_env` below - the canonical variable name is
+``SMART_SLICE_PARSER_<FIELD>`` (e.g. ``SMART_SLICE_PARSER_MAX_XML_BYTES``).
+"""
+import os
+from dataclasses import fields
+from typing import Optional
+
+__all__ = ["integer_setting", "setting", "_env_flag", "_load_limits_from_env"]
+
+
+def _env_flag(name: str, default: bool = False) -> bool:
+    """Boolean environment switch; anything but a truthy literal means ``default``."""
+    raw = os.environ.get(name)
+    if raw is None:
+        return default
+    return raw.strip().lower() in ("1", "true", "yes", "on")
+
+
+def setting(name: str, default=None, config: Optional[dict] = None):
+    """Read a setting: environment first, then the optional ``config`` mapping."""
+    value = os.environ.get(name)
+    if value is not None:
+        return value
+    if config:
+        value = config.get(name)
+        if value is not None:
+            return value
+    return default
+
+
+def integer_setting(name: str, default: int, minimum: int = 1, config: Optional[dict] = None) -> int:
+    """``setting`` coerced to a positive int, falling back to ``default`` on any problem."""
+    try:
+        raw = setting(name, default, config)
+        if isinstance(raw, bool):
+            return default
+        value = int(raw)
+        return value if value >= minimum else default
+    except (TypeError, ValueError, OverflowError):
+        return default
+
+
+def _load_limits_from_env(klass, config: Optional[dict] = None):
+    """Build a frozen dataclass instance of ``klass`` from ``SMART_SLICE_PARSER_<FIELD>``.
+
+    Shared by ``ParserLimits.load`` so the environment-variable naming convention
+    is defined in exactly one place.
+    """
+    defaults = {field.name: field.default for field in fields(klass)}
+    resolved = {
+        name: integer_setting("SMART_SLICE_PARSER_" + name.upper(), default, config=config)
+        for name, default in defaults.items()
+    }
+    return klass(**resolved)

smart_slice/_i18n.py

@@ -0,0 +1,37 @@
+# coding=utf-8
+"""Minimal gettext shim.
+
+The slicing core ships user facing error strings.  Wrapping them in ``gettext``
+keeps them translatable without depending on a web framework: applications that
+want translations can install their own catalogues for the ``smart_slice``
+domain, everyone else gets the source strings unchanged.
+"""
+import gettext as _gettext
+import os
+from typing import Any
+
+__all__ = ["gettext", "install_translations"]
+
+_DOMAIN = "smart_slice"
+_translation = _gettext.NullTranslations()
+
+
+def install_translations(localedir: str = None, languages=None) -> None:
+    """Load a catalogue for the ``smart_slice`` domain.
+
+    Missing catalogues are not an error: the null translation (identity) stays
+    installed so callers never have to guard.
+    """
+    global _translation
+    localedir = localedir or os.path.join(os.path.dirname(os.path.dirname(__file__)), "locale")
+    try:
+        _translation = _gettext.translation(_DOMAIN, localedir=localedir, languages=languages, fallback=True)
+    except Exception:  # noqa: BLE001 - translation is an enhancement, never fatal
+        _translation = _gettext.NullTranslations()
+
+
+def gettext(message: Any) -> str:
+    """Translate ``message``; returns it unchanged when no catalogue is installed."""
+    if not isinstance(message, str):
+        return str(message)
+    return _translation.gettext(message)

smart_slice/_logging.py

@@ -0,0 +1,22 @@
+# coding=utf-8
+"""Logging setup.
+
+The library never configures handlers or levels on the root logger; it only emits
+records through ``logging.getLogger("smart_slice")`` so that the embedding
+application stays in charge of log routing.  Per-module child loggers are used
+(``smart_slice.handlers.pdf`` ...) which makes selective silencing easy.
+"""
+import logging
+
+__all__ = ["logger", "get_logger"]
+
+ROOT_LOGGER_NAME = "smart_slice"
+
+logger = logging.getLogger(ROOT_LOGGER_NAME)
+
+
+def get_logger(name: str = "") -> logging.Logger:
+    """Return a child logger under the ``smart_slice`` namespace."""
+    if not name:
+        return logger
+    return logging.getLogger(f"{ROOT_LOGGER_NAME}.{name}")

smart_slice/_markdown.py

@@ -0,0 +1,33 @@
+# coding=utf-8
+"""Markdown reference scanners.
+
+Derived from the source platform text helpers (only these two pure-text scanners were
+carried over; the rest of that module was framework-specific).
+
+Both are used when a document arrives as a bundle (a zip of markdown pages plus
+their images): the inner text references sibling files by relative path, and those
+references have to be rewritten to the ``./oss/file/{id}`` form once the assets are
+collected.
+"""
+import re
+from typing import List
+
+__all__ = ["parse_md_image", "parse_md_file_link"]
+
+
+def parse_md_image(content: str) -> List[str]:
+    """Return every markdown image reference (``![alt](target)``) found in ``content``."""
+    matches = re.finditer(r"!\[.*?\]\(.*?\)", content)
+    return [match.group() for match in matches]
+
+
+def parse_md_file_link(content: str) -> List[str]:
+    """Return non-image markdown links like [text](file.mp4) and HTML src/href referencing files."""
+    results = []
+    # Regular markdown links (not images): [text](path)
+    for m in re.finditer(r"(?<!!)\[.*?\]\((.*?)\)", content):
+        results.append(m.group())
+    # HTML tags with src attribute, e.g. <video src="...">
+    for m in re.finditer(r'<\w+[^>]+\bsrc=["\']([^"\']+)["\']', content):
+        results.append(m.group())
+    return results

smart_slice/_optional.py

@@ -0,0 +1,114 @@
+# coding=utf-8
+"""Optional-dependency bookkeeping.
+
+Every format parser is an optional extra (see ``pyproject.toml``).  A handler
+whose parser is not installed must degrade to "this format is not supported"
+instead of breaking ``import smart_slice`` for everybody.
+
+This module is the single place that knows which top-level import belongs to
+which pip extra, so the error text a user sees always names the exact command
+that fixes it::
+
+    Unsupported file format: .pptx (install smart-slice[office] to enable it)
+
+Handlers use it in two places:
+
+``support()``
+    returns False when the parser is missing, so dispatch skips the handler and
+    a later one (or the 400 from the service layer) answers instead;
+``handle()`` / ``get_content()``
+    calls :func:`require` first, so a direct call - or a delegation from another
+    handler such as ``WpsSplitHandle`` - raises a clear 400 rather than a
+    ``NameError``.
+"""
+from typing import Dict, List, Optional
+
+from .exceptions import UnsupportedFormatError
+
+__all__ = ["EXTRA_FOR", "REQUIREMENT_FOR", "extra_for", "require", "missing_hint", "installed"]
+
+#: top-level import name -> pip extra that provides it
+EXTRA_FOR: Dict[str, str] = {
+    "markdownify": "markup",
+    "bs4": "markup",
+    "docx": "office",
+    "pptx": "office",
+    "openpyxl": "office",
+    "xlrd": "office",
+    "olefile": "office",
+    "striprtf": "office",
+    "pypdf": "pdf",
+    "PIL": "pdf",
+    "extract_msg": "mail",
+    "py7zr": "archive",
+    "mobi": "ebook",
+    "pillow_heif": "image",
+    "jieba": "keywords",
+    "rapidocr_onnxruntime": "ocr",
+    "uuid_utils": "fastuuid",
+}
+
+#: top-level import name -> the requirement string quoted in the error message
+REQUIREMENT_FOR: Dict[str, str] = {
+    "markdownify": "markdownify>=0.13",
+    "bs4": "beautifulsoup4>=4.12",
+    "docx": "python-docx>=1.1",
+    "pptx": "python-pptx>=1.0",
+    "openpyxl": "openpyxl>=3.1",
+    "xlrd": "xlrd>=2.0",
+    "olefile": "olefile>=0.46",
+    "striprtf": "striprtf>=0.0.26",
+    "pypdf": "pypdf>=4.0",
+    "PIL": "pillow>=10.0",
+    "extract_msg": "extract-msg>=0.48",
+    "py7zr": "py7zr>=0.20",
+    "mobi": "mobi>=0.3.3",
+    "pillow_heif": "pillow-heif>=0.13",
+    "jieba": "jieba>=0.42",
+    "rapidocr_onnxruntime": "rapidocr-onnxruntime>=1.3",
+    "uuid_utils": "uuid-utils>=0.6",
+}
+
+
+def extra_for(module: str) -> str:
+    """Return the pip extra that provides ``module`` (``all`` when unknown)."""
+    return EXTRA_FOR.get(module.split(".")[0], "all")
+
+
+def missing_hint(module: str) -> str:
+    """Install instruction for a missing parser, e.g. ``pip install smart-slice[office]``."""
+    return f"pip install smart-slice[{extra_for(module)}]"
+
+
+def installed(module: str) -> bool:
+    """Is the optional parser importable right now?"""
+    import importlib.util
+
+    try:
+        return importlib.util.find_spec(module.split(".")[0]) is not None
+    except (ImportError, ValueError):  # pragma: no cover - broken install
+        return False
+
+
+def require(module: str, extension: Optional[str] = None) -> None:
+    """Raise :class:`UnsupportedFormatError` unless ``module`` is importable.
+
+    :param module:    top-level import name, e.g. ``"openpyxl"``
+    :param extension: the file extension being parsed, used in the message
+    """
+    if installed(module):
+        return
+    subject = extension or module
+    raise UnsupportedFormatError(
+        f"Unsupported file format: {subject} "
+        f"(missing optional parser '{module}'; run {missing_hint(module)})"
+    )
+
+
+def missing_extras() -> List[Dict[str, str]]:
+    """Every optional parser that is not importable, as dicts for introspection."""
+    out = []
+    for module, requirement in REQUIREMENT_FOR.items():
+        if not installed(module):
+            out.append({"module": module, "extra": EXTRA_FOR[module], "requirement": requirement})
+    return out

smart_slice/_speedup_py.py

@@ -0,0 +1,64 @@
+# coding=utf-8
+r"""Pure-Python implementation of the optional C accelerator.
+
+Mirrors :mod:`smart_slice._speedup` (C) exactly, function for function, so the
+chunker can fall back to it on platforms without a compiler and tests can assert
+the two agree.  This module is the *reference* implementation: the C version must
+match it, not the other way around.
+
+Both exist only to remove repeated whole-document regex scans; neither changes
+what a heading is.  See :func:`smart_slice.chunker._candidate_titles` for how the
+result is consumed and for the guard that keeps custom pattern lists on the regex
+path.
+"""
+from typing import List, Tuple
+
+__all__ = ["scan_heading_candidates", "IMPLEMENTATION"]
+
+IMPLEMENTATION = "python"
+
+
+def scan_heading_candidates(text: str) -> List[Tuple[int, int, int]]:
+    r"""Report each line that could be an ATX heading, in document order.
+
+    Acceptance rule (identical to the canonical patterns in
+    :mod:`smart_slice.patterns`):
+
+    - the line's first character is ``#`` with **no** leading whitespace, since the
+      patterns anchor with ``(?<=^)`` / ``(?<=\\n)`` immediately before the hashes;
+    - followed by a run of ``#`` and then a single ASCII space (U+0020).  A tab does
+      not qualify - every pattern requires a literal space, which is also why
+      ``## `` never matches a ``###`` line;
+    The scan is deliberately a **superset** of the regular expressions: it does not
+    reproduce their extra guards (the level-1 ``(?!-\*- coding:)`` exclusion, the
+    ``(?!#)`` level guards).  A superset is the only safe direction for a prefilter -
+    reporting an extra candidate merely costs one regex scan that returns empty,
+    whereas missing one would silently drop headings.  Keeping the scan free of
+    per-level special cases is also what lets the C and Python versions stay
+    trivially identical.
+
+    :param text: the (code-fence-masked) document text
+    :return: ``[(line_start, line_end, hashes), ...]`` where ``line_start`` is the
+             index of the first ``#`` and ``line_end`` is one past the last
+             character of the line (newline excluded)
+    """
+    candidates: List[Tuple[int, int, int]] = []
+    length = len(text)
+    line_start = 0
+
+    while line_start < length:
+        line_end = text.find("\n", line_start)
+        if line_end == -1:
+            line_end = length
+
+        cursor = line_start
+        while cursor < line_end and text[cursor] == "#":
+            cursor += 1
+        hashes = cursor - line_start
+
+        if hashes and cursor < line_end and text[cursor] == " ":
+            candidates.append((line_start, line_end, hashes))
+
+        line_start = line_end + 1
+
+    return candidates

smart_slice/_uuid.py

@@ -0,0 +1,40 @@
+# coding=utf-8
+"""UUID helpers - drop-in replacement for ``uuid_utils.compat``.
+
+Image assets extracted from documents get a fresh id which is then embedded in
+the markdown reference (``./oss/file/{id}``).  UUIDv7 is preferred because ids
+stay time-sortable; ``uuid_utils`` provides it as a compiled extension.  When
+that optional dependency is absent, a small pure-python v7 generator is used so
+the package keeps working with a stdlib-only install.
+"""
+import os
+import time
+import uuid as _stdlib_uuid
+
+__all__ = ["UUID", "uuid7", "uuid4"]
+
+UUID = _stdlib_uuid.UUID
+
+
+def uuid4() -> UUID:
+    return _stdlib_uuid.uuid4()
+
+
+def _uuid7_pure_python() -> UUID:
+    """RFC 9562 style v7 UUID: 48-bit unix-ms timestamp + random tail."""
+    timestamp_ms = int(time.time() * 1000)
+    rand = bytearray(os.urandom(10))
+    rand[0] = (rand[0] & 0x0F) | 0x70  # version 7
+    rand[2] = (rand[2] & 0x3F) | 0x80  # RFC 4122 variant
+    body = timestamp_ms.to_bytes(6, "big") + bytes(rand)
+    return UUID(bytes=body)
+
+
+try:  # prefer the compiled implementation when installed
+    from uuid_utils import uuid7 as _uuid7_impl
+except Exception:  # noqa: BLE001 - optional dependency
+    _uuid7_impl = _uuid7_pure_python
+
+
+def uuid7() -> UUID:
+    return _uuid7_impl()

smart_slice/_validation.py

@@ -0,0 +1,174 @@
+# coding=utf-8
+# Derived from the source platform document-slicing layer (see docs/PORTING.md).
+# Imports and framework-facing symbols were re-routed; the parsing and
+# splitting logic is unchanged.
+
+
+from smart_slice._i18n import gettext as _
+from smart_slice.exceptions import SliceError, ResourceLimitError
+from smart_slice._config import _load_limits_from_env, integer_setting
+
+import codecs
+import io
+import re
+import xml.etree.ElementTree as ET
+import zipfile
+from contextlib import contextmanager
+from dataclasses import dataclass, fields
+
+from charset_normalizer import detect
+
+
+
+@dataclass(frozen=True)
+class ParserLimits:
+    max_input_bytes: int = 128 * 1024 * 1024
+    max_xml_bytes: int = 32 * 1024 * 1024
+    max_package_members: int = 10000
+    max_package_bytes: int = 128 * 1024 * 1024
+    max_xml_elements: int = 100000
+    max_xml_depth: int = 128
+    max_table_cells: int = 1000000
+    max_mime_parts: int = 1000
+    max_mime_depth: int = 32
+    max_odf_text_bytes: int = 32 * 1024 * 1024
+
+    @classmethod
+    def load(cls, config=None):
+        return _load_limits_from_env(cls, config=config)
+
+
+BINARY_SIGNATURES = (
+    b'PK\x03\x04', b'PK\x05\x06', b'PK\x07\x08', b'\xd0\xcf\x11\xe0',
+    b'%PDF-', b'\x89PNG', b'\xff\xd8\xff', b'GIF87a', b'GIF89a', b'RIFF',
+    b'MZ', b'\x7fELF', b'\x1f\x8b', b'BZh', b'\xfd7zXZ', b'7z\xbc\xaf', b'Rar!',
+)
+CONTROL_CHARACTERS = re.compile(r'[\x00-\x08\x0b\x0e-\x1f\x7f-\x9f]')
+
+
+def validate_input(buffer, limits=None):
+    limits = limits or ParserLimits.load()
+    if len(buffer) > limits.max_input_bytes:
+        raise ResourceLimitError('File exceeds the parser input byte limit')
+
+
+def decode_text(buffer, encoding=None, detector=detect):
+    validate_input(buffer)
+    if not buffer:
+        return ''
+    if buffer.startswith(BINARY_SIGNATURES):
+        raise SliceError(400, 'Binary content cannot be parsed as text')
+    bom_encoding = None
+    for marker, candidate in (
+        (codecs.BOM_UTF32_LE, 'utf-32'), (codecs.BOM_UTF32_BE, 'utf-32'),
+        (codecs.BOM_UTF8, 'utf-8-sig'),
+        (codecs.BOM_UTF16_LE, 'utf-16'), (codecs.BOM_UTF16_BE, 'utf-16'),
+    ):
+        if buffer.startswith(marker):
+            bom_encoding = candidate
+            break
+    if not bom_encoding and b'\x00' in buffer:
+        raise SliceError(400, 'Binary content or Unicode text without a byte order mark')
+    if bom_encoding or encoding:
+        encodings = [bom_encoding or encoding]
+    else:
+        detected = detector(buffer[:65536]) or {}
+        encodings = ['utf-8']
+        candidate = detected.get('encoding')
+        confidence = detected.get('confidence')
+        try:
+            confidence = 1.0 if confidence is None else float(confidence)
+        except (TypeError, ValueError, OverflowError):
+            confidence = 0.0
+        if isinstance(candidate, str) and 0.5 < confidence <= 1.0:
+            if not candidate.lower().replace('_', '-').startswith(('utf-16', 'utf-32')):
+                encodings.append(candidate)
+    for candidate in encodings:
+        try:
+            content = buffer.decode(candidate)
+        except (UnicodeError, LookupError):
+            continue
+        if CONTROL_CHARACTERS.search(content):
+            raise SliceError(400, 'Binary control characters are not valid text')
+        return content
+    raise SliceError(400, 'Text encoding is invalid or unsupported')
+
+
+class _BoundedTreeBuilder(ET.TreeBuilder):
+    def __init__(self, limits):
+        super().__init__()
+        self.limits = limits
+        self.depth = 0
+        self.elements = 0
+
+    def doctype(self, name, pubid, system):
+        raise SliceError(400, 'DTD and entity declarations are not supported')
+
+    def start(self, tag, attrs):
+        self.depth += 1
+        self.elements += 1
+        if self.depth > self.limits.max_xml_depth or self.elements > self.limits.max_xml_elements:
+            raise ResourceLimitError('XML exceeds the depth or element limit')
+        return super().start(tag, attrs)
+
+    def end(self, tag):
+        self.depth -= 1
+        return super().end(tag)
+
+
+def parse_xml(buffer, limits=None):
+    limits = limits or ParserLimits.load()
+    if len(buffer) > limits.max_xml_bytes:
+        raise ResourceLimitError('XML exceeds the parser byte limit')
+    try:
+        return ET.fromstring(buffer, parser=ET.XMLParser(target=_BoundedTreeBuilder(limits)))
+    except ET.ParseError as error:
+        raise SliceError(400, 'Invalid or damaged XML document') from error
+
+
+@contextmanager
+def document_package(buffer, limits=None):
+    limits = limits or ParserLimits.load()
+    validate_input(buffer, limits)
+    if buffer.startswith(b'\xd0\xcf\x11\xe0'):
+        raise SliceError(400, 'Encrypted or legacy binary Office files are not supported by this parser')
+    try:
+        with zipfile.ZipFile(io.BytesIO(buffer)) as archive:
+            members = archive.infolist()
+            if len(members) > limits.max_package_members:
+                raise ResourceLimitError('Document package exceeds the member count limit')
+            if any(member.flag_bits & 1 for member in members):
+                raise SliceError(400, 'Encrypted document packages are not supported')
+            if sum(member.file_size for member in members) > limits.max_package_bytes:
+                raise ResourceLimitError('Document package exceeds the expanded byte limit')
+            names = [member.filename for member in members]
+            if len(set(names)) != len(names):
+                raise SliceError(400, 'Document package contains duplicate member names')
+            yield archive
+    except (zipfile.BadZipFile, KeyError, NotImplementedError, RuntimeError) as error:
+        if isinstance(error, ResourceLimitError):
+            raise
+        raise SliceError(400, 'Invalid, encrypted or damaged document package') from error
+
+
+def read_package_xml(archive, name, limits=None):
+    limits = limits or ParserLimits.load()
+    if archive.getinfo(name).file_size > limits.max_xml_bytes:
+        raise ResourceLimitError('XML member exceeds the parser byte limit')
+    with archive.open(name) as reader:
+        content = reader.read(limits.max_xml_bytes + 1)
+    return parse_xml(content, limits)
+
+
+def validate_ooxml(buffer, main_part, content_types):
+    limits = ParserLimits.load()
+    with document_package(buffer, limits) as archive:
+        root = read_package_xml(archive, '[Content_Types].xml', limits)
+        namespace = '{http://schemas.openxmlformats.org/package/2006/content-types}'
+        if root.tag != namespace + 'Types':
+            raise SliceError(400, 'Invalid Office content types manifest')
+        matches = [element.get('ContentType') for element in root.findall(namespace + 'Override')
+                   if element.get('PartName') == '/' + main_part]
+        if len(matches) != 1 or matches[0] not in content_types:
+            raise SliceError(400, 'Office document content type does not match the supported format')
+        read_package_xml(archive, main_part, limits)