tryworks 0.1.0__py3-none-any.whl

tryworks/__init__.py

@@ -0,0 +1,16 @@
+"""tryworks: partition documents into LLM-ready elements with zero required dependencies.
+
+    from tryworks.partition.auto import partition
+    from tryworks.chunking.title import chunk_by_title
+
+    elements = partition("report.docx")
+    chunks = chunk_by_title(elements, max_characters=1000)
+
+The module layout mirrors ``unstructured``: replace ``unstructured.`` with ``tryworks.`` in imports,
+or call ``tryworks.alias_as_unstructured()`` to serve code you cannot edit.
+"""
+
+from tryworks.__version__ import __version__
+from tryworks.compat import alias_as_unstructured
+
+__all__ = ["__version__", "alias_as_unstructured"]

tryworks/__main__.py

@@ -0,0 +1,72 @@
+"""Command line: ``tryworks report.pdf`` prints the document's elements as JSON."""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from typing import Optional, Sequence
+
+from tryworks.__version__ import __version__
+
+
+def _render(elements: list, fmt: str) -> str:
+    from tryworks.staging import base
+
+    if fmt == "json":
+        return base.elements_to_json(elements, indent=2)
+    if fmt == "ndjson":
+        return base.elements_to_ndjson(elements)
+    if fmt == "markdown":
+        return base.elements_to_md(elements)
+    if fmt == "csv":
+        return base.convert_to_csv(elements)
+    return base.convert_to_text(elements)
+
+
+def main(argv: Optional[Sequence[str]] = None) -> int:
+    parser = argparse.ArgumentParser(
+        prog="tryworks",
+        description="Partition a document (pdf, docx, pptx, xlsx, html, md, txt, csv, tsv, eml, json) into elements.",
+    )
+    parser.add_argument("source", help="path to a document, or an http(s) URL")
+    parser.add_argument("-f", "--format", choices=["json", "ndjson", "text", "markdown", "csv"], default="json")
+    parser.add_argument("-o", "--output", help="write to this file instead of standard output")
+    parser.add_argument("--content-type", help="MIME type, when the extension does not say")
+    parser.add_argument("--chunking-strategy", choices=["basic", "by_title"], help="chunk the elements")
+    parser.add_argument("--max-characters", type=int, help="chunk size limit (default 500)")
+    parser.add_argument("--include-page-breaks", action="store_true", help="emit PageBreak elements")
+    parser.add_argument("--version", action="version", version=f"tryworks {__version__}")
+    args = parser.parse_args(argv)
+
+    from tryworks.partition.auto import UnsupportedFileFormatError, partition
+
+    kwargs: dict = {"content_type": args.content_type}
+    if args.chunking_strategy:
+        kwargs["chunking_strategy"] = args.chunking_strategy
+    if args.max_characters:
+        kwargs["max_characters"] = args.max_characters
+    if args.include_page_breaks:
+        kwargs["include_page_breaks"] = True
+    source = args.source
+    try:
+        if source.lower().startswith(("http://", "https://")):
+            elements = partition(url=source, **kwargs)
+        else:
+            elements = partition(filename=source, **kwargs)
+    except (UnsupportedFileFormatError, FileNotFoundError, ValueError, NotImplementedError, ImportError) as e:
+        print(f"tryworks: {e}", file=sys.stderr)
+        return 2
+
+    rendered = _render(elements, args.format)
+    if args.output:
+        with open(args.output, "w", encoding="utf-8", newline="") as f:
+            f.write(rendered)
+    else:
+        sys.stdout.buffer.write(rendered.encode("utf-8"))
+        if not rendered.endswith("\n"):
+            sys.stdout.buffer.write(b"\n")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

tryworks/__version__.py

@@ -0,0 +1 @@
+__version__ = "0.1.0"

tryworks/_patterns.py

@@ -0,0 +1,48 @@
+"""Regular expressions shared by the text classifiers and the cleaners."""
+
+from __future__ import annotations
+
+import re
+
+# -- characters that mark a bulleted line on their own --
+UNICODE_BULLETS = (
+    "\u0095\u00b7\u2022\u2023\u2043\u204c\u204d\u2219\u25a0\u25a1\u25aa\u25ab\u25b6\u25ba"
+    "\u25cb\u25cf\u25d8\u25e6\u2619\u2765\u2767\u27a2\u27a4\u29be\u29bf\uf0a7\uf0b7"
+)
+# -- ASCII-ish markers only count when followed by whitespace, so "**bold**", "-5 degrees"
+# -- and "+49 30 1234" are not taken for list items.
+BULLET_RE = re.compile(rf"^\s*(?:[{UNICODE_BULLETS}]|[-*+\u2013\u2014](?=\s))")
+BULLET_PREFIX_RE = re.compile(rf"^\s*(?:[{UNICODE_BULLETS}]+|[-*+\u2013\u2014](?=\s))\s*")
+
+NUMBERED_LIST_RE = re.compile(r"^\s*\d{1,3}(?:\.\d{1,3})*[.)]\s+\S")
+
+EMAIL_ADDRESS_RE = re.compile(r"^[A-Za-z0-9._%+-]+@[A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)*\.[A-Za-z]{2,}$")
+
+US_PHONE_NUMBER_RE = re.compile(
+    r"(?:\+?1[\s.-]?)?(?:\(\d{3}\)\s?|\b\d{3}[\s.-])?\b\d{3}[\s.-]\d{4}\b"
+)
+
+_US_STATES = (
+    "Alabama|Alaska|Arizona|Arkansas|California|Colorado|Connecticut|Delaware|Florida|Georgia|"
+    "Hawaii|Idaho|Illinois|Indiana|Iowa|Kansas|Kentucky|Louisiana|Maine|Maryland|Massachusetts|"
+    "Michigan|Minnesota|Mississippi|Missouri|Montana|Nebraska|Nevada|New Hampshire|New Jersey|"
+    "New Mexico|New York|North Carolina|North Dakota|Ohio|Oklahoma|Oregon|Pennsylvania|"
+    "Rhode Island|South Carolina|South Dakota|Tennessee|Texas|Utah|Vermont|Virginia|Washington|"
+    "West Virginia|Wisconsin|Wyoming|District of Columbia"
+)
+_US_STATE_CODES = (
+    "AL|AK|AZ|AR|CA|CO|CT|DE|DC|FL|GA|HI|ID|IL|IN|IA|KS|KY|LA|ME|MD|MA|MI|MN|MS|MO|MT|NE|NV|NH|"
+    "NJ|NM|NY|NC|ND|OH|OK|OR|PA|RI|SC|SD|TN|TX|UT|VT|VA|WA|WV|WI|WY"
+)
+US_CITY_STATE_ZIP_RE = re.compile(
+    rf"^[A-Za-z][A-Za-z .'\-]*,\s*(?:{_US_STATES}|{_US_STATE_CODES}),?\s+\d{{5}}(?:-\d{{4}})?$",
+    re.IGNORECASE,
+)
+
+ENDS_IN_PUNCT_RE = re.compile(r"[^\w\s]\Z")
+
+# -- paragraph grouping --
+BLANK_LINE_SPLIT_RE = re.compile(r"\n[ \t\f\v\r]*\n\s*")
+LINE_SPLIT_RE = re.compile(r"[ \t\f\v\r]*\n[ \t\f\v\r]*")
+
+WORD_RE = re.compile(r"\w+(?:['\u2019-]\w+)*|[^\w\s]")

tryworks/chunking/__init__.py

@@ -0,0 +1,25 @@
+"""Chunking strategies: "basic" and "by_title"."""
+
+from __future__ import annotations
+
+from typing import Any, Iterable
+
+from tryworks.chunking.base import CHUNK_MAX_CHARS_DEFAULT, CHUNK_MULTI_PAGE_DEFAULT
+from tryworks.documents.elements import Element
+
+__all__ = ["CHUNK_MAX_CHARS_DEFAULT", "CHUNK_MULTI_PAGE_DEFAULT", "chunk"]
+
+_TITLE_ONLY = ("combine_text_under_n_chars", "multipage_sections")
+
+
+def chunk(elements: Iterable[Element], chunking_strategy: str, **kwargs: Any) -> list[Element]:
+    """Apply a named chunking strategy, as ``partition(..., chunking_strategy=...)`` does."""
+    if chunking_strategy == "by_title":
+        from tryworks.chunking.title import chunk_by_title
+
+        return chunk_by_title(elements, **kwargs)
+    if chunking_strategy == "basic":
+        from tryworks.chunking.basic import chunk_elements
+
+        return chunk_elements(elements, **{k: v for k, v in kwargs.items() if k not in _TITLE_ONLY})
+    raise ValueError(f"Unsupported chunking strategy {chunking_strategy!r}; use 'basic' or 'by_title'.")

tryworks/chunking/base.py

@@ -0,0 +1,356 @@
+"""Chunking machinery shared by the "basic" and "by_title" strategies.
+
+Chunking runs in three passes:
+
+1. Pre-chunking groups consecutive elements into the largest runs that fit ``max_characters``,
+   starting a new run at every semantic boundary (a Title for "by_title", optionally a page
+   change) and giving each Table a run of its own.
+2. Combining (by_title only) merges a small run of text into the next one while the first is
+   under ``combine_text_under_n_chars`` and the result still fits.
+3. Chunking turns each run into a CompositeElement (texts joined by a blank line) and splits a
+   run that is still too long at a newline, else a space. Tables pass through whole when they
+   fit, otherwise they are split row by row into TableChunk elements.
+"""
+
+from __future__ import annotations
+
+import copy
+import html
+import re
+from typing import Any, Callable, Iterable, Iterator, Optional
+
+from tryworks.documents.elements import (
+    CompositeElement,
+    Element,
+    ElementMetadata,
+    PageBreak,
+    Table,
+    TableChunk,
+    Text,
+    Title,
+)
+
+CHUNK_MAX_CHARS_DEFAULT = 500
+CHUNK_MULTI_PAGE_DEFAULT = True
+TEXT_SEPARATOR = "\n\n"
+
+BoundaryPredicate = Callable[[Element], bool]
+
+# -- how each metadata field is merged when several elements become one chunk --
+_DROP = frozenset(
+    """category_depth coordinates detection_class_prob detection_origin header_footer_type image_url
+    image_path image_base64 image_mime_type is_continuation is_extracted link_start_indexes links
+    max_characters orig_elements parent_id routing routing_score table_id chunk_index
+    num_carried_over_header_rows segment_start_seconds segment_end_seconds key_value_pairs""".split()
+)
+_LIST_CONCATENATE = frozenset(["emphasized_text_contents", "emphasized_text_tags", "link_texts", "link_urls"])
+_LIST_UNIQUE = frozenset(["languages"])
+_STRING_CONCATENATE = frozenset(["text_as_html"])
+
+_TR_RE = re.compile(r"<tr\b[^>]*>.*?</tr>|<tr\s*/>", re.DOTALL)
+_TAG_RE = re.compile(r"<[^>]+>")
+
+
+class ChunkingOptions:
+    """Validated chunking parameters, with unstructured's defaults."""
+
+    def __init__(
+        self,
+        *,
+        max_characters: Optional[int] = None,
+        new_after_n_chars: Optional[int] = None,
+        overlap: Optional[int] = None,
+        overlap_all: Optional[bool] = None,
+        combine_text_under_n_chars: Optional[int] = None,
+        include_orig_elements: Optional[bool] = None,
+        repeat_table_headers: Optional[bool] = None,
+        skip_table_chunking: Optional[bool] = None,
+        isolate_table: Optional[bool] = None,
+        max_tokens: Optional[int] = None,
+        new_after_n_tokens: Optional[int] = None,
+        tokenizer: Optional[Any] = None,
+        combine_default_to_max: bool = False,
+    ):
+        if max_tokens is not None or new_after_n_tokens is not None or tokenizer is not None:
+            raise NotImplementedError("Token-based chunking is not supported by tryworks; use max_characters.")
+        self.hard_max = CHUNK_MAX_CHARS_DEFAULT if max_characters is None else max_characters
+        if self.hard_max <= 0:
+            raise ValueError(f"'max_characters' argument must be > 0, got {self.hard_max}")
+        if new_after_n_chars is not None and new_after_n_chars < 0:
+            raise ValueError(f"'new_after_n_chars' argument must be >= 0, got {new_after_n_chars}")
+        self.soft_max = self.hard_max if new_after_n_chars is None else min(new_after_n_chars, self.hard_max)
+        self.overlap = overlap or 0
+        if self.overlap < 0 or self.overlap >= self.hard_max:
+            raise ValueError(f"'overlap' argument must be >= 0 and less than `max_characters`, got {self.overlap}")
+        self.inter_chunk_overlap = self.overlap if overlap_all else 0
+        if combine_text_under_n_chars is None:
+            combine_text_under_n_chars = self.hard_max if combine_default_to_max else 0
+        if combine_text_under_n_chars < 0:
+            raise ValueError(f"'combine_text_under_n_chars' argument must be >= 0, got {combine_text_under_n_chars}")
+        if combine_text_under_n_chars > self.hard_max:
+            raise ValueError(
+                "'combine_text_under_n_chars' argument must not exceed `max_characters` value, "
+                f"got {combine_text_under_n_chars}"
+            )
+        self.combine_text_under_n_chars = combine_text_under_n_chars
+        self.include_orig_elements = True if include_orig_elements is None else bool(include_orig_elements)
+        # -- accepted for API compatibility; tryworks' partitioners never mark header rows --
+        self.repeat_table_headers = True if repeat_table_headers is None else bool(repeat_table_headers)
+        self.skip_table_chunking = bool(skip_table_chunking)
+        self.isolate_table = True if isolate_table is None else bool(isolate_table)
+        if self.skip_table_chunking and not self.isolate_table:
+            raise ValueError("'skip_table_chunking=True' requires 'isolate_table=True' (the default).")
+
+
+def _is_table(element: Element) -> bool:
+    return isinstance(element, Table)
+
+
+def _joined_length(elements: list[Element]) -> int:
+    return len(TEXT_SEPARATOR.join(e.text for e in elements if e.text))
+
+
+class PreChunk:
+    """A run of elements that will become one chunk (or several, if it must be split)."""
+
+    def __init__(self, elements: list[Element], opts: ChunkingOptions):
+        self.elements = elements
+        self.opts = opts
+        self.overlap_prefix = ""
+
+    @property
+    def is_table(self) -> bool:
+        return len(self.elements) == 1 and _is_table(self.elements[0])
+
+    @property
+    def contains_table(self) -> bool:
+        return any(_is_table(e) for e in self.elements)
+
+    @property
+    def text(self) -> str:
+        segments = ([self.overlap_prefix] if self.overlap_prefix else []) + [e.text for e in self.elements if e.text]
+        return TEXT_SEPARATOR.join(segments)
+
+    def overlap_tail(self) -> str:
+        n = self.opts.inter_chunk_overlap
+        return self.text[-n:].strip() if n else ""
+
+    def can_combine(self, other: PreChunk) -> bool:
+        if _joined_length(self.elements) >= self.opts.combine_text_under_n_chars:
+            return False
+        if self.opts.isolate_table and (self.contains_table or other.contains_table):
+            return False
+        return _joined_length(self.elements + other.elements) <= self.opts.hard_max
+
+    def combine(self, other: PreChunk) -> PreChunk:
+        return PreChunk(self.elements + other.elements, self.opts)
+
+
+def iter_pre_chunks(
+    elements: Iterable[Element], opts: ChunkingOptions, boundaries: tuple[BoundaryPredicate, ...]
+) -> Iterator[PreChunk]:
+    current: list[Element] = []
+
+    def fits(element: Element) -> bool:
+        if not current:
+            return True
+        if opts.isolate_table and (_is_table(element) or any(_is_table(e) for e in current)):
+            return False
+        length = _joined_length(current)
+        if length >= opts.soft_max:
+            return False
+        return _joined_length(current + [element]) <= opts.hard_max
+
+    for element in elements:
+        if not isinstance(element, Text):
+            continue  # -- CheckBox and other text-less elements are dropped --
+        # -- evaluate every predicate: stateful ones (page tracking) must see each element --
+        new_unit = any([predicate(element) for predicate in boundaries])
+        if current and (new_unit or not fits(element)):
+            yield PreChunk(current, opts)
+            current = []
+        current.append(element)
+    if current:
+        yield PreChunk(current, opts)
+
+
+def combine_pre_chunks(pre_chunks: Iterable[PreChunk], opts: ChunkingOptions) -> Iterator[PreChunk]:
+    accumulated: Optional[PreChunk] = None
+    for pre_chunk in pre_chunks:
+        if accumulated is None:
+            accumulated = pre_chunk
+        elif accumulated.can_combine(pre_chunk):
+            accumulated = accumulated.combine(pre_chunk)
+        else:
+            yield accumulated
+            accumulated = pre_chunk
+    if accumulated is not None:
+        yield accumulated
+
+
+def split_text(s: str, maxlen: int, overlap: int = 0) -> tuple[str, str]:
+    """Split ``s`` into a fragment of at most ``maxlen`` characters and the remainder.
+
+    The split is made at the last newline before ``maxlen``, else the last space, else exactly at
+    ``maxlen``. The remainder begins up to ``overlap`` characters before the split point, moved
+    forward to a word boundary.
+    """
+    if len(s) <= maxlen:
+        return s, ""
+    for separator in ("\n", " "):
+        cut = s.rfind(separator, 0, maxlen + 1)
+        fragment = s[:cut].rstrip() if cut > 0 else ""
+        if not fragment:
+            continue
+        rest_start = cut + 1
+        if overlap:
+            back = max(cut - overlap, 0)
+            boundary = s.find(" ", back, cut)
+            rest_start = boundary + 1 if boundary != -1 else back
+        return fragment, s[rest_start:].lstrip()
+    return s[:maxlen], s[maxlen - overlap :]
+
+
+def consolidate_metadata(elements: list[Element]) -> ElementMetadata:
+    """Merge metadata of several elements: first value, concatenation or union, per field."""
+    merged: dict[str, Any] = {}
+    for element in elements:
+        for name, value in element.metadata.fields.items():
+            if name in _DROP:
+                continue
+            if name in _LIST_CONCATENATE:
+                merged.setdefault(name, []).extend(value)
+            elif name in _LIST_UNIQUE:
+                bucket = merged.setdefault(name, [])
+                bucket.extend(v for v in value if v not in bucket)
+            elif name in _STRING_CONCATENATE:
+                merged[name] = merged.get(name, "") + value
+            elif name == "enrichment_origins":
+                target = merged.setdefault(name, {})
+                for key, records in value.items():
+                    existing = target.setdefault(key, [])
+                    existing.extend(r for r in records if r not in existing)
+            elif name not in merged:
+                merged[name] = copy.deepcopy(value)
+    metadata = ElementMetadata()
+    for name, value in merged.items():
+        setattr(metadata, name, value)
+    return metadata
+
+
+def _orig_elements(elements: list[Element]) -> list[Element]:
+    out = []
+    for element in elements:
+        clone = copy.deepcopy(element)
+        clone.metadata.orig_elements = None
+        out.append(clone)
+    return out
+
+
+def _iter_text_chunks(pre_chunk: PreChunk, opts: ChunkingOptions) -> Iterator[Element]:
+    metadata = consolidate_metadata([e for e in pre_chunk.elements if not isinstance(e, PageBreak)])
+    if opts.include_orig_elements:
+        metadata.orig_elements = _orig_elements(pre_chunk.elements)
+    remainder = pre_chunk.text
+    index = 0
+    while remainder:
+        fragment, remainder = split_text(remainder, opts.hard_max, opts.overlap)
+        chunk_metadata = copy.deepcopy(metadata)
+        if index:
+            chunk_metadata.is_continuation = True
+        yield CompositeElement(text=fragment, metadata=chunk_metadata)
+        index += 1
+
+
+def _row_text(rows: list[str]) -> str:
+    return " ".join(html.unescape(" ".join(_TAG_RE.sub(" ", row).split())) for row in rows).strip()
+
+
+def _iter_table_chunks(table: Table, opts: ChunkingOptions, overlap_prefix: str) -> Iterator[Element]:
+    text = f"{overlap_prefix} {table.text}".strip() if overlap_prefix else table.text
+    table_html = table.metadata.text_as_html
+    if opts.skip_table_chunking or (len(text) <= opts.hard_max and (table_html is None or len(table_html) <= opts.hard_max)):
+        whole = copy.deepcopy(table)
+        whole.text = text
+        whole._element_id = None
+        if opts.include_orig_elements:
+            whole.metadata.orig_elements = _orig_elements([table])
+        yield whole
+        return
+
+    pieces: list[tuple[str, Optional[str]]] = []
+
+    def add_text_pieces(value: str) -> None:
+        remainder = value
+        while remainder:
+            fragment, remainder = split_text(remainder, opts.hard_max, opts.overlap)
+            pieces.append((fragment, None))
+
+    rows = _TR_RE.findall(table_html) if table_html else []
+    if rows:
+        batch: list[str] = []
+        for row in rows:
+            if len(f"<table>{row}</table>") > opts.hard_max or len(_row_text([row])) > opts.hard_max:
+                if batch:
+                    pieces.append((_row_text(batch), f"<table>{''.join(batch)}</table>"))
+                    batch = []
+                add_text_pieces(_row_text([row]))
+                continue
+            candidate = batch + [row]
+            too_big = len(f"<table>{''.join(candidate)}</table>") > opts.hard_max or len(_row_text(candidate)) > opts.hard_max
+            if batch and too_big:
+                pieces.append((_row_text(batch), f"<table>{''.join(batch)}</table>"))
+                batch = [row]
+            else:
+                batch = candidate
+        if batch:
+            pieces.append((_row_text(batch), f"<table>{''.join(batch)}</table>"))
+    else:
+        add_text_pieces(text)
+
+    base = copy.deepcopy(table.metadata)
+    base.orig_elements = None
+    for index, (piece_text, piece_html) in enumerate(pieces):
+        metadata = copy.deepcopy(base)
+        metadata.text_as_html = piece_html
+        metadata.table_id = table.id
+        metadata.chunk_index = index
+        metadata.is_continuation = True if index else None
+        if opts.include_orig_elements:
+            metadata.orig_elements = _orig_elements([table])
+        yield TableChunk(text=piece_text, metadata=metadata)
+
+
+def iter_chunks(pre_chunks: Iterable[PreChunk], opts: ChunkingOptions) -> Iterator[Element]:
+    previous_tail = ""
+    for pre_chunk in pre_chunks:
+        pre_chunk.overlap_prefix = previous_tail if opts.inter_chunk_overlap else ""
+        if pre_chunk.is_table and opts.isolate_table:
+            yield from _iter_table_chunks(pre_chunk.elements[0], opts, pre_chunk.overlap_prefix)  # type: ignore[arg-type]
+        else:
+            yield from _iter_text_chunks(pre_chunk, opts)
+        pre_chunk.overlap_prefix = ""
+        previous_tail = pre_chunk.overlap_tail()
+
+
+def is_title(element: Element) -> bool:
+    return isinstance(element, Title)
+
+
+def is_on_next_page() -> BoundaryPredicate:
+    """A stateful predicate that is True for the first element on each new page."""
+    state: dict[str, Optional[int]] = {"page": None}
+
+    def predicate(element: Element) -> bool:
+        page = element.metadata.page_number
+        if page is None:
+            return False
+        if state["page"] is None:
+            state["page"] = page
+            return False
+        if page != state["page"]:
+            state["page"] = page
+            return True
+        return False
+
+    return predicate

tryworks/chunking/basic.py

@@ -0,0 +1,48 @@
+"""Fill chunks greedily up to a size limit, same API as ``unstructured.chunking.basic``."""
+
+from __future__ import annotations
+
+from typing import Any, Iterable, Iterator, Optional
+
+from tryworks.chunking.base import ChunkingOptions, iter_chunks, iter_pre_chunks
+from tryworks.documents.elements import Element
+
+
+def iter_chunk_elements(
+    elements: Iterable[Element],
+    *,
+    include_orig_elements: Optional[bool] = None,
+    max_characters: Optional[int] = None,
+    max_tokens: Optional[int] = None,
+    new_after_n_chars: Optional[int] = None,
+    new_after_n_tokens: Optional[int] = None,
+    overlap: Optional[int] = None,
+    overlap_all: Optional[bool] = None,
+    tokenizer: Optional[Any] = None,
+    repeat_table_headers: Optional[bool] = None,
+    skip_table_chunking: Optional[bool] = None,
+    isolate_table: Optional[bool] = None,
+) -> Iterator[Element]:
+    """Lazily yield chunks; see ``chunk_elements``."""
+    opts = ChunkingOptions(
+        max_characters=max_characters,
+        new_after_n_chars=new_after_n_chars,
+        overlap=overlap,
+        overlap_all=overlap_all,
+        include_orig_elements=include_orig_elements,
+        repeat_table_headers=repeat_table_headers,
+        skip_table_chunking=skip_table_chunking,
+        isolate_table=isolate_table,
+        max_tokens=max_tokens,
+        new_after_n_tokens=new_after_n_tokens,
+        tokenizer=tokenizer,
+    )
+    yield from iter_chunks(iter_pre_chunks(elements, opts, ()), opts)
+
+
+def chunk_elements(elements: Iterable[Element], **kwargs: Any) -> list[Element]:
+    """Pack consecutive elements into chunks of up to ``max_characters`` (default 500).
+
+    Sections are not respected; tables still get chunks of their own.
+    """
+    return list(iter_chunk_elements(elements, **kwargs))

tryworks/chunking/title.py

@@ -0,0 +1,69 @@
+"""Chunk elements into sections that start at each Title, same API as ``unstructured.chunking.title``."""
+
+from __future__ import annotations
+
+from typing import Any, Iterable, Iterator, Optional
+
+from tryworks.chunking.base import (
+    CHUNK_MULTI_PAGE_DEFAULT,
+    ChunkingOptions,
+    combine_pre_chunks,
+    is_on_next_page,
+    is_title,
+    iter_chunks,
+    iter_pre_chunks,
+)
+from tryworks.documents.elements import Element
+
+
+def iter_chunks_by_title(
+    elements: Iterable[Element],
+    *,
+    combine_text_under_n_chars: Optional[int] = None,
+    include_orig_elements: Optional[bool] = None,
+    max_characters: Optional[int] = None,
+    max_tokens: Optional[int] = None,
+    multipage_sections: Optional[bool] = None,
+    new_after_n_chars: Optional[int] = None,
+    new_after_n_tokens: Optional[int] = None,
+    overlap: Optional[int] = None,
+    overlap_all: Optional[bool] = None,
+    tokenizer: Optional[Any] = None,
+    repeat_table_headers: Optional[bool] = None,
+    skip_table_chunking: Optional[bool] = None,
+    isolate_table: Optional[bool] = None,
+) -> Iterator[Element]:
+    """Lazily yield chunks; see ``chunk_by_title``."""
+    opts = ChunkingOptions(
+        max_characters=max_characters,
+        new_after_n_chars=new_after_n_chars,
+        overlap=overlap,
+        overlap_all=overlap_all,
+        combine_text_under_n_chars=combine_text_under_n_chars,
+        include_orig_elements=include_orig_elements,
+        repeat_table_headers=repeat_table_headers,
+        skip_table_chunking=skip_table_chunking,
+        isolate_table=isolate_table,
+        max_tokens=max_tokens,
+        new_after_n_tokens=new_after_n_tokens,
+        tokenizer=tokenizer,
+        combine_default_to_max=True,
+    )
+    multipage = CHUNK_MULTI_PAGE_DEFAULT if multipage_sections is None else multipage_sections
+    boundaries = (is_title,) if multipage else (is_title, is_on_next_page())
+    pre_chunks = iter_pre_chunks(elements, opts, boundaries)
+    if opts.combine_text_under_n_chars > 0:
+        pre_chunks = combine_pre_chunks(pre_chunks, opts)
+    yield from iter_chunks(pre_chunks, opts)
+
+
+def chunk_by_title(elements: Iterable[Element], **kwargs: Any) -> list[Element]:
+    """Group elements into chunks of up to ``max_characters`` (default 500), one section per Title.
+
+    ``new_after_n_chars`` closes a chunk early once it reaches that size.
+    ``combine_text_under_n_chars`` (default ``max_characters``) merges small sections.
+    ``multipage_sections=False`` also starts a new section on every page.
+    ``overlap`` repeats that many characters across split chunks (all chunks with ``overlap_all``).
+    ``include_orig_elements`` (default True) keeps the source elements in ``metadata.orig_elements``.
+    """
+    return list(iter_chunks_by_title(elements, **kwargs))