flowmark 0.1.0__tar.gz

flowmark-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Joshua Levy
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

flowmark-0.1.0/PKG-INFO

@@ -0,0 +1,53 @@
+Metadata-Version: 2.3
+Name: flowmark
+Version: 0.1.0
+Summary: Better line wrapping and formatting for plaintext and Markdown
+License: MIT
+Author: Joshua Levy
+Author-email: joshua@cal.berkeley.edu
+Requires-Python: >=3.10,<4.0
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Dist: marko (>=2.1.2,<3.0.0)
+Requires-Dist: regex (>=2024.11.6,<2025.0.0)
+Project-URL: Repository, https://github.com/jlevy/flowmark
+Description-Content-Type: text/markdown
+
+# flowmark
+
+Flowmark is a new Python implementation of text line wrapping and filling.
+
+It can be used as a more flexible alternative to Python's
+[`textwrap`](https://docs.python.org/3/library/textwrap.html) with a few more features,
+such as full customizability of initial and subsequent indentation strings, and more
+control over when to split words, so it won't break lines within HTML tags.
+
+It also combines line wrapping with support for Markdown and offers Markdown
+auto-formatting, like [markdownfmt](https://github.com/shurcooL/markdownfmt), also with
+controllable line wrapping options.
+
+One key use case is to normalize Markdown in a standard, readable way that makes diffs
+easy to read and use on GitHub.
+This can be useful for documentation workflows and also to compare LLM outputs that are
+Markdown.
+
+Finally, it has options to use heuristics to split on sentences, which can make diffs
+much more readable. (For an example of this, look at the source to this readme file.)
+
+It aims to be small and simple and have only a few dependencies, currently only
+[`marko`](https://github.com/frostming/marko) and
+[`regex`](https://pypi.org/project/regex/).
+
+This is a new and simple package (previously I'd implemented something like this
+[for Atom](https://github.com/jlevy/atom-flowmark)) but I plan to add more support for
+command line usage and VSCode/Cursor auto-formatting in the future.
+
+* * *
+
+*This project was built from
+[simple-modern-poetry](https://github.com/jlevy/simple-modern-poetry).*
+

flowmark-0.1.0/README.md

@@ -0,0 +1,33 @@
+# flowmark
+
+Flowmark is a new Python implementation of text line wrapping and filling.
+
+It can be used as a more flexible alternative to Python's
+[`textwrap`](https://docs.python.org/3/library/textwrap.html) with a few more features,
+such as full customizability of initial and subsequent indentation strings, and more
+control over when to split words, so it won't break lines within HTML tags.
+
+It also combines line wrapping with support for Markdown and offers Markdown
+auto-formatting, like [markdownfmt](https://github.com/shurcooL/markdownfmt), also with
+controllable line wrapping options.
+
+One key use case is to normalize Markdown in a standard, readable way that makes diffs
+easy to read and use on GitHub.
+This can be useful for documentation workflows and also to compare LLM outputs that are
+Markdown.
+
+Finally, it has options to use heuristics to split on sentences, which can make diffs
+much more readable. (For an example of this, look at the source to this readme file.)
+
+It aims to be small and simple and have only a few dependencies, currently only
+[`marko`](https://github.com/frostming/marko) and
+[`regex`](https://pypi.org/project/regex/).
+
+This is a new and simple package (previously I'd implemented something like this
+[for Atom](https://github.com/jlevy/atom-flowmark)) but I plan to add more support for
+command line usage and VSCode/Cursor auto-formatting in the future.
+
+* * *
+
+*This project was built from
+[simple-modern-poetry](https://github.com/jlevy/simple-modern-poetry).*

flowmark-0.1.0/pyproject.toml

@@ -0,0 +1,90 @@
+[tool.poetry]
+name = "flowmark"
+# Keep this a dev version, as the dynamic versioning plugin is used for actual release versions:
+version = "0.1.0"
+description = "Better line wrapping and formatting for plaintext and Markdown"
+authors = ["Joshua Levy <joshua@cal.berkeley.edu>"]
+readme = "README.md"
+license = "MIT"
+repository = "https://github.com/jlevy/flowmark"
+requires-poetry = ">=2.0"
+
+# Useful plugins:
+# https://github.com/python-poetry/poetry-plugin-shell
+# https://github.com/mtkennerly/poetry-dynamic-versioning
+# https://github.com/MousaZeidBaker/poetry-plugin-up
+[tool.poetry.requires-plugins]
+poetry-plugin-shell = "^1.0.1"
+poetry-dynamic-versioning = "^1.7.0"
+poetry-plugin-up = "^0.9.0"
+
+[tool.poetry.dependencies]
+python = "^3.10"
+marko = "^2.1.2"
+regex = "^2024.11.6"
+
+[tool.poetry.group.dev.dependencies]
+black = "^24.10.0"
+pytest = "^8.3.4"
+ruff = "^0.4.10"
+usort = "^1.0.8.post1"
+mypy = "^1.15.0"
+codespell = "^2.4.1"
+rich = "^13.9.4"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"
+
+# Auto-generated version for builds based on git tag or commit.
+#
+# Preferred styles of versioning (PEP 440 compliant):
+#   1.2.3               (stable release)
+#   1.2.3a4             (pre-release, no dash or dots)
+#   1.2.3rc1            (release candidate)
+#   1.2.3.dev4+abcdef12 (development build, .dev for dev releases and +commit hash)
+#
+# Tags with a "v" prefix (e.g., v1.2.3) or additional dots (e.g., v1.2.3.rc1) are recognized
+# but normalized to the preferred style for distribution.
+# See: https://sam.hooke.me/note/2023/08/poetry-automatically-generated-package-version-from-git-commit/
+[tool.poetry-dynamic-versioning]
+enable = false
+vcs = "git"
+pattern = "^v?(?P<base>\\d+\\.\\d+\\.\\d+)(-?((?P<stage>[a-zA-Z]+)\\.?(?P<revision>\\d+)?))?"
+format-jinja = """{%- if distance == 0 -%}
+        {{- base -}}
+        {%- if stage -%}
+            {{ stage }}{{ revision | default("") }}
+        {%- endif -%}
+    {%- else -%}
+        {{- base }}.dev{{ distance }}+{{commit}}
+    {%- endif -%}"""
+
+[tool.black]
+line-length = 88
+
+[tool.ruff]
+line-length = 88
+
+[tool.ruff.lint]
+ignore = ["E402", "E731", "E712"]
+
+[tool.mypy]
+disable_error_code = [
+    "import-untyped",
+]
+
+[tool.codespell]
+# ignore-words-list = "foo,bar"
+# skip = "foo.py,bar.py"
+
+[tool.pytest.ini_options]
+python_files = ["*.py"]
+python_classes = ["Test*"]
+python_functions = ["test_*"]
+testpaths = [
+    "src",
+    "tests",
+]
+norecursedirs = []
+filterwarnings = []

flowmark-0.1.0/src/flowmark/__init__.py

@@ -0,0 +1,19 @@
+__all__ = (
+    "fill_text",
+    "fill_markdown",
+    "normalize_markdown",
+    "wrap_paragraph",
+    "wrap_paragraph_lines",
+    "wrap_lines_to_width",
+    "wrap_lines_using_sentences",
+    "Wrap",
+)
+
+from .markdown_filling import (
+    fill_markdown,
+    normalize_markdown,
+    wrap_lines_to_width,
+    wrap_lines_using_sentences,
+)
+from .text_filling import fill_text, Wrap
+from .text_wrapping import wrap_paragraph, wrap_paragraph_lines

flowmark-0.1.0/src/flowmark/markdown_filling.py

@@ -0,0 +1,419 @@
+"""
+Auto-formatting of Markdown text.
+
+This is similar to what is offered by
+[markdownfmt](https://github.com/shurcooL/markdownfmt) but with a few adaptations,
+including more aggressive normalization and support for wrapping of lines
+semi-semantically (e.g. on sentence boundaries when appropriate).
+(See [here](https://github.com/shurcooL/markdownfmt/issues/17) for some old
+discussion on why line wrapping this way is convenient.)
+"""
+
+import re
+from contextlib import contextmanager
+from textwrap import dedent
+from typing import Callable, cast, Generator, List, Protocol, Tuple
+
+from marko import block, inline
+from marko.block import HTMLBlock
+from marko.parser import Parser
+from marko.renderer import Renderer
+from marko.source import Source
+
+from .sentence_split_regex import split_sentences_regex
+from .text_wrapping import DEFAULT_LEN_FUNCTION, wrap_paragraph, wrap_paragraph_lines
+
+
+DEFAULT_WRAP_WIDTH = 88
+"""
+Default wrap width for Markdown content. This is a compromise between traditional
+but sometimes impractically narrow 80-char console width and being too wide to
+read comfortably for text, markup, and code. 88 is the same as Black.
+"""
+
+
+DEFAULT_MIN_LINE_LEN = 20
+"""Default minimum line length for sentence breaking."""
+
+
+class LineWrapper(Protocol):
+    """Takes a text string and any indents to use, and returns the wrapped text."""
+
+    def __call__(
+        self, text: str, initial_indent: str, subsequent_indent: str
+    ) -> str: ...
+
+
+class SentenceSplitter(Protocol):
+    """Takes a text string and returns a list of sentences."""
+
+    def __call__(self, text: str) -> List[str]: ...
+
+
+def _normalize_html_comments(text: str, break_str: str = "\n\n") -> str:
+    """
+    Put HTML comments as standalone paragraphs.
+    """
+
+    # Small hack to avoid changing frontmatter format, for the rare corner
+    # case where Markdown contains HTML-style frontmatter.
+    def not_frontmatter(text: str) -> bool:
+        return "<!---" not in text
+
+    # TODO: Probably want do this for <div>s too.
+    return _ensure_surrounding_breaks(
+        text, [("<!--", "-->")], break_str=break_str, filter=not_frontmatter
+    )
+
+
+def _ensure_surrounding_breaks(
+    html: str,
+    tag_pairs: List[Tuple[str, str]],
+    filter: Callable[[str], bool] = lambda _: True,
+    break_str: str = "\n\n",
+) -> str:
+    for start_tag, end_tag in tag_pairs:
+        pattern = re.compile(
+            rf"(\s*{re.escape(start_tag)}.*?{re.escape(end_tag)}\s*)", re.DOTALL
+        )
+
+        def replacer(match: re.Match[str]) -> str:
+            if not filter(match.group(0)):
+                return match.group(0)
+
+            content = match.group(1).strip()
+            before = after = break_str
+
+            if match.start() == 0:
+                before = ""
+            if match.end() == len(html):
+                after = ""
+
+            return f"{before}{content}{after}"
+
+        html = re.sub(pattern, replacer, html)
+
+    return html
+
+
+# XXX Turn off Marko's parsing of block HTML.
+# Block parsing with comments or block elements has some counterintuitive issues:
+# https://github.com/frostming/marko/issues/202
+# Another solution might be to always put a newline after a closing block tag during
+# normalization, to avoid this confusion?
+# For now, just ignoring block tags.
+class CustomHTMLBlock(HTMLBlock):
+    @classmethod
+    def match(cls, source: Source) -> int | bool:
+        return False
+
+
+class CustomParser(Parser):
+    def __init__(self) -> None:
+        super().__init__()
+        self.block_elements["HTMLBlock"] = CustomHTMLBlock
+
+
+class _MarkdownNormalizer(Renderer):
+    """
+    Render Markdown in normalized form. This is the internal implementation.
+    You likely want to use `normalize_markdown()` instead.
+    Based on: https://github.com/frostming/marko/blob/master/marko/md_renderer.py
+    """
+
+    def __init__(self, line_wrapper: LineWrapper) -> None:
+        super().__init__()
+        self._prefix: str = (
+            ""  # The prefix on the first line, with a bullet, such as `  - `.
+        )
+        self._second_prefix: str = ""  # The prefix on subsequent lines, such as `    `.
+        self._suppress_item_break: bool = True
+        self._line_wrapper = line_wrapper
+
+    def __enter__(self) -> "_MarkdownNormalizer":
+        self._prefix = ""
+        self._second_prefix = ""
+        return super().__enter__()
+
+    @contextmanager
+    def container(
+        self, prefix: str, second_prefix: str = ""
+    ) -> Generator[None, None, None]:
+        old_prefix, old_second_prefix = self._prefix, self._second_prefix
+        self._prefix += prefix
+        self._second_prefix += second_prefix
+        yield
+        self._prefix, self._second_prefix = old_prefix, old_second_prefix
+
+    def render_paragraph(self, element: block.Paragraph) -> str:
+        # Suppress item breaks on list items following a top-level paragraph.
+        if not self._prefix:
+            self._suppress_item_break = True
+        children = self.render_children(element)
+        wrapped_text = self._line_wrapper(
+            children,
+            self._prefix,
+            self._second_prefix,
+        )
+        self._prefix = self._second_prefix
+        return wrapped_text + "\n"
+
+    def render_list(self, element: block.List) -> str:
+        result: List[str] = []
+        if element.ordered:
+            for num, child in enumerate(element.children, element.start):
+                with self.container(f"{num}. ", " " * (len(str(num)) + 2)):
+                    result.append(self.render(child))
+        else:
+            for child in element.children:
+                with self.container(f"{element.bullet} ", "  "):
+                    result.append(self.render(child))
+
+        self._prefix = self._second_prefix
+        return "".join(result)
+
+    def render_list_item(self, element: block.ListItem) -> str:
+        result = ""
+        # We want all list items to have two newlines between them.
+        if self._suppress_item_break:
+            self._suppress_item_break = False
+        else:
+            # Add the newline between paragraphs. Normally this would be an empty line but
+            # within a quote block it would be the secondary prefix, like `> `.
+            result += self._second_prefix.strip() + "\n"
+        result += self.render_children(element)
+        return result
+
+    def render_quote(self, element: block.Quote) -> str:
+        with self.container("> ", "> "):
+            result = self.render_children(element).rstrip("\n")
+        self._prefix = self._second_prefix
+        return f"{result}\n"
+
+    def _render_code(self, element: block.CodeBlock | block.FencedCode) -> str:
+        # Preserve code content without reformatting.
+        code_child = cast(inline.RawText, element.children[0])
+        code_content = code_child.children.rstrip("\n")
+        lang = element.lang if isinstance(element, block.FencedCode) else ""
+        extra = element.extra if isinstance(element, block.FencedCode) else ""
+        extra_text = f" {extra}" if extra else ""
+        lang_text = f"{lang}{extra_text}" if lang else ""
+        lines = [f"{self._prefix}```{lang_text}"]
+        lines.extend(
+            f"{self._second_prefix}{line}" for line in code_content.splitlines()
+        )
+        lines.append(f"{self._second_prefix}```")
+        self._prefix = self._second_prefix
+        return "\n".join(lines) + "\n"
+
+    def render_fenced_code(self, element: block.FencedCode) -> str:
+        return self._render_code(element)
+
+    def render_code_block(self, element: block.CodeBlock) -> str:
+        # Convert indented code blocks to fenced code blocks.
+        return self._render_code(element)
+
+    def render_html_block(self, element: block.HTMLBlock) -> str:
+        result = f"{self._prefix}{element.body}"
+        self._prefix = self._second_prefix
+        return result
+
+    def render_thematic_break(self, element: block.ThematicBreak) -> str:
+        result = f"{self._prefix}* * *\n"
+        self._prefix = self._second_prefix
+        return result
+
+    def render_heading(self, element: block.Heading) -> str:
+        result = (
+            f"{self._prefix}{'#' * element.level} {self.render_children(element)}\n"
+        )
+        self._prefix = self._second_prefix
+        return result
+
+    def render_setext_heading(self, element: block.SetextHeading) -> str:
+        return self.render_heading(cast("block.Heading", element))
+
+    def render_blank_line(self, element: block.BlankLine) -> str:
+        if self._prefix.strip():
+            result = f"{self._prefix}\n"
+        else:
+            result = "\n"
+        self._suppress_item_break = True
+        self._prefix = self._second_prefix
+        return result
+
+    def render_link_ref_def(self, element: block.LinkRefDef) -> str:
+        link_text = element.dest
+        if element.title:
+            link_text += f" {element.title}"
+        return f"[{element.label}]: {link_text}\n"
+
+    def render_emphasis(self, element: inline.Emphasis) -> str:
+        return f"*{self.render_children(element)}*"
+
+    def render_strong_emphasis(self, element: inline.StrongEmphasis) -> str:
+        return f"**{self.render_children(element)}**"
+
+    def render_inline_html(self, element: inline.InlineHTML) -> str:
+        return cast(str, element.children)
+
+    def render_link(self, element: inline.Link) -> str:
+        link_text = self.render_children(element)
+        link_title = (
+            '"{}"'.format(element.title.replace('"', '\\"')) if element.title else None
+        )
+        assert self.root_node
+        label = next(
+            (
+                k
+                for k, v in self.root_node.link_ref_defs.items()
+                if v == (element.dest, link_title)
+            ),
+            None,
+        )
+        if label is not None:
+            if label == link_text:
+                return f"[{label}]"
+            return f"[{link_text}][{label}]"
+        title = f" {link_title}" if link_title is not None else ""
+        return f"[{link_text}]({element.dest}{title})"
+
+    def render_auto_link(self, element: inline.AutoLink) -> str:
+        return f"<{element.dest}>"
+
+    def render_image(self, element: inline.Image) -> str:
+        template = "![{}]({}{})"
+        title = (
+            ' "{}"'.format(element.title.replace('"', '\\"')) if element.title else ""
+        )
+        return template.format(self.render_children(element), element.dest, title)
+
+    def render_literal(self, element: inline.Literal) -> str:
+        return f"\\{element.children}"
+
+    def render_raw_text(self, element: inline.RawText) -> str:
+        from marko.ext.pangu import PANGU_RE
+
+        return re.sub(PANGU_RE, " ", element.children)
+
+    def render_line_break(self, element: inline.LineBreak) -> str:
+        return "\n" if element.soft else "\\\n"
+
+    def render_code_span(self, element: inline.CodeSpan) -> str:
+        text = element.children
+        if text and (text[0] == "`" or text[-1] == "`"):
+            return f"`` {text} ``"
+        return f"`{element.children}`"
+
+
+def split_sentences_no_min_length(text: str) -> List[str]:
+    return split_sentences_regex(text, min_length=0)
+
+
+def wrap_lines_to_width(
+    text: str,
+    initial_indent: str,
+    subsequent_indent: str,
+    width: int = DEFAULT_WRAP_WIDTH,
+) -> str:
+    """
+    Wrap lines of text to a given width.
+    """
+    return wrap_paragraph(
+        text,
+        width=width,
+        initial_indent=initial_indent,
+        subsequent_indent=subsequent_indent,
+    )
+
+
+def wrap_lines_using_sentences(
+    text: str,
+    initial_indent: str,
+    subsequent_indent: str,
+    split_sentences: SentenceSplitter = split_sentences_no_min_length,
+    width: int = DEFAULT_WRAP_WIDTH,
+    min_line_len: int = DEFAULT_MIN_LINE_LEN,
+    len_fn: Callable[[str], int] = DEFAULT_LEN_FUNCTION,
+) -> str:
+    """
+    Wrap lines of text to a given width but also keep sentences on their own lines.
+    If the last line ends up shorter than min_line_len, it's combined with the next sentence.
+    """
+    text = text.replace("\n", " ")
+    lines: List[str] = []
+    first_line = True
+    length = len_fn
+    initial_indent_len = len_fn(initial_indent)
+    subsequent_indent_len = len_fn(subsequent_indent)
+
+    sentences = split_sentences(text)
+
+    for i, sentence in enumerate(sentences):
+        current_column = initial_indent_len if first_line else subsequent_indent_len
+        if len(lines) > 0 and length(lines[-1]) < min_line_len:
+            current_column += length(lines[-1])
+
+        wrapped = wrap_paragraph_lines(
+            sentence,
+            width=width,
+            initial_column=current_column,
+            subsequent_offset=subsequent_indent_len,
+        )
+        # If last line is shorter than min_line_len, combine with next line.
+        # Also handles if the first word doesn't fit.
+        if (
+            len(lines) > 0
+            and length(lines[-1]) < min_line_len
+            and length(lines[-1]) + 1 + length(wrapped[0]) <= width
+        ):
+            lines[-1] += " " + wrapped[0]
+            wrapped.pop(0)
+
+        lines.extend(wrapped)
+
+        first_line = False
+
+    # Now insert the indents and assemble the paragraph.
+    if initial_indent and len(lines) > 0:
+        lines[0] = initial_indent + lines[0]
+    if subsequent_indent and len(lines) > 1:
+        lines[1:] = [subsequent_indent + line for line in lines[1:]]
+
+    return "\n".join(lines)
+
+
+def normalize_markdown(
+    markdown_text: str, line_wrapper: LineWrapper = wrap_lines_using_sentences
+) -> str:
+    """
+    Normalize Markdown text. Wraps lines and adds line breaks within paragraphs and on
+    best-guess estimations of sentences, to make diffs more readable.
+
+    Also enforces that all list items have two newlines between them, so that items
+    are separate paragraphs when viewed as plaintext.
+    """
+    markdown_text = markdown_text.strip() + "\n"
+
+    # If we want to normalize HTML blocks or comments.
+    markdown_text = _normalize_html_comments(markdown_text)
+
+    # Normalize the markdown and wrap lines.
+    parser = CustomParser()
+    parsed = parser.parse(markdown_text)
+    result = _MarkdownNormalizer(line_wrapper).render(parsed)
+    return result
+
+
+def fill_markdown(
+    markdown_text: str,
+    dedent_input: bool = True,
+    line_wrapper: LineWrapper = wrap_lines_to_width,
+) -> str:
+    """
+    Normalize and wrap Markdown text filling paragraphs to the full width.
+    Also dedents and strips the input, so it can be used on docstrings.
+    """
+    if dedent_input:
+        markdown_text = dedent(markdown_text).strip()
+    return normalize_markdown(markdown_text, line_wrapper=line_wrapper)

flowmark-0.1.0/src/flowmark/sentence_split_regex.py

@@ -0,0 +1,62 @@
+from typing import Callable, List
+
+import regex
+
+
+# These heuristics are from Flowmark:
+# https://github.com/jlevy/atom-flowmark/blob/master/lib/remark-smart-word-wrap.js#L17-L33
+
+# They work pretty well when used for formatting and editing documents in English.
+# Note this is smarter than Python textwrap's simple heuristic:
+# https://github.com/python/cpython/blob/main/Lib/textwrap.py#L105-L110
+
+# Heuristic: End of sentence must be two letters or more, with the last letter lowercase,
+# followed by a period, exclamation point, question mark. A final or preceding parenthesis
+# or quote is allowed.
+#
+# Does not break on colon or semicolon currently as that seems to have false positives too
+# often with code or other syntax.
+#
+# XXX: Could also handle rare cases with both quotes and parentheses at sentence end
+# but may not be worth it. Also does not detect sentences ending in numerals, which
+# tends to cause too many false positives. Should be OK for most Latin languages but
+# may need to rethink the 2-letter restriction for some languages.
+SENTENCE_RE = regex.compile(r"(\b\p{L}+[\p{Ll}])([.?!]['\"’”)]?|['\"’”)][.?!]) *$")
+
+# Second heuristic: Very short sentences often not so useful.
+SENTENCE_MIN_LENGTH = 15
+
+
+def heuristic_end_of_sentence(word: str) -> bool:
+    return bool(SENTENCE_RE.search(word))
+
+
+def split_sentences_regex(
+    text: str,
+    heuristic: Callable[[str], bool] = heuristic_end_of_sentence,
+    min_length: int = SENTENCE_MIN_LENGTH,
+) -> List[str]:
+    """
+    Split text into sentences using an approximate, fast regex heuristic. (English.)
+    Goal is to be conservative, not perfect, avoiding excessive breaks.
+
+    :param text: The text to split into sentences.
+    :param heuristic: A callable that returns True if text ends at the end of a sentence.
+    :param min_length: The minimum length of a sentence in characters.
+    :return: A list of sentences.
+    """
+    words = text.split()
+    sentences: List[str] = []
+    sentence: List[str] = []
+    words_len = 0
+    for word in words:
+        sentence.append(word)
+        words_len += len(word)
+        sentence_len = words_len + len(sentence) - 1
+        if heuristic(word) and sentence_len >= min_length:
+            sentences.append(" ".join(sentence))
+            sentence = []
+            words_len = 0
+    if sentence:
+        sentences.append(" ".join(sentence))
+    return sentences

flowmark-0.1.0/src/flowmark/text_filling.py

@@ -0,0 +1,145 @@
+import re
+from enum import Enum
+from typing import Callable, List
+
+from .text_wrapping import (
+    DEFAULT_LEN_FUNCTION,
+    html_md_word_splitter,
+    WordSplitter,
+    wrap_paragraph,
+)
+
+DEFAULT_WRAP_WIDTH = 88
+
+DEFAULT_INDENT = "    "
+
+
+def split_paragraphs(text: str) -> List[str]:
+    return [p.strip() for p in re.split(r"\n{2,}", text)]
+
+
+class Wrap(Enum):
+    """
+    A few convenient text wrapping styles.
+    """
+
+    NONE = "none"
+    """No wrapping."""
+
+    WRAP = "wrap"
+    """Basic wrapping but preserves whitespace within paragraphs."""
+
+    WRAP_FULL = "wrap_full"
+    """Wraps and also normalizes whitespace."""
+
+    WRAP_INDENT = "wrap_indent"
+    """Wrap and also indent."""
+
+    INDENT_ONLY = "indent_only"
+    """Just indent."""
+
+    HANGING_INDENT = "hanging_indent"
+    """Wrap with hanging indent (indented except for the first line)."""
+
+    MARKDOWN_ITEM = "markdown_item"
+    """2-space hanging indent for markdown list items."""
+
+    @property
+    def initial_indent(self) -> str:
+        if self in [Wrap.INDENT_ONLY, Wrap.WRAP_INDENT]:
+            return DEFAULT_INDENT
+        else:
+            return ""
+
+    @property
+    def subsequent_indent(self) -> str:
+        if self == Wrap.MARKDOWN_ITEM:
+            return "  "
+        elif self in [Wrap.INDENT_ONLY, Wrap.WRAP_INDENT, Wrap.HANGING_INDENT]:
+            return DEFAULT_INDENT
+        else:
+            return ""
+
+    @property
+    def should_wrap(self) -> bool:
+        return self in [
+            Wrap.WRAP,
+            Wrap.WRAP_FULL,
+            Wrap.WRAP_INDENT,
+            Wrap.HANGING_INDENT,
+            Wrap.MARKDOWN_ITEM,
+        ]
+
+    @property
+    def initial_indent_first_para_only(self) -> bool:
+        return self in [Wrap.HANGING_INDENT, Wrap.MARKDOWN_ITEM]
+
+    @property
+    def replace_whitespace(self) -> bool:
+        return self in [Wrap.WRAP_FULL, Wrap.WRAP_INDENT, Wrap.HANGING_INDENT]
+
+
+def fill_text(
+    text: str,
+    text_wrap=Wrap.WRAP,
+    width=DEFAULT_WRAP_WIDTH,
+    extra_indent: str = "",
+    empty_indent: str = "",
+    initial_column: int = 0,
+    word_splitter: WordSplitter = html_md_word_splitter,
+    len_fn: Callable[[str], int] = DEFAULT_LEN_FUNCTION,
+) -> str:
+    """
+    Most flexible way to wrap and fill any number of paragraphs of plain text, with
+    both text wrap options and extra indentation. Use for plain text.
+
+    By default, uses the HTML and Markdown aware word splitter. This is probably
+    what you want, but you can also use the `simple_word_splitter` plaintext wrapping.
+    """
+
+    if not text_wrap.should_wrap:
+        indent = (
+            extra_indent + DEFAULT_INDENT
+            if text_wrap == Wrap.INDENT_ONLY
+            else extra_indent
+        )
+        lines = text.splitlines()
+        if lines:
+            return "\n".join(indent + line for line in lines)
+        else:
+            return empty_indent
+    else:
+        # Common settings for all wrap modes.
+        empty_indent = empty_indent.strip()
+        initial_indent = extra_indent + text_wrap.initial_indent
+        subsequent_indent = extra_indent + text_wrap.subsequent_indent
+
+        # These vary by wrap mode.
+        width = width - len_fn(subsequent_indent)
+        replace_whitespace = text_wrap.replace_whitespace
+
+        paragraphs = split_paragraphs(text)
+        wrapped_paragraphs = []
+
+        # Wrap each paragraph.
+        for i, paragraph in enumerate(paragraphs):
+            # Special case for hanging indent modes.
+            # Hang the first line of the first paragraph. All other paragraphs are indented.
+            if text_wrap.initial_indent_first_para_only and i > 0:
+                initial_indent = subsequent_indent
+
+            wrapped_paragraphs.append(
+                wrap_paragraph(
+                    paragraph,
+                    width=width,
+                    initial_indent=initial_indent,
+                    subsequent_indent=subsequent_indent,
+                    initial_column=initial_column,
+                    replace_whitespace=replace_whitespace,
+                    word_splitter=word_splitter,
+                    len_fn=len_fn,
+                )
+            )
+
+        para_sep = f"\n{empty_indent}\n"
+        return para_sep.join(wrapped_paragraphs)

flowmark-0.1.0/src/flowmark/text_wrapping.py

@@ -0,0 +1,155 @@
+import re
+from typing import Callable, List, Protocol, Tuple
+
+
+DEFAULT_LEN_FUNCTION = len
+"""
+Default length function to use for wrapping.
+By default this is just character length, but this can be overridden, for example
+to use a smarter function that does not count ANSI escape codes.
+"""
+
+
+class WordSplitter(Protocol):
+    def __call__(self, text: str) -> List[str]: ...
+
+
+def simple_word_splitter(text: str) -> List[str]:
+    """
+    Split words on whitespace. This is like Python's normal `textwrap`.
+    """
+    return text.split()
+
+
+class _HtmlMdWordSplitter:
+    def __init__(self):
+        # Sequences of whitespace-delimited words that should be coalesced and treated
+        # like a single word.
+        self.patterns = [
+            # HTML tags:
+            (r"<[^>]+", r"[^<>]+>[^<>]*"),
+            (r"<[^>]+", r"[^<>]+", r"[^<>]+>[^<>]*"),
+            # Markdown links:
+            (r"\[", r"[^\[\]]+\][^\[\]]*"),
+            (r"\[", r"[^\[\]]+", r"[^\[\]]+\][^\[\]]*"),
+        ]
+        self.compiled_patterns = [
+            tuple(re.compile(pattern) for pattern in pattern_group)
+            for pattern_group in self.patterns
+        ]
+
+    def __call__(self, text: str) -> List[str]:
+        words = text.split()
+        result = []
+        i = 0
+        while i < len(words):
+            coalesced = self.coalesce_words(words[i:])
+            if coalesced > 0:
+                result.append(" ".join(words[i : i + coalesced]))
+                i += coalesced
+            else:
+                result.append(words[i])
+                i += 1
+        return result
+
+    def coalesce_words(self, words: List[str]) -> int:
+        for pattern_group in self.compiled_patterns:
+            if self.match_pattern_group(words, pattern_group):
+                return len(pattern_group)
+        return 0
+
+    def match_pattern_group(self, words: List[str], patterns: Tuple[re.Pattern, ...]) -> bool:
+        if len(words) < len(patterns):
+            return False
+
+        return all(pattern.match(word) for pattern, word in zip(patterns, words))
+
+
+html_md_word_splitter: WordSplitter = _HtmlMdWordSplitter()
+"""
+Split words, but not within HTML tags or Markdown links.
+"""
+
+
+def wrap_paragraph_lines(
+    text: str,
+    width: int,
+    initial_column: int = 0,
+    subsequent_offset: int = 0,
+    replace_whitespace: bool = True,
+    drop_whitespace: bool = True,
+    splitter: WordSplitter = html_md_word_splitter,
+    len_fn: Callable[[str], int] = DEFAULT_LEN_FUNCTION,
+) -> List[str]:
+    """
+    Wrap a single paragraph of text, returning a list of wrapped lines.
+    Rewritten to simplify and generalize Python's textwrap.py.
+    """
+    if replace_whitespace:
+        text = re.sub(r"\s+", " ", text)
+
+    words = splitter(text)
+
+    lines: List[str] = []
+    current_line: List[str] = []
+    current_width = initial_column
+
+    # Walk through words, breaking them into lines.
+    for word in words:
+        word_width = len_fn(word)
+
+        space_width = 1 if current_line else 0
+        if current_width + word_width + space_width <= width:
+            # Add word to current line.
+            current_line.append(word)
+            current_width += word_width + space_width
+        else:
+            # Start a new line.
+            if current_line:
+                line = " ".join(current_line)
+                if drop_whitespace:
+                    line = line.strip()
+                lines.append(line)
+            current_line = [word]
+            current_width = subsequent_offset + word_width
+
+    if current_line:
+        line = " ".join(current_line)
+        if drop_whitespace:
+            line = line.strip()
+        lines.append(line)
+
+    return lines
+
+
+def wrap_paragraph(
+    text: str,
+    width: int,
+    initial_indent: str = "",
+    subsequent_indent: str = "",
+    initial_column: int = 0,
+    replace_whitespace: bool = True,
+    drop_whitespace: bool = True,
+    word_splitter: WordSplitter = html_md_word_splitter,
+    len_fn: Callable[[str], int] = DEFAULT_LEN_FUNCTION,
+) -> str:
+    """
+    Wrap lines of a single paragraph of plain text, returning a new string.
+    By default, uses an HTML- and Markdown-aware word splitter.
+    """
+    lines = wrap_paragraph_lines(
+        text=text,
+        width=width,
+        replace_whitespace=replace_whitespace,
+        drop_whitespace=drop_whitespace,
+        splitter=word_splitter,
+        initial_column=initial_column + len_fn(initial_indent),
+        subsequent_offset=len_fn(subsequent_indent),
+        len_fn=len_fn,
+    )
+    # Now insert indents on first and subsequent lines, if needed.
+    if initial_indent and initial_column == 0 and len(lines) > 0:
+        lines[0] = initial_indent + lines[0]
+    if subsequent_indent and len(lines) > 1:
+        lines[1:] = [subsequent_indent + line for line in lines[1:]]
+    return "\n".join(lines)