MarkdownHeaderTextSplitter 0.1.0__tar.gz

markdownheadertextsplitter-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Wenxi Wang
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

markdownheadertextsplitter-0.1.0/MarkdownHeaderTextSplitter.egg-info/PKG-INFO

@@ -0,0 +1,74 @@
+Metadata-Version: 2.4
+Name: MarkdownHeaderTextSplitter
+Version: 0.1.0
+Summary: A heading-aware Markdown splitter with a simple local UI
+Author: Wenxi Wang
+License: MIT
+Project-URL: Homepage, https://example.com/MarkdownHeaderTextSplitter
+Project-URL: Repository, https://example.com/MarkdownHeaderTextSplitter
+Keywords: markdown,splitter,rag,chunking,ui
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Text Processing :: Markup :: Markdown
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: streamlit>=1.32
+Dynamic: license-file
+
+# MarkdownHeaderTextSplitter
+
+A small, original, heading-aware Markdown splitter with a simple local UI for preparing Markdown files for RAG or inspection.
+
+## What it does
+
+- Accepts a `.md` file in a local UI
+- Respects Markdown heading structure like `#`, `##`, and `###`
+- Lets you choose how many chunks you want
+- Outputs the split Markdown chunks as separate `.md` files inside a downloadable `.zip`
+- Keeps the implementation lightweight and original instead of copying third-party splitter code
+
+## Install
+
+```bash
+pip install MarkdownHeaderTextSplitter
+```
+
+## Run
+
+```bash
+MarkdownHeaderTextSplitter
+```
+
+This launches a local Streamlit app in your browser.
+
+## Notes
+
+- This package does **not** vendor or copy code from LangChain or other companies.
+- The idea of heading-aware splitting is common, but you should still do your own name, trademark, licensing, and patent review before publishing publicly.
+- PyPI package-name availability can change over time, so confirm the final project name before upload.
+
+## File structure
+
+```text
+MarkdownHeaderTextSplitter_pypi/
+  README.md
+  LICENSE
+  pyproject.toml
+  markdownheadertextsplitter/
+    __init__.py
+    app.py
+    cli.py
+    core.py
+  dist/
+    MarkdownHeaderTextSplitter-0.1.0.tar.gz
+    markdownheadertextsplitter-0.1.0-py3-none-any.whl
+```

markdownheadertextsplitter-0.1.0/MarkdownHeaderTextSplitter.egg-info/SOURCES.txt

@@ -0,0 +1,13 @@
+LICENSE
+README.md
+pyproject.toml
+MarkdownHeaderTextSplitter.egg-info/PKG-INFO
+MarkdownHeaderTextSplitter.egg-info/SOURCES.txt
+MarkdownHeaderTextSplitter.egg-info/dependency_links.txt
+MarkdownHeaderTextSplitter.egg-info/entry_points.txt
+MarkdownHeaderTextSplitter.egg-info/requires.txt
+MarkdownHeaderTextSplitter.egg-info/top_level.txt
+markdownheadertextsplitter/__init__.py
+markdownheadertextsplitter/app.py
+markdownheadertextsplitter/cli.py
+markdownheadertextsplitter/core.py

markdownheadertextsplitter-0.1.0/MarkdownHeaderTextSplitter.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

markdownheadertextsplitter-0.1.0/MarkdownHeaderTextSplitter.egg-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+MarkdownHeaderTextSplitter = markdownheadertextsplitter.cli:main

markdownheadertextsplitter-0.1.0/MarkdownHeaderTextSplitter.egg-info/requires.txt

@@ -0,0 +1 @@
+streamlit>=1.32

markdownheadertextsplitter-0.1.0/MarkdownHeaderTextSplitter.egg-info/top_level.txt

@@ -0,0 +1 @@
+markdownheadertextsplitter

markdownheadertextsplitter-0.1.0/PKG-INFO

@@ -0,0 +1,74 @@
+Metadata-Version: 2.4
+Name: MarkdownHeaderTextSplitter
+Version: 0.1.0
+Summary: A heading-aware Markdown splitter with a simple local UI
+Author: Wenxi Wang
+License: MIT
+Project-URL: Homepage, https://example.com/MarkdownHeaderTextSplitter
+Project-URL: Repository, https://example.com/MarkdownHeaderTextSplitter
+Keywords: markdown,splitter,rag,chunking,ui
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Text Processing :: Markup :: Markdown
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: streamlit>=1.32
+Dynamic: license-file
+
+# MarkdownHeaderTextSplitter
+
+A small, original, heading-aware Markdown splitter with a simple local UI for preparing Markdown files for RAG or inspection.
+
+## What it does
+
+- Accepts a `.md` file in a local UI
+- Respects Markdown heading structure like `#`, `##`, and `###`
+- Lets you choose how many chunks you want
+- Outputs the split Markdown chunks as separate `.md` files inside a downloadable `.zip`
+- Keeps the implementation lightweight and original instead of copying third-party splitter code
+
+## Install
+
+```bash
+pip install MarkdownHeaderTextSplitter
+```
+
+## Run
+
+```bash
+MarkdownHeaderTextSplitter
+```
+
+This launches a local Streamlit app in your browser.
+
+## Notes
+
+- This package does **not** vendor or copy code from LangChain or other companies.
+- The idea of heading-aware splitting is common, but you should still do your own name, trademark, licensing, and patent review before publishing publicly.
+- PyPI package-name availability can change over time, so confirm the final project name before upload.
+
+## File structure
+
+```text
+MarkdownHeaderTextSplitter_pypi/
+  README.md
+  LICENSE
+  pyproject.toml
+  markdownheadertextsplitter/
+    __init__.py
+    app.py
+    cli.py
+    core.py
+  dist/
+    MarkdownHeaderTextSplitter-0.1.0.tar.gz
+    markdownheadertextsplitter-0.1.0-py3-none-any.whl
+```

markdownheadertextsplitter-0.1.0/README.md

@@ -0,0 +1,48 @@
+# MarkdownHeaderTextSplitter
+
+A small, original, heading-aware Markdown splitter with a simple local UI for preparing Markdown files for RAG or inspection.
+
+## What it does
+
+- Accepts a `.md` file in a local UI
+- Respects Markdown heading structure like `#`, `##`, and `###`
+- Lets you choose how many chunks you want
+- Outputs the split Markdown chunks as separate `.md` files inside a downloadable `.zip`
+- Keeps the implementation lightweight and original instead of copying third-party splitter code
+
+## Install
+
+```bash
+pip install MarkdownHeaderTextSplitter
+```
+
+## Run
+
+```bash
+MarkdownHeaderTextSplitter
+```
+
+This launches a local Streamlit app in your browser.
+
+## Notes
+
+- This package does **not** vendor or copy code from LangChain or other companies.
+- The idea of heading-aware splitting is common, but you should still do your own name, trademark, licensing, and patent review before publishing publicly.
+- PyPI package-name availability can change over time, so confirm the final project name before upload.
+
+## File structure
+
+```text
+MarkdownHeaderTextSplitter_pypi/
+  README.md
+  LICENSE
+  pyproject.toml
+  markdownheadertextsplitter/
+    __init__.py
+    app.py
+    cli.py
+    core.py
+  dist/
+    MarkdownHeaderTextSplitter-0.1.0.tar.gz
+    markdownheadertextsplitter-0.1.0-py3-none-any.whl
+```

markdownheadertextsplitter-0.1.0/markdownheadertextsplitter/__init__.py

@@ -0,0 +1,17 @@
+"""MarkdownHeaderTextSplitter package."""
+
+__all__ = [
+    "parse_markdown_sections",
+    "split_markdown_into_chunks",
+    "chunk_manifest",
+    "build_zip_bytes",
+]
+
+__version__ = "0.1.0"
+
+from .core import (  # noqa: E402
+    build_zip_bytes,
+    chunk_manifest,
+    parse_markdown_sections,
+    split_markdown_into_chunks,
+)

markdownheadertextsplitter-0.1.0/markdownheadertextsplitter/app.py

@@ -0,0 +1,68 @@
+from __future__ import annotations
+
+import streamlit as st
+
+from markdownheadertextsplitter.core import build_zip_bytes, chunk_manifest, split_markdown_into_chunks
+
+
+st.set_page_config(
+    page_title="MarkdownHeaderTextSplitter",
+    page_icon="🧩",
+    layout="wide",
+)
+
+st.title("MarkdownHeaderTextSplitter")
+st.caption("Split Markdown into smaller heading-aware chunks for RAG, review, or preprocessing.")
+
+with st.sidebar:
+    st.header("Settings")
+    target_chunks = st.slider("Target number of chunks", min_value=1, max_value=40, value=6)
+    max_heading_level = st.select_slider("Respect headings through level", options=[1, 2, 3, 4, 5, 6], value=3)
+    include_preamble = st.checkbox("Keep preamble before first heading", value=True)
+    hard_max_chars = st.number_input(
+        "Safety cap: max characters per chunk (0 = disabled)",
+        min_value=0,
+        max_value=50000,
+        value=0,
+        step=100,
+    )
+
+uploaded = st.file_uploader("Drag a Markdown file here", type=["md"])
+
+if uploaded is None:
+    st.info("Upload a .md file to preview and export split chunks.")
+    st.stop()
+
+raw_bytes = uploaded.read()
+text = raw_bytes.decode("utf-8", errors="ignore")
+
+chunks = split_markdown_into_chunks(
+    text,
+    target_chunks=target_chunks,
+    max_heading_level=max_heading_level,
+    include_preamble=include_preamble,
+    hard_max_chars=hard_max_chars,
+)
+manifest = chunk_manifest(chunks, uploaded.name)
+zip_bytes = build_zip_bytes(chunks, uploaded.name)
+
+col1, col2 = st.columns([1, 2])
+with col1:
+    st.metric("Output chunks", len(chunks))
+    st.metric("Input characters", len(text))
+    st.metric("Avg chunk characters", int(sum(len(c) for c in chunks) / max(1, len(chunks))))
+    st.download_button(
+        label="Download split .zip",
+        data=zip_bytes,
+        file_name=f"{uploaded.name.rsplit('.', 1)[0]}_chunks.zip",
+        mime="application/zip",
+    )
+
+with col2:
+    st.subheader("Chunk manifest")
+    st.dataframe(manifest, use_container_width=True)
+
+st.subheader("Chunk preview")
+for item, chunk in zip(manifest, chunks):
+    with st.expander(f"{item['file_name']} — {item['char_count']} chars"):
+        st.code(chunk, language="markdown")

markdownheadertextsplitter-0.1.0/markdownheadertextsplitter/cli.py

@@ -0,0 +1,17 @@
+from __future__ import annotations
+
+from pathlib import Path
+import sys
+
+
+def main() -> None:
+    try:
+        from streamlit.web import cli as stcli
+    except Exception as exc:  # pragma: no cover
+        raise SystemExit(
+            "Streamlit is required to launch the UI. Install the package dependencies first."
+        ) from exc
+
+    app_path = Path(__file__).with_name("app.py")
+    sys.argv = ["streamlit", "run", str(app_path)]
+    raise SystemExit(stcli.main())

markdownheadertextsplitter-0.1.0/markdownheadertextsplitter/core.py

@@ -0,0 +1,258 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from io import BytesIO
+import json
+import math
+import re
+import zipfile
+from pathlib import Path
+from typing import Iterable, List
+
+HEADING_RE = re.compile(r"^\\?(#{1,6})\s+(.*\S)\s*$")
+
+
+@dataclass
+class Section:
+    heading_level: int
+    heading_text: str
+    content: str
+
+    @property
+    def char_count(self) -> int:
+        return len(self.content)
+
+
+def _normalize_newlines(text: str) -> str:
+    return text.replace("\r\n", "\n").replace("\r", "\n")
+
+
+def parse_markdown_sections(
+    text: str,
+    *,
+    max_heading_level: int = 3,
+    include_preamble: bool = True,
+) -> List[Section]:
+    """Split a markdown document into ordered sections based on headings.
+
+    The implementation is intentionally simple and original:
+    - Headings up to ``max_heading_level`` start a new section.
+    - The heading line is kept inside the section content.
+    - Optional preamble text before the first heading becomes its own section.
+    """
+    text = _normalize_newlines(text).strip("\n")
+    if not text:
+        return []
+
+    sections: List[Section] = []
+    current_lines: List[str] = []
+    current_level = 0
+    current_title = "Preamble"
+    saw_heading = False
+
+    for line in text.split("\n"):
+        match = HEADING_RE.match(line)
+        if match:
+            level = len(match.group(1))
+            title = match.group(2).strip()
+            if level <= max_heading_level:
+                if current_lines and (include_preamble or saw_heading):
+                    sections.append(
+                        Section(
+                            heading_level=current_level,
+                            heading_text=current_title,
+                            content="\n".join(current_lines).strip() + "\n",
+                        )
+                    )
+                current_lines = [line]
+                current_level = level
+                current_title = title
+                saw_heading = True
+                continue
+        current_lines.append(line)
+
+    if current_lines and (include_preamble or saw_heading):
+        sections.append(
+            Section(
+                heading_level=current_level,
+                heading_text=current_title,
+                content="\n".join(current_lines).strip() + "\n",
+            )
+        )
+
+    if not sections:
+        return [Section(heading_level=0, heading_text="Document", content=text + "\n")]
+    return sections
+
+
+def _split_text_by_paragraphs(text: str) -> List[str]:
+    parts = [part.strip() for part in re.split(r"\n\s*\n", text) if part.strip()]
+    return [part + "\n\n" for part in parts] or [text]
+
+
+def _split_text_by_size(text: str, target_parts: int) -> List[str]:
+    if target_parts <= 1 or len(text) < 2:
+        return [text]
+    paragraphs = _split_text_by_paragraphs(text)
+    if len(paragraphs) >= target_parts:
+        total_chars = sum(len(p) for p in paragraphs)
+        desired = max(1, math.ceil(total_chars / target_parts))
+        out: List[str] = []
+        bucket: List[str] = []
+        bucket_size = 0
+        remaining_parts = target_parts
+        remaining_paragraphs = paragraphs[:]
+
+        for para in remaining_paragraphs:
+            remaining_paragraphs_left = len(remaining_paragraphs)
+            remaining_paragraphs.pop(0)
+            bucket.append(para)
+            bucket_size += len(para)
+
+            must_close = len(out) + 1 == target_parts
+            enough_size = bucket_size >= desired
+            enough_remaining = len(remaining_paragraphs) >= (remaining_parts - 1)
+
+            if must_close or (enough_size and enough_remaining):
+                out.append("".join(bucket).strip() + "\n")
+                bucket = []
+                bucket_size = 0
+                remaining_parts -= 1
+
+        if bucket:
+            out.append("".join(bucket).strip() + "\n")
+        return [chunk for chunk in out if chunk.strip()]
+
+    # Fallback: raw char slicing if there are not enough paragraphs.
+    text = text.strip()
+    step = max(1, math.ceil(len(text) / target_parts))
+    out = []
+    start = 0
+    while start < len(text):
+        out.append(text[start : start + step].strip() + "\n")
+        start += step
+    return [chunk for chunk in out if chunk.strip()]
+
+
+def _merge_sections_to_target(sections: List[Section], target_chunks: int) -> List[str]:
+    if target_chunks <= 1 or len(sections) <= 1:
+        return ["\n".join(section.content.strip() for section in sections if section.content.strip()) + "\n"]
+
+    total_chars = sum(section.char_count for section in sections)
+    desired = max(1, math.ceil(total_chars / target_chunks))
+
+    chunks: List[str] = []
+    bucket: List[str] = []
+    bucket_chars = 0
+    remaining_sections = sections[:]
+    remaining_buckets = target_chunks
+
+    while remaining_sections:
+        section = remaining_sections.pop(0)
+        bucket.append(section.content.strip())
+        bucket_chars += section.char_count
+
+        must_close = len(chunks) + 1 == target_chunks
+        enough_size = bucket_chars >= desired
+        enough_remaining = len(remaining_sections) >= (remaining_buckets - 1)
+
+        if must_close or (enough_size and enough_remaining):
+            chunks.append("\n\n".join(bucket).strip() + "\n")
+            bucket = []
+            bucket_chars = 0
+            remaining_buckets -= 1
+
+    if bucket:
+        chunks.append("\n\n".join(bucket).strip() + "\n")
+
+    return [chunk for chunk in chunks if chunk.strip()]
+
+
+def _expand_sections_to_target(sections: List[Section], target_chunks: int) -> List[str]:
+    chunks = [section.content.strip() + "\n" for section in sections if section.content.strip()]
+    if not chunks:
+        return []
+
+    while len(chunks) < target_chunks:
+        idx = max(range(len(chunks)), key=lambda i: len(chunks[i]))
+        biggest = chunks[idx]
+        if len(biggest) < 400:
+            break
+        split_parts = _split_text_by_size(biggest, 2)
+        if len(split_parts) <= 1:
+            break
+        chunks = chunks[:idx] + split_parts + chunks[idx + 1 :]
+
+    return chunks
+
+
+def _apply_hard_max_chars(chunks: Iterable[str], hard_max_chars: int) -> List[str]:
+    out: List[str] = []
+    for chunk in chunks:
+        if hard_max_chars <= 0 or len(chunk) <= hard_max_chars:
+            out.append(chunk.strip() + "\n")
+            continue
+        parts = _split_text_by_size(chunk, max(2, math.ceil(len(chunk) / hard_max_chars)))
+        out.extend(part.strip() + "\n" for part in parts if part.strip())
+    return out
+
+
+def split_markdown_into_chunks(
+    text: str,
+    *,
+    target_chunks: int = 6,
+    max_heading_level: int = 3,
+    include_preamble: bool = True,
+    hard_max_chars: int = 0,
+) -> List[str]:
+    sections = parse_markdown_sections(
+        text,
+        max_heading_level=max_heading_level,
+        include_preamble=include_preamble,
+    )
+    if not sections:
+        return []
+
+    if target_chunks <= 1:
+        chunks = ["\n\n".join(section.content.strip() for section in sections).strip() + "\n"]
+    elif len(sections) == target_chunks:
+        chunks = [section.content.strip() + "\n" for section in sections]
+    elif len(sections) > target_chunks:
+        chunks = _merge_sections_to_target(sections, target_chunks)
+    else:
+        chunks = _expand_sections_to_target(sections, target_chunks)
+
+    chunks = _apply_hard_max_chars(chunks, hard_max_chars)
+    return [chunk for chunk in chunks if chunk.strip()]
+
+
+def chunk_manifest(chunks: Iterable[str], source_name: str) -> list[dict]:
+    manifest = []
+    for index, chunk in enumerate(chunks, start=1):
+        line_count = chunk.count("\n") + 1
+        manifest.append(
+            {
+                "chunk_index": index,
+                "file_name": f"{Path(source_name).stem}_chunk_{index:03d}.md",
+                "char_count": len(chunk),
+                "line_count": line_count,
+                "preview": chunk[:180].replace("\n", " ").strip(),
+            }
+        )
+    return manifest
+
+
+def build_zip_bytes(chunks: Iterable[str], source_name: str) -> bytes:
+    chunks = list(chunks)
+    manifest = chunk_manifest(chunks, source_name)
+    stem = Path(source_name).stem or "markdown"
+
+    buffer = BytesIO()
+    with zipfile.ZipFile(buffer, mode="w", compression=zipfile.ZIP_DEFLATED) as zf:
+        for item, chunk in zip(manifest, chunks):
+            zf.writestr(item["file_name"], chunk)
+        zf.writestr(
+            f"{stem}_manifest.json",
+            json.dumps(manifest, indent=2, ensure_ascii=False),
+        )
+    return buffer.getvalue()

markdownheadertextsplitter-0.1.0/pyproject.toml

@@ -0,0 +1,36 @@
+[build-system]
+requires = ["setuptools>=61.0"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "MarkdownHeaderTextSplitter"
+version = "0.1.0"
+description = "A heading-aware Markdown splitter with a simple local UI"
+authors = [{name="Wenxi Wang"}]
+readme = "README.md"
+requires-python = ">=3.9"
+license = {text = "MIT"}
+dependencies = [
+    "streamlit>=1.32",
+]
+keywords = ["markdown", "splitter", "rag", "chunking", "ui"]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: Text Processing :: Markup :: Markdown",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+]
+
+[project.urls]
+Homepage = "https://example.com/MarkdownHeaderTextSplitter"
+Repository = "https://example.com/MarkdownHeaderTextSplitter"
+
+[project.scripts]
+MarkdownHeaderTextSplitter = "markdownheadertextsplitter.cli:main"

markdownheadertextsplitter-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+