split-markdown4gpt 1.0.8__tar.gz → 2.7.7__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. {split_markdown4gpt-1.0.8 → split_markdown4gpt-2.7.7}/.gitignore +1 -0
  2. split_markdown4gpt-2.7.7/AUTHORS.md +4 -0
  3. split_markdown4gpt-2.7.7/CHANGELOG.md +36 -0
  4. split_markdown4gpt-2.7.7/PKG-INFO +143 -0
  5. split_markdown4gpt-2.7.7/README.md +102 -0
  6. split_markdown4gpt-2.7.7/docs/api.md +13 -0
  7. split_markdown4gpt-2.7.7/docs/assets/icon.png +0 -0
  8. split_markdown4gpt-2.7.7/docs/changelog.md +3 -0
  9. split_markdown4gpt-2.7.7/docs/index.md +44 -0
  10. split_markdown4gpt-2.7.7/docs/usage.md +119 -0
  11. split_markdown4gpt-2.7.7/docs/why-splitting.md +78 -0
  12. split_markdown4gpt-2.7.7/pyproject.toml +131 -0
  13. split_markdown4gpt-2.7.7/src/split_markdown4gpt/__init__.py +30 -0
  14. {split_markdown4gpt-1.0.8 → split_markdown4gpt-2.7.7}/src/split_markdown4gpt/__main__.py +2 -2
  15. split_markdown4gpt-2.7.7/src/split_markdown4gpt/_version.py +24 -0
  16. split_markdown4gpt-2.7.7/src/split_markdown4gpt/splitter.py +486 -0
  17. split_markdown4gpt-2.7.7/tests/conftest.py +10 -0
  18. split_markdown4gpt-2.7.7/tests/test.md +31 -0
  19. split_markdown4gpt-2.7.7/tests/test_splitter.py +353 -0
  20. split_markdown4gpt-1.0.8/.DS_Store +0 -0
  21. split_markdown4gpt-1.0.8/.coveragerc +0 -28
  22. split_markdown4gpt-1.0.8/.github/workflows/ci.yml +0 -80
  23. split_markdown4gpt-1.0.8/.isort.cfg +0 -3
  24. split_markdown4gpt-1.0.8/.pre-commit-config.yaml +0 -67
  25. split_markdown4gpt-1.0.8/API.md +0 -329
  26. split_markdown4gpt-1.0.8/AUTHORS.md +0 -3
  27. split_markdown4gpt-1.0.8/PKG-INFO +0 -144
  28. split_markdown4gpt-1.0.8/README.md +0 -117
  29. split_markdown4gpt-1.0.8/pydoc-markdown.yml +0 -6
  30. split_markdown4gpt-1.0.8/pyproject.toml +0 -9
  31. split_markdown4gpt-1.0.8/setup.cfg +0 -94
  32. split_markdown4gpt-1.0.8/setup.py +0 -21
  33. split_markdown4gpt-1.0.8/src/split_markdown4gpt/__init__.py +0 -18
  34. split_markdown4gpt-1.0.8/src/split_markdown4gpt/splitter.py +0 -392
  35. split_markdown4gpt-1.0.8/src/split_markdown4gpt.egg-info/PKG-INFO +0 -144
  36. split_markdown4gpt-1.0.8/src/split_markdown4gpt.egg-info/SOURCES.txt +0 -28
  37. split_markdown4gpt-1.0.8/src/split_markdown4gpt.egg-info/dependency_links.txt +0 -1
  38. split_markdown4gpt-1.0.8/src/split_markdown4gpt.egg-info/entry_points.txt +0 -2
  39. split_markdown4gpt-1.0.8/src/split_markdown4gpt.egg-info/not-zip-safe +0 -1
  40. split_markdown4gpt-1.0.8/src/split_markdown4gpt.egg-info/requires.txt +0 -15
  41. split_markdown4gpt-1.0.8/src/split_markdown4gpt.egg-info/top_level.txt +0 -1
  42. split_markdown4gpt-1.0.8/tests/conftest.py +0 -10
  43. split_markdown4gpt-1.0.8/tests/test.md +0 -34
  44. split_markdown4gpt-1.0.8/tests/test_splitter.py +0 -11
  45. split_markdown4gpt-1.0.8/tox.ini +0 -93
  46. {split_markdown4gpt-1.0.8 → split_markdown4gpt-2.7.7}/LICENSE.txt +0 -0
@@ -52,3 +52,4 @@ MANIFEST
52
52
  .venv*/
53
53
  .conda*/
54
54
  .python-version
55
+ src/split_markdown4gpt/_version.py
@@ -0,0 +1,4 @@
1
+ # Contributors
2
+
3
+ - Adam Twardoch <adam+github@twardoch.com>
4
+
@@ -0,0 +1,36 @@
1
+ # Changelog
2
+
3
+ All notable changes to `split-markdown4gpt` are documented here.
4
+
5
+ ## [Unreleased]
6
+
7
+ ### Changed
8
+ - **Build system**: migrated from `setup.cfg` / `setup.py` + `setuptools_scm` to a
9
+ fully self-contained `pyproject.toml` using `hatchling` + `hatch-vcs`.
10
+ - **Linting**: replaced `black` / `isort` / `flake8` with `ruff` (configured in
11
+ `pyproject.toml`).
12
+ - **CI**: updated GitHub Actions from `actions/checkout@v2` / `setup-python@v2` to
13
+ `@v4` / `@v5`; added Python 3.11 and 3.12 to the test matrix; switched from
14
+ plain `pip install` to `hatch run test`.
15
+ - **Type hints**: modernised from `typing.List / Dict / Tuple` to built-in generics
16
+ (`list`, `dict`, `tuple`); added `from __future__ import annotations` throughout.
17
+ - **`meta_data()`**: fixed silent bug in the `except` clause (`meta, data = None`
18
+ would raise `TypeError`; corrected to `meta, data = None, md`).
19
+ - **`OPENAI_MODELS`**: added `gpt-4-turbo`, `gpt-4o`, and `gpt-4o-mini` entries.
20
+ - **`load_md_path()`**: added explicit `encoding="utf-8"` to `open()`.
21
+ - **`__init__.py`**: removed Python 3.7 compatibility shim; use
22
+ `importlib.metadata` directly; added `__all__`.
23
+
24
+ ### Added
25
+ - MkDocs Material documentation site (`docs/`, `mkdocs.yml`) with:
26
+ - *Why Splitting Matters* page explaining context-window constraints.
27
+ - *Usage* page with CLI and Python examples.
28
+ - *API Reference* page generated from docstrings.
29
+ - Comprehensive test suite (`tests/test_splitter.py`) expanded from 1 test to
30
+ 34 tests covering: `meta_data()`, construction, all three load paths,
31
+ token counting, chunking behaviour, token-limit enforcement, front-matter
32
+ stripping, edge cases (empty input, only front matter, very long paragraph).
33
+
34
+ ## [2.7.5] — 2024
35
+
36
+ - Earlier releases — see git history.
@@ -0,0 +1,143 @@
1
+ Metadata-Version: 2.5
2
+ Name: split_markdown4gpt
3
+ Version: 2.7.7
4
+ Summary: Split large Markdown files into token-bounded chunks for LLM context windows
5
+ Project-URL: Homepage, https://github.com/twardoch/split-markdown4gpt
6
+ Project-URL: Documentation, https://twardoch.github.io/split-markdown4gpt/
7
+ Project-URL: Source, https://github.com/twardoch/split-markdown4gpt
8
+ Project-URL: Bug Tracker, https://github.com/twardoch/split-markdown4gpt/issues
9
+ Author-email: Adam Twardoch <adam+github@twardoch.com>
10
+ License: Apache-2.0
11
+ License-File: AUTHORS.md
12
+ License-File: LICENSE.txt
13
+ Keywords: chunking,context-window,data-preprocessing,gpt,gpt-3,gpt-4,llm,markdown,mistletoe,natural-language-processing,nlp,openai,openai-gpt,python,split-text,summarization,text-analysis,text-processing,text-summarization,text-tokenization
14
+ Classifier: Development Status :: 4 - Beta
15
+ Classifier: Intended Audience :: Developers
16
+ Classifier: Intended Audience :: Science/Research
17
+ Classifier: License :: OSI Approved :: Apache Software License
18
+ Classifier: Operating System :: OS Independent
19
+ Classifier: Programming Language :: Python
20
+ Classifier: Programming Language :: Python :: 3
21
+ Classifier: Programming Language :: Python :: 3.10
22
+ Classifier: Programming Language :: Python :: 3.11
23
+ Classifier: Programming Language :: Python :: 3.12
24
+ Classifier: Topic :: Text Processing
25
+ Classifier: Topic :: Text Processing :: Markup :: Markdown
26
+ Requires-Python: >=3.10
27
+ Requires-Dist: fire>=0.5.0
28
+ Requires-Dist: mistletoe>=1.1.0
29
+ Requires-Dist: python-frontmatter>=1.0.0
30
+ Requires-Dist: pyyaml>=5.1
31
+ Requires-Dist: regex>=2023.6.3
32
+ Requires-Dist: syntok>=1.4.4
33
+ Requires-Dist: tiktoken>=0.4.0
34
+ Provides-Extra: docs
35
+ Requires-Dist: mkdocs-material>=9.0; extra == 'docs'
36
+ Requires-Dist: mkdocstrings[python]>=0.20; extra == 'docs'
37
+ Provides-Extra: testing
38
+ Requires-Dist: pytest-cov>=4.0; extra == 'testing'
39
+ Requires-Dist: pytest>=7.0; extra == 'testing'
40
+ Description-Content-Type: text/markdown
41
+
42
+ # split-markdown4gpt: split large Markdown for LLM context windows
43
+
44
+ LLMs have finite context windows. A 50,000-word documentation site won't fit in one API call. `split-markdown4gpt` breaks Markdown files into token-bounded chunks that respect document structure — so each chunk stays coherent rather than cutting mid-sentence or mid-section.
45
+
46
+ ## Why splitting matters
47
+
48
+ Every LLM API call has a hard token limit (e.g. 4,096 tokens for GPT-3.5, 8,192 for GPT-4). Exceeding it raises an error. Naive character-based chunking cuts through sentences, code blocks, and headings — destroying context. This tool:
49
+
50
+ 1. Parses the Markdown AST (via `mistletoe`)
51
+ 2. Counts tokens using the same tokenizer the model uses (via `tiktoken`)
52
+ 3. Packs as many complete sections as fit within the limit
53
+ 4. When a single section exceeds the limit, falls back to sentence-level splitting (via `syntok`)
54
+
55
+ The result: chunks that respect heading hierarchy, never split a sentence across chunks, and are as large as possible without exceeding the token budget.
56
+
57
+ ## Install
58
+
59
+ ```bash
60
+ pip install split_markdown4gpt
61
+ ```
62
+
63
+ ## CLI
64
+
65
+ ```bash
66
+ mdsplit4gpt large_doc.md --model gpt-3.5-turbo --limit 4096 --separator "=== SPLIT ==="
67
+ ```
68
+
69
+ Outputs all sections joined by the separator. Redirect to a file or pipe to another tool.
70
+
71
+ | Flag | Default | Description |
72
+ |------|---------|-------------|
73
+ | `--model` | `gpt-3.5-turbo` | Tokenizer model (determines token counting) |
74
+ | `--limit` | model max | Max tokens per chunk |
75
+ | `--separator` | `=== SPLIT ===` | String inserted between chunks in output |
76
+
77
+ ## Python API
78
+
79
+ ```python
80
+ from split_markdown4gpt import split
81
+
82
+ # Returns a list of Markdown strings, each within the token limit
83
+ chunks = split("large_doc.md", model="gpt-3.5-turbo", limit=4096)
84
+
85
+ for i, chunk in enumerate(chunks):
86
+ response = openai_client.chat.completions.create(
87
+ model="gpt-3.5-turbo", messages=[{"role": "user", "content": chunk}]
88
+ )
89
+ ```
90
+
91
+ Also accepts a string or file-like object:
92
+
93
+ ```python
94
+ chunks = split(markdown_string, model="gpt-4", limit=8000)
95
+ chunks = split(open("doc.md"), limit=2000)
96
+ ```
97
+
98
+ ## How it works
99
+
100
+ ```
101
+ split(md)
102
+ ├── meta_data() strip YAML front matter
103
+ ├── MarkdownLLMSplitter.build_md_dict()
104
+ │ ├── mistletoe Document parse Markdown AST
105
+ │ └── build dict tree keyed by heading level
106
+ ├── calculate_sizes() count tokens recursively (cached with lru_cache)
107
+ └── get_sections_from_md_dict_by_limit()
108
+ ├── walk tree, accumulate blocks until limit reached
109
+ └── oversized blocks → sentence-split via syntok
110
+ ```
111
+
112
+ ### `MarkdownLLMSplitter` class
113
+
114
+ ```python
115
+ from split_markdown4gpt import MarkdownLLMSplitter
116
+
117
+ splitter = MarkdownLLMSplitter(gptok_model="gpt-4", gptok_limit=6000)
118
+ splitter.load_md("doc.md")
119
+ splitter.build()
120
+
121
+ for section in splitter.gen_section_dicts():
122
+ print(section["gptok_size"], "tokens")
123
+ print(section["md"])
124
+ ```
125
+
126
+ Methods: `load_md()`, `load_md_path()`, `load_md_str()`, `build()`, `list_section_texts()`, `list_section_dicts()`, `gen_section_texts()`, `gen_section_dicts()`, `split()`.
127
+
128
+ ## Supported models
129
+
130
+ Any model supported by `tiktoken`, including all `gpt-3.5-turbo` and `gpt-4` variants. The limit defaults to the model's published context window.
131
+
132
+ ## Dependencies
133
+
134
+ - `tiktoken` — OpenAI-compatible token counting
135
+ - `mistletoe` — Markdown AST parser
136
+ - `syntok` — sentence segmentation for oversized sections
137
+ - `frontmatter` — YAML front matter stripping
138
+ - `fire` — CLI
139
+ - `regex`, `PyYAML` — utilities
140
+
141
+ ## License
142
+
143
+ Apache License 2.0. Copyright © 2023 Adam Twardoch.
@@ -0,0 +1,102 @@
1
+ # split-markdown4gpt: split large Markdown for LLM context windows
2
+
3
+ LLMs have finite context windows. A 50,000-word documentation site won't fit in one API call. `split-markdown4gpt` breaks Markdown files into token-bounded chunks that respect document structure — so each chunk stays coherent rather than cutting mid-sentence or mid-section.
4
+
5
+ ## Why splitting matters
6
+
7
+ Every LLM API call has a hard token limit (e.g. 4,096 tokens for GPT-3.5, 8,192 for GPT-4). Exceeding it raises an error. Naive character-based chunking cuts through sentences, code blocks, and headings — destroying context. This tool:
8
+
9
+ 1. Parses the Markdown AST (via `mistletoe`)
10
+ 2. Counts tokens using the same tokenizer the model uses (via `tiktoken`)
11
+ 3. Packs as many complete sections as fit within the limit
12
+ 4. When a single section exceeds the limit, falls back to sentence-level splitting (via `syntok`)
13
+
14
+ The result: chunks that respect heading hierarchy, never split a sentence across chunks, and are as large as possible without exceeding the token budget.
15
+
16
+ ## Install
17
+
18
+ ```bash
19
+ pip install split_markdown4gpt
20
+ ```
21
+
22
+ ## CLI
23
+
24
+ ```bash
25
+ mdsplit4gpt large_doc.md --model gpt-3.5-turbo --limit 4096 --separator "=== SPLIT ==="
26
+ ```
27
+
28
+ Outputs all sections joined by the separator. Redirect to a file or pipe to another tool.
29
+
30
+ | Flag | Default | Description |
31
+ |------|---------|-------------|
32
+ | `--model` | `gpt-3.5-turbo` | Tokenizer model (determines token counting) |
33
+ | `--limit` | model max | Max tokens per chunk |
34
+ | `--separator` | `=== SPLIT ===` | String inserted between chunks in output |
35
+
36
+ ## Python API
37
+
38
+ ```python
39
+ from split_markdown4gpt import split
40
+
41
+ # Returns a list of Markdown strings, each within the token limit
42
+ chunks = split("large_doc.md", model="gpt-3.5-turbo", limit=4096)
43
+
44
+ for i, chunk in enumerate(chunks):
45
+ response = openai_client.chat.completions.create(
46
+ model="gpt-3.5-turbo", messages=[{"role": "user", "content": chunk}]
47
+ )
48
+ ```
49
+
50
+ Also accepts a string or file-like object:
51
+
52
+ ```python
53
+ chunks = split(markdown_string, model="gpt-4", limit=8000)
54
+ chunks = split(open("doc.md"), limit=2000)
55
+ ```
56
+
57
+ ## How it works
58
+
59
+ ```
60
+ split(md)
61
+ ├── meta_data() strip YAML front matter
62
+ ├── MarkdownLLMSplitter.build_md_dict()
63
+ │ ├── mistletoe Document parse Markdown AST
64
+ │ └── build dict tree keyed by heading level
65
+ ├── calculate_sizes() count tokens recursively (cached with lru_cache)
66
+ └── get_sections_from_md_dict_by_limit()
67
+ ├── walk tree, accumulate blocks until limit reached
68
+ └── oversized blocks → sentence-split via syntok
69
+ ```
70
+
71
+ ### `MarkdownLLMSplitter` class
72
+
73
+ ```python
74
+ from split_markdown4gpt import MarkdownLLMSplitter
75
+
76
+ splitter = MarkdownLLMSplitter(gptok_model="gpt-4", gptok_limit=6000)
77
+ splitter.load_md("doc.md")
78
+ splitter.build()
79
+
80
+ for section in splitter.gen_section_dicts():
81
+ print(section["gptok_size"], "tokens")
82
+ print(section["md"])
83
+ ```
84
+
85
+ Methods: `load_md()`, `load_md_path()`, `load_md_str()`, `build()`, `list_section_texts()`, `list_section_dicts()`, `gen_section_texts()`, `gen_section_dicts()`, `split()`.
86
+
87
+ ## Supported models
88
+
89
+ Any model supported by `tiktoken`, including all `gpt-3.5-turbo` and `gpt-4` variants. The limit defaults to the model's published context window.
90
+
91
+ ## Dependencies
92
+
93
+ - `tiktoken` — OpenAI-compatible token counting
94
+ - `mistletoe` — Markdown AST parser
95
+ - `syntok` — sentence segmentation for oversized sections
96
+ - `frontmatter` — YAML front matter stripping
97
+ - `fire` — CLI
98
+ - `regex`, `PyYAML` — utilities
99
+
100
+ ## License
101
+
102
+ Apache License 2.0. Copyright © 2023 Adam Twardoch.
@@ -0,0 +1,13 @@
1
+ # API Reference
2
+
3
+ ## Module: `split_markdown4gpt`
4
+
5
+ ::: split_markdown4gpt.splitter
6
+ options:
7
+ members:
8
+ - meta_data
9
+ - OPENAI_MODELS
10
+ - MarkdownLLMSplitter
11
+ - split
12
+ show_source: true
13
+ docstring_style: google
@@ -0,0 +1,3 @@
1
+ # Changelog
2
+
3
+ See [CHANGELOG.md](../CHANGELOG.md) in the repository root.
@@ -0,0 +1,44 @@
1
+ # split-markdown4gpt
2
+
3
+ **Split large Markdown files into token-bounded chunks for LLM context windows.**
4
+
5
+ LLMs have finite context windows. A 50,000-word documentation site will not fit in one API call. `split-markdown4gpt` breaks Markdown files into token-bounded chunks that respect document structure — each chunk stays coherent rather than cutting mid-sentence or mid-section.
6
+
7
+ ## Quick start
8
+
9
+ ```bash
10
+ pip install split_markdown4gpt
11
+ ```
12
+
13
+ ```python
14
+ from split_markdown4gpt import split
15
+
16
+ chunks = split("large_doc.md", model="gpt-4", limit=6000)
17
+ for chunk in chunks:
18
+ response = client.chat.completions.create(
19
+ model="gpt-4",
20
+ messages=[{"role": "user", "content": chunk}],
21
+ )
22
+ ```
23
+
24
+ Or from the command line:
25
+
26
+ ```bash
27
+ mdsplit4gpt large_doc.md --model gpt-3.5-turbo --limit 4096
28
+ ```
29
+
30
+ ## How it works
31
+
32
+ ```
33
+ split(md)
34
+ ├── meta_data() strip YAML front matter
35
+ ├── build_md_dict()
36
+ │ ├── mistletoe parse Markdown AST
37
+ │ └── heading tree keyed by level
38
+ ├── calculate_sizes() count tokens (tiktoken, cached)
39
+ └── get_sections_from_md_dict_by_limit()
40
+ ├── walk tree, accumulate blocks until limit reached
41
+ └── oversized blocks → sentence-split via syntok
42
+ ```
43
+
44
+ See [Why Splitting Matters](why-splitting.md) for the motivation, [Usage](usage.md) for the full API, and the [API Reference](api.md) for generated documentation.
@@ -0,0 +1,119 @@
1
+ # Usage
2
+
3
+ ## Installation
4
+
5
+ ```bash
6
+ pip install split_markdown4gpt
7
+ ```
8
+
9
+ ## CLI
10
+
11
+ ```bash
12
+ mdsplit4gpt large_doc.md --model gpt-3.5-turbo --limit 4096 --separator "=== SPLIT ==="
13
+ ```
14
+
15
+ Outputs all chunks joined by the separator. Redirect to a file or pipe to another tool.
16
+
17
+ | Flag | Default | Description |
18
+ |---|---|---|
19
+ | `--model` | `gpt-3.5-turbo` | Tiktoken model (determines token counting) |
20
+ | `--limit` | model max | Max tokens per chunk |
21
+ | `--separator` | `=== SPLIT ===` | String inserted between chunks in output |
22
+
23
+ ## Python — quick API
24
+
25
+ ```python
26
+ from split_markdown4gpt import split
27
+
28
+ # From a file path
29
+ chunks = split("large_doc.md", model="gpt-3.5-turbo", limit=4096)
30
+
31
+ # From a string
32
+ chunks = split(markdown_string, model="gpt-4", limit=8000)
33
+
34
+ # From a file object
35
+ with open("doc.md") as f:
36
+ chunks = split(f, limit=2000)
37
+
38
+ for chunk in chunks:
39
+ # Each chunk is a Markdown string within the token limit
40
+ print(len(chunk), "chars")
41
+ ```
42
+
43
+ ## Python — class API
44
+
45
+ Use `MarkdownLLMSplitter` directly for more control:
46
+
47
+ ```python
48
+ from split_markdown4gpt import MarkdownLLMSplitter
49
+
50
+ splitter = MarkdownLLMSplitter(gptok_model="gpt-4", gptok_limit=6000)
51
+
52
+ # Load content
53
+ splitter.load_md("doc.md") # from path
54
+ splitter.load_md_str(md_string) # from string
55
+ splitter.load_md_file(file_obj) # from file object
56
+
57
+ # Build sections
58
+ splitter.build()
59
+
60
+ # Access results
61
+ texts = splitter.list_section_texts() # list[str]
62
+ dicts = splitter.list_section_dicts() # list[{"md": str, "gptok_size": int}]
63
+
64
+ # Or use generators
65
+ for text in splitter.gen_section_texts():
66
+ print(text)
67
+
68
+ for section in splitter.gen_section_dicts():
69
+ print(section["gptok_size"], "tokens:", section["md"][:80])
70
+ ```
71
+
72
+ ## Supported models
73
+
74
+ Any model supported by `tiktoken`, including all `gpt-3.5-turbo` and `gpt-4` variants.
75
+
76
+ ```python
77
+ from split_markdown4gpt.splitter import OPENAI_MODELS
78
+
79
+ print(OPENAI_MODELS)
80
+ # {"gpt-4": 8192, "gpt-4-turbo": 128000, "gpt-4o": 128000, ...}
81
+ ```
82
+
83
+ ## Processing with OpenAI
84
+
85
+ ```python
86
+ from openai import OpenAI
87
+ from split_markdown4gpt import split
88
+
89
+ client = OpenAI()
90
+
91
+ chunks = split("large_doc.md", model="gpt-4", limit=6000)
92
+
93
+ summaries = []
94
+ for chunk in chunks:
95
+ response = client.chat.completions.create(
96
+ model="gpt-4",
97
+ messages=[
98
+ {
99
+ "role": "system",
100
+ "content": "Summarise the following documentation section.",
101
+ },
102
+ {"role": "user", "content": chunk},
103
+ ],
104
+ )
105
+ summaries.append(response.choices[0].message.content)
106
+
107
+ final_summary = "\n\n".join(summaries)
108
+ ```
109
+
110
+ ## YAML front matter
111
+
112
+ Front matter is automatically stripped before chunking. Metadata is available on the splitter instance:
113
+
114
+ ```python
115
+ splitter = MarkdownLLMSplitter()
116
+ splitter.load_md("doc.md")
117
+ print(splitter.md_meta) # {"title": "...", "author": "..."}
118
+ splitter.build()
119
+ ```
@@ -0,0 +1,78 @@
1
+ # Why Splitting Matters for LLM Context Windows
2
+
3
+ ## The context-window problem
4
+
5
+ Every LLM API call operates within a hard token limit — a ceiling on the combined length of the prompt and the generated response. Common limits are:
6
+
7
+ | Model | Context window |
8
+ |---|---|
9
+ | GPT-3.5 Turbo | 4,096 tokens |
10
+ | GPT-3.5 Turbo 16k | 16,384 tokens |
11
+ | GPT-4 | 8,192 tokens |
12
+ | GPT-4 Turbo / GPT-4o | 128,000 tokens |
13
+
14
+ A "token" is roughly 0.75 words in English, so 4,096 tokens is about 3,000 words — just under five pages of dense text. Even 128k tokens is only around 90,000 words; a single large codebase documentation site can easily exceed that.
15
+
16
+ Exceeding the limit raises an API error. The only remedy is to send less text per call.
17
+
18
+ ## Why naive splitting fails
19
+
20
+ The obvious fix — splitting on character count or word count — destroys document coherence:
21
+
22
+ - A sentence split across two chunks loses its meaning.
23
+ - A code block cut in the middle becomes invalid.
24
+ - A heading at the end of one chunk has no body; the body appears in the next chunk with no heading.
25
+ - A bullet list split mid-item confuses the model about list structure.
26
+
27
+ Models trained on coherent text produce incoherent output when fed incoherent input.
28
+
29
+ ## Structure-aware chunking
30
+
31
+ `split-markdown4gpt` parses the Markdown document as an abstract syntax tree (AST) using `mistletoe`, then works with complete structural units:
32
+
33
+ 1. **Headings** define section boundaries. The splitter respects the heading hierarchy (H1 → H2 → H3) so a chunk always begins at a logical section start.
34
+ 2. **Paragraphs, code blocks, lists, and tables** are treated as atomic units. They are never broken mid-block.
35
+ 3. **Token counting** uses `tiktoken` — the same tokenizer the OpenAI models use — so the count is exact rather than approximated.
36
+ 4. **Greedy packing** fills each chunk as full as possible without exceeding the limit, minimising the total number of API calls needed.
37
+ 5. **Sentence-level fallback** handles the edge case where a single paragraph exceeds the limit. `syntok` splits it into sentences and packs those instead.
38
+
39
+ ## Example
40
+
41
+ Given this document (simplified):
42
+
43
+ ```markdown
44
+ # Introduction
45
+
46
+ OpenAI's GPT models have changed how developers build applications...
47
+ [500 tokens]
48
+
49
+ ## Background
50
+
51
+ The transformer architecture, introduced in 2017...
52
+ [800 tokens]
53
+
54
+ ## Key capabilities
55
+
56
+ GPT-4 can reason, code, summarise, and translate...
57
+ [600 tokens]
58
+
59
+ # Implementation guide
60
+
61
+ To use the API in Python, first install the SDK...
62
+ [3000 tokens]
63
+ ```
64
+
65
+ With a 4,096-token limit, naive character splitting might cut the Implementation guide mid-sentence. `split-markdown4gpt` instead produces:
66
+
67
+ **Chunk 1** (1,900 tokens): Introduction + Background + Key capabilities — all three sections fit together.
68
+
69
+ **Chunk 2** (3,000 tokens): Implementation guide — the entire section in one piece.
70
+
71
+ Each chunk is coherent, self-contained, and well within the limit.
72
+
73
+ ## Practical impact
74
+
75
+ - **Summarisation pipelines**: Each chunk can be summarised independently; summaries can then be re-combined.
76
+ - **Question answering over long docs**: Send chunks sequentially; keep only chunks where the model finds relevant content.
77
+ - **Fine-tuning data preparation**: Produce training examples that are each within the context window.
78
+ - **RAG (retrieval-augmented generation)**: Chunks map naturally to vector-store documents; structure-aware chunking produces more semantically coherent embeddings than character-split chunks.
@@ -0,0 +1,131 @@
1
+ [build-system]
2
+ requires = ["hatchling", "hatch-vcs"]
3
+ build-backend = "hatchling.build"
4
+
5
+ [project]
6
+ name = "split_markdown4gpt"
7
+ description = "Split large Markdown files into token-bounded chunks for LLM context windows"
8
+ readme = "README.md"
9
+ license = { text = "Apache-2.0" }
10
+ authors = [{ name = "Adam Twardoch", email = "adam+github@twardoch.com" }]
11
+ keywords = [
12
+ "python",
13
+ "nlp",
14
+ "markdown",
15
+ "natural-language-processing",
16
+ "text-analysis",
17
+ "openai",
18
+ "text-summarization",
19
+ "summarization",
20
+ "text-processing",
21
+ "gpt",
22
+ "data-preprocessing",
23
+ "mistletoe",
24
+ "split-text",
25
+ "text-tokenization",
26
+ "openai-gpt",
27
+ "gpt-3",
28
+ "gpt-4",
29
+ "llm",
30
+ "context-window",
31
+ "chunking",
32
+ ]
33
+ classifiers = [
34
+ "Development Status :: 4 - Beta",
35
+ "Programming Language :: Python",
36
+ "Programming Language :: Python :: 3",
37
+ "Programming Language :: Python :: 3.10",
38
+ "Programming Language :: Python :: 3.11",
39
+ "Programming Language :: Python :: 3.12",
40
+ "Intended Audience :: Developers",
41
+ "Intended Audience :: Science/Research",
42
+ "Topic :: Text Processing",
43
+ "Topic :: Text Processing :: Markup :: Markdown",
44
+ "License :: OSI Approved :: Apache Software License",
45
+ "Operating System :: OS Independent",
46
+ ]
47
+ requires-python = ">=3.10"
48
+ dependencies = [
49
+ "fire>=0.5.0",
50
+ "mistletoe>=1.1.0",
51
+ "python-frontmatter>=1.0.0",
52
+ "PyYAML>=5.1",
53
+ "regex>=2023.6.3",
54
+ "syntok>=1.4.4",
55
+ "tiktoken>=0.4.0",
56
+ ]
57
+ dynamic = ["version"]
58
+
59
+ [project.urls]
60
+ Homepage = "https://github.com/twardoch/split-markdown4gpt"
61
+ Documentation = "https://twardoch.github.io/split-markdown4gpt/"
62
+ Source = "https://github.com/twardoch/split-markdown4gpt"
63
+ "Bug Tracker" = "https://github.com/twardoch/split-markdown4gpt/issues"
64
+
65
+ [project.optional-dependencies]
66
+ testing = [
67
+ "pytest>=7.0",
68
+ "pytest-cov>=4.0",
69
+ ]
70
+ docs = [
71
+ "mkdocs-material>=9.0",
72
+ "mkdocstrings[python]>=0.20",
73
+ ]
74
+
75
+ [project.scripts]
76
+ mdsplit4gpt = "split_markdown4gpt.__main__:cli"
77
+
78
+ [tool.hatch.version]
79
+ source = "vcs"
80
+
81
+ [tool.hatch.build.hooks.vcs]
82
+ version-file = "src/split_markdown4gpt/_version.py"
83
+
84
+ [tool.hatch.build.targets.wheel]
85
+ packages = ["src/split_markdown4gpt"]
86
+
87
+ [tool.hatch.build.targets.sdist]
88
+ include = [
89
+ "/src",
90
+ "/tests",
91
+ "/docs",
92
+ "README.md",
93
+ "CHANGELOG.md",
94
+ "LICENSE.txt",
95
+ ]
96
+
97
+ [tool.hatch.envs.default]
98
+ dependencies = [
99
+ "pytest>=7.0",
100
+ "pytest-cov>=4.0",
101
+ "ruff>=0.4.0",
102
+ "mypy>=1.8",
103
+ ]
104
+
105
+ [tool.hatch.envs.default.scripts]
106
+ test = "pytest {args:tests}"
107
+ test-cov = "pytest --cov=split_markdown4gpt --cov-report=term-missing {args:tests}"
108
+ lint = ["ruff check --fix .", "ruff format ."]
109
+ typecheck = "mypy src/split_markdown4gpt"
110
+
111
+ [tool.pytest.ini_options]
112
+ addopts = "--verbose"
113
+ testpaths = ["tests"]
114
+
115
+ [tool.ruff]
116
+ target-version = "py310"
117
+ line-length = 88
118
+
119
+ [tool.ruff.lint]
120
+ select = ["E", "W", "F", "I", "UP", "B", "C4", "N"]
121
+ ignore = ["E501", "B019"] # B019: lru_cache on method is intentional here
122
+
123
+ [tool.ruff.lint.isort]
124
+ known-first-party = ["split_markdown4gpt"]
125
+
126
+ [tool.mypy]
127
+ python_version = "3.10"
128
+ strict = false
129
+ warn_return_any = true
130
+ warn_unused_configs = true
131
+ ignore_missing_imports = true