split-markdown4gpt 1.0.9__tar.gz → 2.7.7__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {split_markdown4gpt-1.0.9 → split_markdown4gpt-2.7.7}/.gitignore +1 -0
- split_markdown4gpt-2.7.7/AUTHORS.md +4 -0
- split_markdown4gpt-2.7.7/CHANGELOG.md +36 -0
- split_markdown4gpt-2.7.7/PKG-INFO +143 -0
- split_markdown4gpt-2.7.7/README.md +102 -0
- split_markdown4gpt-2.7.7/docs/api.md +13 -0
- split_markdown4gpt-2.7.7/docs/assets/icon.png +0 -0
- split_markdown4gpt-2.7.7/docs/changelog.md +3 -0
- split_markdown4gpt-2.7.7/docs/index.md +44 -0
- split_markdown4gpt-2.7.7/docs/usage.md +119 -0
- split_markdown4gpt-2.7.7/docs/why-splitting.md +78 -0
- split_markdown4gpt-2.7.7/pyproject.toml +131 -0
- split_markdown4gpt-2.7.7/src/split_markdown4gpt/__init__.py +30 -0
- {split_markdown4gpt-1.0.9 → split_markdown4gpt-2.7.7}/src/split_markdown4gpt/__main__.py +2 -2
- split_markdown4gpt-2.7.7/src/split_markdown4gpt/_version.py +24 -0
- split_markdown4gpt-2.7.7/src/split_markdown4gpt/splitter.py +486 -0
- split_markdown4gpt-2.7.7/tests/conftest.py +10 -0
- split_markdown4gpt-2.7.7/tests/test.md +31 -0
- split_markdown4gpt-2.7.7/tests/test_splitter.py +353 -0
- split_markdown4gpt-1.0.9/.DS_Store +0 -0
- split_markdown4gpt-1.0.9/.coveragerc +0 -28
- split_markdown4gpt-1.0.9/.github/workflows/ci.yml +0 -80
- split_markdown4gpt-1.0.9/.isort.cfg +0 -3
- split_markdown4gpt-1.0.9/.pre-commit-config.yaml +0 -67
- split_markdown4gpt-1.0.9/API.md +0 -329
- split_markdown4gpt-1.0.9/AUTHORS.md +0 -3
- split_markdown4gpt-1.0.9/PKG-INFO +0 -144
- split_markdown4gpt-1.0.9/README.md +0 -117
- split_markdown4gpt-1.0.9/pydoc-markdown.yml +0 -6
- split_markdown4gpt-1.0.9/pyproject.toml +0 -9
- split_markdown4gpt-1.0.9/setup.cfg +0 -94
- split_markdown4gpt-1.0.9/setup.py +0 -21
- split_markdown4gpt-1.0.9/src/split_markdown4gpt/__init__.py +0 -18
- split_markdown4gpt-1.0.9/src/split_markdown4gpt/splitter.py +0 -392
- split_markdown4gpt-1.0.9/src/split_markdown4gpt.egg-info/PKG-INFO +0 -144
- split_markdown4gpt-1.0.9/src/split_markdown4gpt.egg-info/SOURCES.txt +0 -28
- split_markdown4gpt-1.0.9/src/split_markdown4gpt.egg-info/dependency_links.txt +0 -1
- split_markdown4gpt-1.0.9/src/split_markdown4gpt.egg-info/entry_points.txt +0 -2
- split_markdown4gpt-1.0.9/src/split_markdown4gpt.egg-info/not-zip-safe +0 -1
- split_markdown4gpt-1.0.9/src/split_markdown4gpt.egg-info/requires.txt +0 -15
- split_markdown4gpt-1.0.9/src/split_markdown4gpt.egg-info/top_level.txt +0 -1
- split_markdown4gpt-1.0.9/tests/conftest.py +0 -10
- split_markdown4gpt-1.0.9/tests/test.md +0 -34
- split_markdown4gpt-1.0.9/tests/test_splitter.py +0 -11
- split_markdown4gpt-1.0.9/tox.ini +0 -93
- {split_markdown4gpt-1.0.9 → split_markdown4gpt-2.7.7}/LICENSE.txt +0 -0
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to `split-markdown4gpt` are documented here.
|
|
4
|
+
|
|
5
|
+
## [Unreleased]
|
|
6
|
+
|
|
7
|
+
### Changed
|
|
8
|
+
- **Build system**: migrated from `setup.cfg` / `setup.py` + `setuptools_scm` to a
|
|
9
|
+
fully self-contained `pyproject.toml` using `hatchling` + `hatch-vcs`.
|
|
10
|
+
- **Linting**: replaced `black` / `isort` / `flake8` with `ruff` (configured in
|
|
11
|
+
`pyproject.toml`).
|
|
12
|
+
- **CI**: updated GitHub Actions from `actions/checkout@v2` / `setup-python@v2` to
|
|
13
|
+
`@v4` / `@v5`; added Python 3.11 and 3.12 to the test matrix; switched from
|
|
14
|
+
plain `pip install` to `hatch run test`.
|
|
15
|
+
- **Type hints**: modernised from `typing.List / Dict / Tuple` to built-in generics
|
|
16
|
+
(`list`, `dict`, `tuple`); added `from __future__ import annotations` throughout.
|
|
17
|
+
- **`meta_data()`**: fixed silent bug in the `except` clause (`meta, data = None`
|
|
18
|
+
would raise `TypeError`; corrected to `meta, data = None, md`).
|
|
19
|
+
- **`OPENAI_MODELS`**: added `gpt-4-turbo`, `gpt-4o`, and `gpt-4o-mini` entries.
|
|
20
|
+
- **`load_md_path()`**: added explicit `encoding="utf-8"` to `open()`.
|
|
21
|
+
- **`__init__.py`**: removed Python 3.7 compatibility shim; use
|
|
22
|
+
`importlib.metadata` directly; added `__all__`.
|
|
23
|
+
|
|
24
|
+
### Added
|
|
25
|
+
- MkDocs Material documentation site (`docs/`, `mkdocs.yml`) with:
|
|
26
|
+
- *Why Splitting Matters* page explaining context-window constraints.
|
|
27
|
+
- *Usage* page with CLI and Python examples.
|
|
28
|
+
- *API Reference* page generated from docstrings.
|
|
29
|
+
- Comprehensive test suite (`tests/test_splitter.py`) expanded from 1 test to
|
|
30
|
+
34 tests covering: `meta_data()`, construction, all three load paths,
|
|
31
|
+
token counting, chunking behaviour, token-limit enforcement, front-matter
|
|
32
|
+
stripping, edge cases (empty input, only front matter, very long paragraph).
|
|
33
|
+
|
|
34
|
+
## [2.7.5] — 2024
|
|
35
|
+
|
|
36
|
+
- Earlier releases — see git history.
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: split_markdown4gpt
|
|
3
|
+
Version: 2.7.7
|
|
4
|
+
Summary: Split large Markdown files into token-bounded chunks for LLM context windows
|
|
5
|
+
Project-URL: Homepage, https://github.com/twardoch/split-markdown4gpt
|
|
6
|
+
Project-URL: Documentation, https://twardoch.github.io/split-markdown4gpt/
|
|
7
|
+
Project-URL: Source, https://github.com/twardoch/split-markdown4gpt
|
|
8
|
+
Project-URL: Bug Tracker, https://github.com/twardoch/split-markdown4gpt/issues
|
|
9
|
+
Author-email: Adam Twardoch <adam+github@twardoch.com>
|
|
10
|
+
License: Apache-2.0
|
|
11
|
+
License-File: AUTHORS.md
|
|
12
|
+
License-File: LICENSE.txt
|
|
13
|
+
Keywords: chunking,context-window,data-preprocessing,gpt,gpt-3,gpt-4,llm,markdown,mistletoe,natural-language-processing,nlp,openai,openai-gpt,python,split-text,summarization,text-analysis,text-processing,text-summarization,text-tokenization
|
|
14
|
+
Classifier: Development Status :: 4 - Beta
|
|
15
|
+
Classifier: Intended Audience :: Developers
|
|
16
|
+
Classifier: Intended Audience :: Science/Research
|
|
17
|
+
Classifier: License :: OSI Approved :: Apache Software License
|
|
18
|
+
Classifier: Operating System :: OS Independent
|
|
19
|
+
Classifier: Programming Language :: Python
|
|
20
|
+
Classifier: Programming Language :: Python :: 3
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
23
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
24
|
+
Classifier: Topic :: Text Processing
|
|
25
|
+
Classifier: Topic :: Text Processing :: Markup :: Markdown
|
|
26
|
+
Requires-Python: >=3.10
|
|
27
|
+
Requires-Dist: fire>=0.5.0
|
|
28
|
+
Requires-Dist: mistletoe>=1.1.0
|
|
29
|
+
Requires-Dist: python-frontmatter>=1.0.0
|
|
30
|
+
Requires-Dist: pyyaml>=5.1
|
|
31
|
+
Requires-Dist: regex>=2023.6.3
|
|
32
|
+
Requires-Dist: syntok>=1.4.4
|
|
33
|
+
Requires-Dist: tiktoken>=0.4.0
|
|
34
|
+
Provides-Extra: docs
|
|
35
|
+
Requires-Dist: mkdocs-material>=9.0; extra == 'docs'
|
|
36
|
+
Requires-Dist: mkdocstrings[python]>=0.20; extra == 'docs'
|
|
37
|
+
Provides-Extra: testing
|
|
38
|
+
Requires-Dist: pytest-cov>=4.0; extra == 'testing'
|
|
39
|
+
Requires-Dist: pytest>=7.0; extra == 'testing'
|
|
40
|
+
Description-Content-Type: text/markdown
|
|
41
|
+
|
|
42
|
+
# split-markdown4gpt: split large Markdown for LLM context windows
|
|
43
|
+
|
|
44
|
+
LLMs have finite context windows. A 50,000-word documentation site won't fit in one API call. `split-markdown4gpt` breaks Markdown files into token-bounded chunks that respect document structure — so each chunk stays coherent rather than cutting mid-sentence or mid-section.
|
|
45
|
+
|
|
46
|
+
## Why splitting matters
|
|
47
|
+
|
|
48
|
+
Every LLM API call has a hard token limit (e.g. 4,096 tokens for GPT-3.5, 8,192 for GPT-4). Exceeding it raises an error. Naive character-based chunking cuts through sentences, code blocks, and headings — destroying context. This tool:
|
|
49
|
+
|
|
50
|
+
1. Parses the Markdown AST (via `mistletoe`)
|
|
51
|
+
2. Counts tokens using the same tokenizer the model uses (via `tiktoken`)
|
|
52
|
+
3. Packs as many complete sections as fit within the limit
|
|
53
|
+
4. When a single section exceeds the limit, falls back to sentence-level splitting (via `syntok`)
|
|
54
|
+
|
|
55
|
+
The result: chunks that respect heading hierarchy, never split a sentence across chunks, and are as large as possible without exceeding the token budget.
|
|
56
|
+
|
|
57
|
+
## Install
|
|
58
|
+
|
|
59
|
+
```bash
|
|
60
|
+
pip install split_markdown4gpt
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
## CLI
|
|
64
|
+
|
|
65
|
+
```bash
|
|
66
|
+
mdsplit4gpt large_doc.md --model gpt-3.5-turbo --limit 4096 --separator "=== SPLIT ==="
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
Outputs all sections joined by the separator. Redirect to a file or pipe to another tool.
|
|
70
|
+
|
|
71
|
+
| Flag | Default | Description |
|
|
72
|
+
|------|---------|-------------|
|
|
73
|
+
| `--model` | `gpt-3.5-turbo` | Tokenizer model (determines token counting) |
|
|
74
|
+
| `--limit` | model max | Max tokens per chunk |
|
|
75
|
+
| `--separator` | `=== SPLIT ===` | String inserted between chunks in output |
|
|
76
|
+
|
|
77
|
+
## Python API
|
|
78
|
+
|
|
79
|
+
```python
|
|
80
|
+
from split_markdown4gpt import split
|
|
81
|
+
|
|
82
|
+
# Returns a list of Markdown strings, each within the token limit
|
|
83
|
+
chunks = split("large_doc.md", model="gpt-3.5-turbo", limit=4096)
|
|
84
|
+
|
|
85
|
+
for i, chunk in enumerate(chunks):
|
|
86
|
+
response = openai_client.chat.completions.create(
|
|
87
|
+
model="gpt-3.5-turbo", messages=[{"role": "user", "content": chunk}]
|
|
88
|
+
)
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
Also accepts a string or file-like object:
|
|
92
|
+
|
|
93
|
+
```python
|
|
94
|
+
chunks = split(markdown_string, model="gpt-4", limit=8000)
|
|
95
|
+
chunks = split(open("doc.md"), limit=2000)
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
## How it works
|
|
99
|
+
|
|
100
|
+
```
|
|
101
|
+
split(md)
|
|
102
|
+
├── meta_data() strip YAML front matter
|
|
103
|
+
├── MarkdownLLMSplitter.build_md_dict()
|
|
104
|
+
│ ├── mistletoe Document parse Markdown AST
|
|
105
|
+
│ └── build dict tree keyed by heading level
|
|
106
|
+
├── calculate_sizes() count tokens recursively (cached with lru_cache)
|
|
107
|
+
└── get_sections_from_md_dict_by_limit()
|
|
108
|
+
├── walk tree, accumulate blocks until limit reached
|
|
109
|
+
└── oversized blocks → sentence-split via syntok
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
### `MarkdownLLMSplitter` class
|
|
113
|
+
|
|
114
|
+
```python
|
|
115
|
+
from split_markdown4gpt import MarkdownLLMSplitter
|
|
116
|
+
|
|
117
|
+
splitter = MarkdownLLMSplitter(gptok_model="gpt-4", gptok_limit=6000)
|
|
118
|
+
splitter.load_md("doc.md")
|
|
119
|
+
splitter.build()
|
|
120
|
+
|
|
121
|
+
for section in splitter.gen_section_dicts():
|
|
122
|
+
print(section["gptok_size"], "tokens")
|
|
123
|
+
print(section["md"])
|
|
124
|
+
```
|
|
125
|
+
|
|
126
|
+
Methods: `load_md()`, `load_md_path()`, `load_md_str()`, `build()`, `list_section_texts()`, `list_section_dicts()`, `gen_section_texts()`, `gen_section_dicts()`, `split()`.
|
|
127
|
+
|
|
128
|
+
## Supported models
|
|
129
|
+
|
|
130
|
+
Any model supported by `tiktoken`, including all `gpt-3.5-turbo` and `gpt-4` variants. The limit defaults to the model's published context window.
|
|
131
|
+
|
|
132
|
+
## Dependencies
|
|
133
|
+
|
|
134
|
+
- `tiktoken` — OpenAI-compatible token counting
|
|
135
|
+
- `mistletoe` — Markdown AST parser
|
|
136
|
+
- `syntok` — sentence segmentation for oversized sections
|
|
137
|
+
- `frontmatter` — YAML front matter stripping
|
|
138
|
+
- `fire` — CLI
|
|
139
|
+
- `regex`, `PyYAML` — utilities
|
|
140
|
+
|
|
141
|
+
## License
|
|
142
|
+
|
|
143
|
+
Apache License 2.0. Copyright © 2023 Adam Twardoch.
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
# split-markdown4gpt: split large Markdown for LLM context windows
|
|
2
|
+
|
|
3
|
+
LLMs have finite context windows. A 50,000-word documentation site won't fit in one API call. `split-markdown4gpt` breaks Markdown files into token-bounded chunks that respect document structure — so each chunk stays coherent rather than cutting mid-sentence or mid-section.
|
|
4
|
+
|
|
5
|
+
## Why splitting matters
|
|
6
|
+
|
|
7
|
+
Every LLM API call has a hard token limit (e.g. 4,096 tokens for GPT-3.5, 8,192 for GPT-4). Exceeding it raises an error. Naive character-based chunking cuts through sentences, code blocks, and headings — destroying context. This tool:
|
|
8
|
+
|
|
9
|
+
1. Parses the Markdown AST (via `mistletoe`)
|
|
10
|
+
2. Counts tokens using the same tokenizer the model uses (via `tiktoken`)
|
|
11
|
+
3. Packs as many complete sections as fit within the limit
|
|
12
|
+
4. When a single section exceeds the limit, falls back to sentence-level splitting (via `syntok`)
|
|
13
|
+
|
|
14
|
+
The result: chunks that respect heading hierarchy, never split a sentence across chunks, and are as large as possible without exceeding the token budget.
|
|
15
|
+
|
|
16
|
+
## Install
|
|
17
|
+
|
|
18
|
+
```bash
|
|
19
|
+
pip install split_markdown4gpt
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
## CLI
|
|
23
|
+
|
|
24
|
+
```bash
|
|
25
|
+
mdsplit4gpt large_doc.md --model gpt-3.5-turbo --limit 4096 --separator "=== SPLIT ==="
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
Outputs all sections joined by the separator. Redirect to a file or pipe to another tool.
|
|
29
|
+
|
|
30
|
+
| Flag | Default | Description |
|
|
31
|
+
|------|---------|-------------|
|
|
32
|
+
| `--model` | `gpt-3.5-turbo` | Tokenizer model (determines token counting) |
|
|
33
|
+
| `--limit` | model max | Max tokens per chunk |
|
|
34
|
+
| `--separator` | `=== SPLIT ===` | String inserted between chunks in output |
|
|
35
|
+
|
|
36
|
+
## Python API
|
|
37
|
+
|
|
38
|
+
```python
|
|
39
|
+
from split_markdown4gpt import split
|
|
40
|
+
|
|
41
|
+
# Returns a list of Markdown strings, each within the token limit
|
|
42
|
+
chunks = split("large_doc.md", model="gpt-3.5-turbo", limit=4096)
|
|
43
|
+
|
|
44
|
+
for i, chunk in enumerate(chunks):
|
|
45
|
+
response = openai_client.chat.completions.create(
|
|
46
|
+
model="gpt-3.5-turbo", messages=[{"role": "user", "content": chunk}]
|
|
47
|
+
)
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
Also accepts a string or file-like object:
|
|
51
|
+
|
|
52
|
+
```python
|
|
53
|
+
chunks = split(markdown_string, model="gpt-4", limit=8000)
|
|
54
|
+
chunks = split(open("doc.md"), limit=2000)
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
## How it works
|
|
58
|
+
|
|
59
|
+
```
|
|
60
|
+
split(md)
|
|
61
|
+
├── meta_data() strip YAML front matter
|
|
62
|
+
├── MarkdownLLMSplitter.build_md_dict()
|
|
63
|
+
│ ├── mistletoe Document parse Markdown AST
|
|
64
|
+
│ └── build dict tree keyed by heading level
|
|
65
|
+
├── calculate_sizes() count tokens recursively (cached with lru_cache)
|
|
66
|
+
└── get_sections_from_md_dict_by_limit()
|
|
67
|
+
├── walk tree, accumulate blocks until limit reached
|
|
68
|
+
└── oversized blocks → sentence-split via syntok
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
### `MarkdownLLMSplitter` class
|
|
72
|
+
|
|
73
|
+
```python
|
|
74
|
+
from split_markdown4gpt import MarkdownLLMSplitter
|
|
75
|
+
|
|
76
|
+
splitter = MarkdownLLMSplitter(gptok_model="gpt-4", gptok_limit=6000)
|
|
77
|
+
splitter.load_md("doc.md")
|
|
78
|
+
splitter.build()
|
|
79
|
+
|
|
80
|
+
for section in splitter.gen_section_dicts():
|
|
81
|
+
print(section["gptok_size"], "tokens")
|
|
82
|
+
print(section["md"])
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
Methods: `load_md()`, `load_md_path()`, `load_md_str()`, `build()`, `list_section_texts()`, `list_section_dicts()`, `gen_section_texts()`, `gen_section_dicts()`, `split()`.
|
|
86
|
+
|
|
87
|
+
## Supported models
|
|
88
|
+
|
|
89
|
+
Any model supported by `tiktoken`, including all `gpt-3.5-turbo` and `gpt-4` variants. The limit defaults to the model's published context window.
|
|
90
|
+
|
|
91
|
+
## Dependencies
|
|
92
|
+
|
|
93
|
+
- `tiktoken` — OpenAI-compatible token counting
|
|
94
|
+
- `mistletoe` — Markdown AST parser
|
|
95
|
+
- `syntok` — sentence segmentation for oversized sections
|
|
96
|
+
- `frontmatter` — YAML front matter stripping
|
|
97
|
+
- `fire` — CLI
|
|
98
|
+
- `regex`, `PyYAML` — utilities
|
|
99
|
+
|
|
100
|
+
## License
|
|
101
|
+
|
|
102
|
+
Apache License 2.0. Copyright © 2023 Adam Twardoch.
|
|
Binary file
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
# split-markdown4gpt
|
|
2
|
+
|
|
3
|
+
**Split large Markdown files into token-bounded chunks for LLM context windows.**
|
|
4
|
+
|
|
5
|
+
LLMs have finite context windows. A 50,000-word documentation site will not fit in one API call. `split-markdown4gpt` breaks Markdown files into token-bounded chunks that respect document structure — each chunk stays coherent rather than cutting mid-sentence or mid-section.
|
|
6
|
+
|
|
7
|
+
## Quick start
|
|
8
|
+
|
|
9
|
+
```bash
|
|
10
|
+
pip install split_markdown4gpt
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
```python
|
|
14
|
+
from split_markdown4gpt import split
|
|
15
|
+
|
|
16
|
+
chunks = split("large_doc.md", model="gpt-4", limit=6000)
|
|
17
|
+
for chunk in chunks:
|
|
18
|
+
response = client.chat.completions.create(
|
|
19
|
+
model="gpt-4",
|
|
20
|
+
messages=[{"role": "user", "content": chunk}],
|
|
21
|
+
)
|
|
22
|
+
```
|
|
23
|
+
|
|
24
|
+
Or from the command line:
|
|
25
|
+
|
|
26
|
+
```bash
|
|
27
|
+
mdsplit4gpt large_doc.md --model gpt-3.5-turbo --limit 4096
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
## How it works
|
|
31
|
+
|
|
32
|
+
```
|
|
33
|
+
split(md)
|
|
34
|
+
├── meta_data() strip YAML front matter
|
|
35
|
+
├── build_md_dict()
|
|
36
|
+
│ ├── mistletoe parse Markdown AST
|
|
37
|
+
│ └── heading tree keyed by level
|
|
38
|
+
├── calculate_sizes() count tokens (tiktoken, cached)
|
|
39
|
+
└── get_sections_from_md_dict_by_limit()
|
|
40
|
+
├── walk tree, accumulate blocks until limit reached
|
|
41
|
+
└── oversized blocks → sentence-split via syntok
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
See [Why Splitting Matters](why-splitting.md) for the motivation, [Usage](usage.md) for the full API, and the [API Reference](api.md) for generated documentation.
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
# Usage
|
|
2
|
+
|
|
3
|
+
## Installation
|
|
4
|
+
|
|
5
|
+
```bash
|
|
6
|
+
pip install split_markdown4gpt
|
|
7
|
+
```
|
|
8
|
+
|
|
9
|
+
## CLI
|
|
10
|
+
|
|
11
|
+
```bash
|
|
12
|
+
mdsplit4gpt large_doc.md --model gpt-3.5-turbo --limit 4096 --separator "=== SPLIT ==="
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
Outputs all chunks joined by the separator. Redirect to a file or pipe to another tool.
|
|
16
|
+
|
|
17
|
+
| Flag | Default | Description |
|
|
18
|
+
|---|---|---|
|
|
19
|
+
| `--model` | `gpt-3.5-turbo` | Tiktoken model (determines token counting) |
|
|
20
|
+
| `--limit` | model max | Max tokens per chunk |
|
|
21
|
+
| `--separator` | `=== SPLIT ===` | String inserted between chunks in output |
|
|
22
|
+
|
|
23
|
+
## Python — quick API
|
|
24
|
+
|
|
25
|
+
```python
|
|
26
|
+
from split_markdown4gpt import split
|
|
27
|
+
|
|
28
|
+
# From a file path
|
|
29
|
+
chunks = split("large_doc.md", model="gpt-3.5-turbo", limit=4096)
|
|
30
|
+
|
|
31
|
+
# From a string
|
|
32
|
+
chunks = split(markdown_string, model="gpt-4", limit=8000)
|
|
33
|
+
|
|
34
|
+
# From a file object
|
|
35
|
+
with open("doc.md") as f:
|
|
36
|
+
chunks = split(f, limit=2000)
|
|
37
|
+
|
|
38
|
+
for chunk in chunks:
|
|
39
|
+
# Each chunk is a Markdown string within the token limit
|
|
40
|
+
print(len(chunk), "chars")
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
## Python — class API
|
|
44
|
+
|
|
45
|
+
Use `MarkdownLLMSplitter` directly for more control:
|
|
46
|
+
|
|
47
|
+
```python
|
|
48
|
+
from split_markdown4gpt import MarkdownLLMSplitter
|
|
49
|
+
|
|
50
|
+
splitter = MarkdownLLMSplitter(gptok_model="gpt-4", gptok_limit=6000)
|
|
51
|
+
|
|
52
|
+
# Load content
|
|
53
|
+
splitter.load_md("doc.md") # from path
|
|
54
|
+
splitter.load_md_str(md_string) # from string
|
|
55
|
+
splitter.load_md_file(file_obj) # from file object
|
|
56
|
+
|
|
57
|
+
# Build sections
|
|
58
|
+
splitter.build()
|
|
59
|
+
|
|
60
|
+
# Access results
|
|
61
|
+
texts = splitter.list_section_texts() # list[str]
|
|
62
|
+
dicts = splitter.list_section_dicts() # list[{"md": str, "gptok_size": int}]
|
|
63
|
+
|
|
64
|
+
# Or use generators
|
|
65
|
+
for text in splitter.gen_section_texts():
|
|
66
|
+
print(text)
|
|
67
|
+
|
|
68
|
+
for section in splitter.gen_section_dicts():
|
|
69
|
+
print(section["gptok_size"], "tokens:", section["md"][:80])
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
## Supported models
|
|
73
|
+
|
|
74
|
+
Any model supported by `tiktoken`, including all `gpt-3.5-turbo` and `gpt-4` variants.
|
|
75
|
+
|
|
76
|
+
```python
|
|
77
|
+
from split_markdown4gpt.splitter import OPENAI_MODELS
|
|
78
|
+
|
|
79
|
+
print(OPENAI_MODELS)
|
|
80
|
+
# {"gpt-4": 8192, "gpt-4-turbo": 128000, "gpt-4o": 128000, ...}
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
## Processing with OpenAI
|
|
84
|
+
|
|
85
|
+
```python
|
|
86
|
+
from openai import OpenAI
|
|
87
|
+
from split_markdown4gpt import split
|
|
88
|
+
|
|
89
|
+
client = OpenAI()
|
|
90
|
+
|
|
91
|
+
chunks = split("large_doc.md", model="gpt-4", limit=6000)
|
|
92
|
+
|
|
93
|
+
summaries = []
|
|
94
|
+
for chunk in chunks:
|
|
95
|
+
response = client.chat.completions.create(
|
|
96
|
+
model="gpt-4",
|
|
97
|
+
messages=[
|
|
98
|
+
{
|
|
99
|
+
"role": "system",
|
|
100
|
+
"content": "Summarise the following documentation section.",
|
|
101
|
+
},
|
|
102
|
+
{"role": "user", "content": chunk},
|
|
103
|
+
],
|
|
104
|
+
)
|
|
105
|
+
summaries.append(response.choices[0].message.content)
|
|
106
|
+
|
|
107
|
+
final_summary = "\n\n".join(summaries)
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
## YAML front matter
|
|
111
|
+
|
|
112
|
+
Front matter is automatically stripped before chunking. Metadata is available on the splitter instance:
|
|
113
|
+
|
|
114
|
+
```python
|
|
115
|
+
splitter = MarkdownLLMSplitter()
|
|
116
|
+
splitter.load_md("doc.md")
|
|
117
|
+
print(splitter.md_meta) # {"title": "...", "author": "..."}
|
|
118
|
+
splitter.build()
|
|
119
|
+
```
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
# Why Splitting Matters for LLM Context Windows
|
|
2
|
+
|
|
3
|
+
## The context-window problem
|
|
4
|
+
|
|
5
|
+
Every LLM API call operates within a hard token limit — a ceiling on the combined length of the prompt and the generated response. Common limits are:
|
|
6
|
+
|
|
7
|
+
| Model | Context window |
|
|
8
|
+
|---|---|
|
|
9
|
+
| GPT-3.5 Turbo | 4,096 tokens |
|
|
10
|
+
| GPT-3.5 Turbo 16k | 16,384 tokens |
|
|
11
|
+
| GPT-4 | 8,192 tokens |
|
|
12
|
+
| GPT-4 Turbo / GPT-4o | 128,000 tokens |
|
|
13
|
+
|
|
14
|
+
A "token" is roughly 0.75 words in English, so 4,096 tokens is about 3,000 words — just under five pages of dense text. Even 128k tokens is only around 90,000 words; a single large codebase documentation site can easily exceed that.
|
|
15
|
+
|
|
16
|
+
Exceeding the limit raises an API error. The only remedy is to send less text per call.
|
|
17
|
+
|
|
18
|
+
## Why naive splitting fails
|
|
19
|
+
|
|
20
|
+
The obvious fix — splitting on character count or word count — destroys document coherence:
|
|
21
|
+
|
|
22
|
+
- A sentence split across two chunks loses its meaning.
|
|
23
|
+
- A code block cut in the middle becomes invalid.
|
|
24
|
+
- A heading at the end of one chunk has no body; the body appears in the next chunk with no heading.
|
|
25
|
+
- A bullet list split mid-item confuses the model about list structure.
|
|
26
|
+
|
|
27
|
+
Models trained on coherent text produce incoherent output when fed incoherent input.
|
|
28
|
+
|
|
29
|
+
## Structure-aware chunking
|
|
30
|
+
|
|
31
|
+
`split-markdown4gpt` parses the Markdown document as an abstract syntax tree (AST) using `mistletoe`, then works with complete structural units:
|
|
32
|
+
|
|
33
|
+
1. **Headings** define section boundaries. The splitter respects the heading hierarchy (H1 → H2 → H3) so a chunk always begins at a logical section start.
|
|
34
|
+
2. **Paragraphs, code blocks, lists, and tables** are treated as atomic units. They are never broken mid-block.
|
|
35
|
+
3. **Token counting** uses `tiktoken` — the same tokenizer the OpenAI models use — so the count is exact rather than approximated.
|
|
36
|
+
4. **Greedy packing** fills each chunk as full as possible without exceeding the limit, minimising the total number of API calls needed.
|
|
37
|
+
5. **Sentence-level fallback** handles the edge case where a single paragraph exceeds the limit. `syntok` splits it into sentences and packs those instead.
|
|
38
|
+
|
|
39
|
+
## Example
|
|
40
|
+
|
|
41
|
+
Given this document (simplified):
|
|
42
|
+
|
|
43
|
+
```markdown
|
|
44
|
+
# Introduction
|
|
45
|
+
|
|
46
|
+
OpenAI's GPT models have changed how developers build applications...
|
|
47
|
+
[500 tokens]
|
|
48
|
+
|
|
49
|
+
## Background
|
|
50
|
+
|
|
51
|
+
The transformer architecture, introduced in 2017...
|
|
52
|
+
[800 tokens]
|
|
53
|
+
|
|
54
|
+
## Key capabilities
|
|
55
|
+
|
|
56
|
+
GPT-4 can reason, code, summarise, and translate...
|
|
57
|
+
[600 tokens]
|
|
58
|
+
|
|
59
|
+
# Implementation guide
|
|
60
|
+
|
|
61
|
+
To use the API in Python, first install the SDK...
|
|
62
|
+
[3000 tokens]
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
With a 4,096-token limit, naive character splitting might cut the Implementation guide mid-sentence. `split-markdown4gpt` instead produces:
|
|
66
|
+
|
|
67
|
+
**Chunk 1** (1,900 tokens): Introduction + Background + Key capabilities — all three sections fit together.
|
|
68
|
+
|
|
69
|
+
**Chunk 2** (3,000 tokens): Implementation guide — the entire section in one piece.
|
|
70
|
+
|
|
71
|
+
Each chunk is coherent, self-contained, and well within the limit.
|
|
72
|
+
|
|
73
|
+
## Practical impact
|
|
74
|
+
|
|
75
|
+
- **Summarisation pipelines**: Each chunk can be summarised independently; summaries can then be re-combined.
|
|
76
|
+
- **Question answering over long docs**: Send chunks sequentially; keep only chunks where the model finds relevant content.
|
|
77
|
+
- **Fine-tuning data preparation**: Produce training examples that are each within the context window.
|
|
78
|
+
- **RAG (retrieval-augmented generation)**: Chunks map naturally to vector-store documents; structure-aware chunking produces more semantically coherent embeddings than character-split chunks.
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling", "hatch-vcs"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "split_markdown4gpt"
|
|
7
|
+
description = "Split large Markdown files into token-bounded chunks for LLM context windows"
|
|
8
|
+
readme = "README.md"
|
|
9
|
+
license = { text = "Apache-2.0" }
|
|
10
|
+
authors = [{ name = "Adam Twardoch", email = "adam+github@twardoch.com" }]
|
|
11
|
+
keywords = [
|
|
12
|
+
"python",
|
|
13
|
+
"nlp",
|
|
14
|
+
"markdown",
|
|
15
|
+
"natural-language-processing",
|
|
16
|
+
"text-analysis",
|
|
17
|
+
"openai",
|
|
18
|
+
"text-summarization",
|
|
19
|
+
"summarization",
|
|
20
|
+
"text-processing",
|
|
21
|
+
"gpt",
|
|
22
|
+
"data-preprocessing",
|
|
23
|
+
"mistletoe",
|
|
24
|
+
"split-text",
|
|
25
|
+
"text-tokenization",
|
|
26
|
+
"openai-gpt",
|
|
27
|
+
"gpt-3",
|
|
28
|
+
"gpt-4",
|
|
29
|
+
"llm",
|
|
30
|
+
"context-window",
|
|
31
|
+
"chunking",
|
|
32
|
+
]
|
|
33
|
+
classifiers = [
|
|
34
|
+
"Development Status :: 4 - Beta",
|
|
35
|
+
"Programming Language :: Python",
|
|
36
|
+
"Programming Language :: Python :: 3",
|
|
37
|
+
"Programming Language :: Python :: 3.10",
|
|
38
|
+
"Programming Language :: Python :: 3.11",
|
|
39
|
+
"Programming Language :: Python :: 3.12",
|
|
40
|
+
"Intended Audience :: Developers",
|
|
41
|
+
"Intended Audience :: Science/Research",
|
|
42
|
+
"Topic :: Text Processing",
|
|
43
|
+
"Topic :: Text Processing :: Markup :: Markdown",
|
|
44
|
+
"License :: OSI Approved :: Apache Software License",
|
|
45
|
+
"Operating System :: OS Independent",
|
|
46
|
+
]
|
|
47
|
+
requires-python = ">=3.10"
|
|
48
|
+
dependencies = [
|
|
49
|
+
"fire>=0.5.0",
|
|
50
|
+
"mistletoe>=1.1.0",
|
|
51
|
+
"python-frontmatter>=1.0.0",
|
|
52
|
+
"PyYAML>=5.1",
|
|
53
|
+
"regex>=2023.6.3",
|
|
54
|
+
"syntok>=1.4.4",
|
|
55
|
+
"tiktoken>=0.4.0",
|
|
56
|
+
]
|
|
57
|
+
dynamic = ["version"]
|
|
58
|
+
|
|
59
|
+
[project.urls]
|
|
60
|
+
Homepage = "https://github.com/twardoch/split-markdown4gpt"
|
|
61
|
+
Documentation = "https://twardoch.github.io/split-markdown4gpt/"
|
|
62
|
+
Source = "https://github.com/twardoch/split-markdown4gpt"
|
|
63
|
+
"Bug Tracker" = "https://github.com/twardoch/split-markdown4gpt/issues"
|
|
64
|
+
|
|
65
|
+
[project.optional-dependencies]
|
|
66
|
+
testing = [
|
|
67
|
+
"pytest>=7.0",
|
|
68
|
+
"pytest-cov>=4.0",
|
|
69
|
+
]
|
|
70
|
+
docs = [
|
|
71
|
+
"mkdocs-material>=9.0",
|
|
72
|
+
"mkdocstrings[python]>=0.20",
|
|
73
|
+
]
|
|
74
|
+
|
|
75
|
+
[project.scripts]
|
|
76
|
+
mdsplit4gpt = "split_markdown4gpt.__main__:cli"
|
|
77
|
+
|
|
78
|
+
[tool.hatch.version]
|
|
79
|
+
source = "vcs"
|
|
80
|
+
|
|
81
|
+
[tool.hatch.build.hooks.vcs]
|
|
82
|
+
version-file = "src/split_markdown4gpt/_version.py"
|
|
83
|
+
|
|
84
|
+
[tool.hatch.build.targets.wheel]
|
|
85
|
+
packages = ["src/split_markdown4gpt"]
|
|
86
|
+
|
|
87
|
+
[tool.hatch.build.targets.sdist]
|
|
88
|
+
include = [
|
|
89
|
+
"/src",
|
|
90
|
+
"/tests",
|
|
91
|
+
"/docs",
|
|
92
|
+
"README.md",
|
|
93
|
+
"CHANGELOG.md",
|
|
94
|
+
"LICENSE.txt",
|
|
95
|
+
]
|
|
96
|
+
|
|
97
|
+
[tool.hatch.envs.default]
|
|
98
|
+
dependencies = [
|
|
99
|
+
"pytest>=7.0",
|
|
100
|
+
"pytest-cov>=4.0",
|
|
101
|
+
"ruff>=0.4.0",
|
|
102
|
+
"mypy>=1.8",
|
|
103
|
+
]
|
|
104
|
+
|
|
105
|
+
[tool.hatch.envs.default.scripts]
|
|
106
|
+
test = "pytest {args:tests}"
|
|
107
|
+
test-cov = "pytest --cov=split_markdown4gpt --cov-report=term-missing {args:tests}"
|
|
108
|
+
lint = ["ruff check --fix .", "ruff format ."]
|
|
109
|
+
typecheck = "mypy src/split_markdown4gpt"
|
|
110
|
+
|
|
111
|
+
[tool.pytest.ini_options]
|
|
112
|
+
addopts = "--verbose"
|
|
113
|
+
testpaths = ["tests"]
|
|
114
|
+
|
|
115
|
+
[tool.ruff]
|
|
116
|
+
target-version = "py310"
|
|
117
|
+
line-length = 88
|
|
118
|
+
|
|
119
|
+
[tool.ruff.lint]
|
|
120
|
+
select = ["E", "W", "F", "I", "UP", "B", "C4", "N"]
|
|
121
|
+
ignore = ["E501", "B019"] # B019: lru_cache on method is intentional here
|
|
122
|
+
|
|
123
|
+
[tool.ruff.lint.isort]
|
|
124
|
+
known-first-party = ["split_markdown4gpt"]
|
|
125
|
+
|
|
126
|
+
[tool.mypy]
|
|
127
|
+
python_version = "3.10"
|
|
128
|
+
strict = false
|
|
129
|
+
warn_return_any = true
|
|
130
|
+
warn_unused_configs = true
|
|
131
|
+
ignore_missing_imports = true
|