split-markdown4gpt 1.0.1__tar.gz → 1.0.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {split_markdown4gpt-1.0.1 → split_markdown4gpt-1.0.2}/PKG-INFO +50 -9
- {split_markdown4gpt-1.0.1 → split_markdown4gpt-1.0.2}/README.md +50 -9
- split_markdown4gpt-1.0.2/src/split_markdown4gpt/__main__.py +44 -0
- {split_markdown4gpt-1.0.1 → split_markdown4gpt-1.0.2}/src/split_markdown4gpt/splitter.py +152 -21
- {split_markdown4gpt-1.0.1 → split_markdown4gpt-1.0.2}/src/split_markdown4gpt.egg-info/PKG-INFO +50 -9
- split_markdown4gpt-1.0.1/src/split_markdown4gpt/__main__.py +0 -27
- {split_markdown4gpt-1.0.1 → split_markdown4gpt-1.0.2}/.DS_Store +0 -0
- {split_markdown4gpt-1.0.1 → split_markdown4gpt-1.0.2}/.coveragerc +0 -0
- {split_markdown4gpt-1.0.1 → split_markdown4gpt-1.0.2}/.github/workflows/ci.yml +0 -0
- {split_markdown4gpt-1.0.1 → split_markdown4gpt-1.0.2}/.gitignore +0 -0
- {split_markdown4gpt-1.0.1 → split_markdown4gpt-1.0.2}/.isort.cfg +0 -0
- {split_markdown4gpt-1.0.1 → split_markdown4gpt-1.0.2}/.pre-commit-config.yaml +0 -0
- {split_markdown4gpt-1.0.1 → split_markdown4gpt-1.0.2}/AUTHORS.md +0 -0
- {split_markdown4gpt-1.0.1 → split_markdown4gpt-1.0.2}/LICENSE.txt +0 -0
- {split_markdown4gpt-1.0.1 → split_markdown4gpt-1.0.2}/pyproject.toml +0 -0
- {split_markdown4gpt-1.0.1 → split_markdown4gpt-1.0.2}/setup.cfg +0 -0
- {split_markdown4gpt-1.0.1 → split_markdown4gpt-1.0.2}/setup.py +0 -0
- {split_markdown4gpt-1.0.1 → split_markdown4gpt-1.0.2}/src/split_markdown4gpt/__init__.py +0 -0
- {split_markdown4gpt-1.0.1 → split_markdown4gpt-1.0.2}/src/split_markdown4gpt.egg-info/SOURCES.txt +0 -0
- {split_markdown4gpt-1.0.1 → split_markdown4gpt-1.0.2}/src/split_markdown4gpt.egg-info/dependency_links.txt +0 -0
- {split_markdown4gpt-1.0.1 → split_markdown4gpt-1.0.2}/src/split_markdown4gpt.egg-info/entry_points.txt +0 -0
- {split_markdown4gpt-1.0.1 → split_markdown4gpt-1.0.2}/src/split_markdown4gpt.egg-info/not-zip-safe +0 -0
- {split_markdown4gpt-1.0.1 → split_markdown4gpt-1.0.2}/src/split_markdown4gpt.egg-info/requires.txt +0 -0
- {split_markdown4gpt-1.0.1 → split_markdown4gpt-1.0.2}/src/split_markdown4gpt.egg-info/top_level.txt +0 -0
- {split_markdown4gpt-1.0.1 → split_markdown4gpt-1.0.2}/tests/conftest.py +0 -0
- {split_markdown4gpt-1.0.1 → split_markdown4gpt-1.0.2}/tests/test.md +0 -0
- {split_markdown4gpt-1.0.1 → split_markdown4gpt-1.0.2}/tests/test_splitter.py +0 -0
- {split_markdown4gpt-1.0.1 → split_markdown4gpt-1.0.2}/tox.ini +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: split_markdown4gpt
|
|
3
|
-
Version: 1.0.
|
|
3
|
+
Version: 1.0.2
|
|
4
4
|
Summary: A Python tool for splitting large Markdown files into smaller sections based on a specified token limit. This is particularly useful for processing large Markdown files with GPT models, as it allows the models to handle the data in manageable chunks.
|
|
5
5
|
Home-page: https://github.com/twardoch/split-markdown4gpt
|
|
6
6
|
Author: Adam Twardoch
|
|
@@ -28,7 +28,7 @@ License-File: LICENSE.txt
|
|
|
28
28
|
|
|
29
29
|
`split_markdown4gpt` is a Python tool designed to split large Markdown files into smaller sections based on a specified token limit. This is particularly useful for processing large Markdown files with GPT models, as it allows the models to handle the data in manageable chunks.
|
|
30
30
|
|
|
31
|
-
_Version 1.0.
|
|
31
|
+
_Version 1.0.2_ (2023-06-18)
|
|
32
32
|
|
|
33
33
|
## Installation
|
|
34
34
|
|
|
@@ -38,7 +38,7 @@ You can install `split_markdown4gpt` via pip:
|
|
|
38
38
|
pip install split_markdown4gpt
|
|
39
39
|
```
|
|
40
40
|
|
|
41
|
-
## CLI
|
|
41
|
+
## CLI Usage
|
|
42
42
|
|
|
43
43
|
After installation, you can use the `mdsplit4gpt` command to split a Markdown file. Here's the basic syntax:
|
|
44
44
|
|
|
@@ -48,32 +48,41 @@ mdsplit4gpt path_to_your_file.md --model gpt-3.5-turbo --limit 4096 --separator
|
|
|
48
48
|
|
|
49
49
|
This command will split the Markdown file at `path_to_your_file.md` into sections, each containing no more than 4096 tokens (as counted by the `gpt-3.5-turbo` model). The sections will be separated by `=== SPLIT ===`.
|
|
50
50
|
|
|
51
|
-
All CLI options:
|
|
51
|
+
All CLI options:
|
|
52
52
|
|
|
53
53
|
```
|
|
54
54
|
NAME
|
|
55
|
-
mdsplit4gpt
|
|
55
|
+
mdsplit4gpt - Splits a Markdown file into sections according to GPT token size limits.
|
|
56
56
|
|
|
57
57
|
SYNOPSIS
|
|
58
58
|
mdsplit4gpt MD_PATH <flags>
|
|
59
59
|
|
|
60
|
+
DESCRIPTION
|
|
61
|
+
This tool loads a Markdown file, and splits its content into sections
|
|
62
|
+
that are within the specified token size limit using the desired GPT tokenizing model. The resulting
|
|
63
|
+
sections are then concatenated using the specified separator and returned as a single string.
|
|
64
|
+
|
|
60
65
|
POSITIONAL ARGUMENTS
|
|
61
66
|
MD_PATH
|
|
62
67
|
Type: Union
|
|
68
|
+
The path of the source Markdown file to be split.
|
|
63
69
|
|
|
64
70
|
FLAGS
|
|
65
71
|
-m, --model=MODEL
|
|
66
72
|
Type: str
|
|
67
73
|
Default: 'gpt-3.5-turbo'
|
|
74
|
+
The GPT tokenizer model to use for calculating token sizes. Defaults to "gpt-3.5-turbo".
|
|
68
75
|
-l, --limit=LIMIT
|
|
69
76
|
Type: Optional[int]
|
|
70
77
|
Default: None
|
|
78
|
+
The maximum number of GPT tokens allowed per section. Defaults to the model's maximum tokens.
|
|
71
79
|
-s, --separator=SEPARATOR
|
|
72
80
|
Type: str
|
|
73
81
|
Default: '=== SPLIT ==='
|
|
82
|
+
The string used to separate sections in the output. Defaults to "=== SPLIT ===".
|
|
74
83
|
```
|
|
75
84
|
|
|
76
|
-
## Python
|
|
85
|
+
## Python Usage
|
|
77
86
|
|
|
78
87
|
You can also use `split_markdown4gpt` in your Python code. Here's a basic example:
|
|
79
88
|
|
|
@@ -87,11 +96,13 @@ for section in sections:
|
|
|
87
96
|
|
|
88
97
|
This code does the same thing as the CLI command above, but in Python.
|
|
89
98
|
|
|
90
|
-
## How it
|
|
99
|
+
## How it Works
|
|
91
100
|
|
|
92
101
|
`split_markdown4gpt` works by tokenizing the input Markdown file using the specified GPT model's tokenizer (default is `gpt-3.5-turbo`). It then splits the file into sections, each containing no more than the specified token limit.
|
|
93
102
|
|
|
94
|
-
The splitting process respects the structure of the Markdown file. It will not split a section (as defined by Markdown headings) across multiple output sections unless the section is longer than the token limit. In that case, it
|
|
103
|
+
The splitting process respects the structure of the Markdown file. It will not split a section (as defined by Markdown headings) across multiple output sections unless the section is longer than the token limit. In that case, it
|
|
104
|
+
|
|
105
|
+
will split the section at the sentence level.
|
|
95
106
|
|
|
96
107
|
The tool uses several libraries to accomplish this:
|
|
97
108
|
|
|
@@ -102,6 +113,37 @@ The tool uses several libraries to accomplish this:
|
|
|
102
113
|
- `syntok` for splitting the text into sentences.
|
|
103
114
|
- `regex` and `PyYAML` for various utility functions.
|
|
104
115
|
|
|
116
|
+
### Use Cases
|
|
117
|
+
|
|
118
|
+
`split_markdown4gpt` is particularly useful in scenarios where you need to process large Markdown files with GPT models. For instance:
|
|
119
|
+
|
|
120
|
+
- **Text Generation**: If you're using a GPT model to generate text based on a large Markdown file, you can use `split_markdown4gpt` to split the file into manageable sections. This allows the GPT model to process the file in chunks, preventing memory overflow errors.
|
|
121
|
+
|
|
122
|
+
- **Data Preprocessing**: In machine learning projects, you often need to preprocess your data before feeding it into your model. If your data is in the form of large Markdown files, `split_markdown4gpt` can help you split these files into smaller sections based on the token limit of your model.
|
|
123
|
+
|
|
124
|
+
- **Document Analysis**: If you're analyzing large Markdown documents (e.g., extracting keywords, summarizing content), you can use `split_markdown4gpt` to break down the documents into smaller sections. This makes the analysis more manageable and efficient.
|
|
125
|
+
|
|
126
|
+
### Usage Examples
|
|
127
|
+
|
|
128
|
+
Here's an example of how you can use `split_markdown4gpt` in your Python code:
|
|
129
|
+
|
|
130
|
+
```python
|
|
131
|
+
from split_markdown4gpt import split
|
|
132
|
+
|
|
133
|
+
# Path to your large Markdown file
|
|
134
|
+
md_file_path = "path_to_your_file.md"
|
|
135
|
+
|
|
136
|
+
# Split the file into sections
|
|
137
|
+
sections = split(md_file_path, model="gpt-3.5-turbo", limit=4096)
|
|
138
|
+
|
|
139
|
+
# Process each section with your GPT model
|
|
140
|
+
for section in sections:
|
|
141
|
+
# Replace this with your GPT model's processing function
|
|
142
|
+
process_with_gpt_model(section)
|
|
143
|
+
```
|
|
144
|
+
|
|
145
|
+
In this example, `process_with_gpt_model` is a placeholder for your GPT model's processing function. You would replace this with the actual function you're using to process the text with your GPT model.
|
|
146
|
+
|
|
105
147
|
## Contributing
|
|
106
148
|
|
|
107
149
|
Contributions to `split_markdown4gpt` are welcome! Please open an issue or submit a pull request on the [GitHub repository](https://github.com/twardoch/split-markdown4gpt).
|
|
@@ -111,4 +153,3 @@ Contributions to `split_markdown4gpt` are welcome! Please open an issue or submi
|
|
|
111
153
|
- Copyright (c) 2023 Adam Twardoch
|
|
112
154
|
- Written with assistance from ChatGPT
|
|
113
155
|
- Licensed under the [Apache License 2.0](./LICENSE.txt)
|
|
114
|
-
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
`split_markdown4gpt` is a Python tool designed to split large Markdown files into smaller sections based on a specified token limit. This is particularly useful for processing large Markdown files with GPT models, as it allows the models to handle the data in manageable chunks.
|
|
4
4
|
|
|
5
|
-
_Version 1.0.
|
|
5
|
+
_Version 1.0.2_ (2023-06-18)
|
|
6
6
|
|
|
7
7
|
## Installation
|
|
8
8
|
|
|
@@ -12,7 +12,7 @@ You can install `split_markdown4gpt` via pip:
|
|
|
12
12
|
pip install split_markdown4gpt
|
|
13
13
|
```
|
|
14
14
|
|
|
15
|
-
## CLI
|
|
15
|
+
## CLI Usage
|
|
16
16
|
|
|
17
17
|
After installation, you can use the `mdsplit4gpt` command to split a Markdown file. Here's the basic syntax:
|
|
18
18
|
|
|
@@ -22,32 +22,41 @@ mdsplit4gpt path_to_your_file.md --model gpt-3.5-turbo --limit 4096 --separator
|
|
|
22
22
|
|
|
23
23
|
This command will split the Markdown file at `path_to_your_file.md` into sections, each containing no more than 4096 tokens (as counted by the `gpt-3.5-turbo` model). The sections will be separated by `=== SPLIT ===`.
|
|
24
24
|
|
|
25
|
-
All CLI options:
|
|
25
|
+
All CLI options:
|
|
26
26
|
|
|
27
27
|
```
|
|
28
28
|
NAME
|
|
29
|
-
mdsplit4gpt
|
|
29
|
+
mdsplit4gpt - Splits a Markdown file into sections according to GPT token size limits.
|
|
30
30
|
|
|
31
31
|
SYNOPSIS
|
|
32
32
|
mdsplit4gpt MD_PATH <flags>
|
|
33
33
|
|
|
34
|
+
DESCRIPTION
|
|
35
|
+
This tool loads a Markdown file, and splits its content into sections
|
|
36
|
+
that are within the specified token size limit using the desired GPT tokenizing model. The resulting
|
|
37
|
+
sections are then concatenated using the specified separator and returned as a single string.
|
|
38
|
+
|
|
34
39
|
POSITIONAL ARGUMENTS
|
|
35
40
|
MD_PATH
|
|
36
41
|
Type: Union
|
|
42
|
+
The path of the source Markdown file to be split.
|
|
37
43
|
|
|
38
44
|
FLAGS
|
|
39
45
|
-m, --model=MODEL
|
|
40
46
|
Type: str
|
|
41
47
|
Default: 'gpt-3.5-turbo'
|
|
48
|
+
The GPT tokenizer model to use for calculating token sizes. Defaults to "gpt-3.5-turbo".
|
|
42
49
|
-l, --limit=LIMIT
|
|
43
50
|
Type: Optional[int]
|
|
44
51
|
Default: None
|
|
52
|
+
The maximum number of GPT tokens allowed per section. Defaults to the model's maximum tokens.
|
|
45
53
|
-s, --separator=SEPARATOR
|
|
46
54
|
Type: str
|
|
47
55
|
Default: '=== SPLIT ==='
|
|
56
|
+
The string used to separate sections in the output. Defaults to "=== SPLIT ===".
|
|
48
57
|
```
|
|
49
58
|
|
|
50
|
-
## Python
|
|
59
|
+
## Python Usage
|
|
51
60
|
|
|
52
61
|
You can also use `split_markdown4gpt` in your Python code. Here's a basic example:
|
|
53
62
|
|
|
@@ -61,11 +70,13 @@ for section in sections:
|
|
|
61
70
|
|
|
62
71
|
This code does the same thing as the CLI command above, but in Python.
|
|
63
72
|
|
|
64
|
-
## How it
|
|
73
|
+
## How it Works
|
|
65
74
|
|
|
66
75
|
`split_markdown4gpt` works by tokenizing the input Markdown file using the specified GPT model's tokenizer (default is `gpt-3.5-turbo`). It then splits the file into sections, each containing no more than the specified token limit.
|
|
67
76
|
|
|
68
|
-
The splitting process respects the structure of the Markdown file. It will not split a section (as defined by Markdown headings) across multiple output sections unless the section is longer than the token limit. In that case, it
|
|
77
|
+
The splitting process respects the structure of the Markdown file. It will not split a section (as defined by Markdown headings) across multiple output sections unless the section is longer than the token limit. In that case, it
|
|
78
|
+
|
|
79
|
+
will split the section at the sentence level.
|
|
69
80
|
|
|
70
81
|
The tool uses several libraries to accomplish this:
|
|
71
82
|
|
|
@@ -76,6 +87,37 @@ The tool uses several libraries to accomplish this:
|
|
|
76
87
|
- `syntok` for splitting the text into sentences.
|
|
77
88
|
- `regex` and `PyYAML` for various utility functions.
|
|
78
89
|
|
|
90
|
+
### Use Cases
|
|
91
|
+
|
|
92
|
+
`split_markdown4gpt` is particularly useful in scenarios where you need to process large Markdown files with GPT models. For instance:
|
|
93
|
+
|
|
94
|
+
- **Text Generation**: If you're using a GPT model to generate text based on a large Markdown file, you can use `split_markdown4gpt` to split the file into manageable sections. This allows the GPT model to process the file in chunks, preventing memory overflow errors.
|
|
95
|
+
|
|
96
|
+
- **Data Preprocessing**: In machine learning projects, you often need to preprocess your data before feeding it into your model. If your data is in the form of large Markdown files, `split_markdown4gpt` can help you split these files into smaller sections based on the token limit of your model.
|
|
97
|
+
|
|
98
|
+
- **Document Analysis**: If you're analyzing large Markdown documents (e.g., extracting keywords, summarizing content), you can use `split_markdown4gpt` to break down the documents into smaller sections. This makes the analysis more manageable and efficient.
|
|
99
|
+
|
|
100
|
+
### Usage Examples
|
|
101
|
+
|
|
102
|
+
Here's an example of how you can use `split_markdown4gpt` in your Python code:
|
|
103
|
+
|
|
104
|
+
```python
|
|
105
|
+
from split_markdown4gpt import split
|
|
106
|
+
|
|
107
|
+
# Path to your large Markdown file
|
|
108
|
+
md_file_path = "path_to_your_file.md"
|
|
109
|
+
|
|
110
|
+
# Split the file into sections
|
|
111
|
+
sections = split(md_file_path, model="gpt-3.5-turbo", limit=4096)
|
|
112
|
+
|
|
113
|
+
# Process each section with your GPT model
|
|
114
|
+
for section in sections:
|
|
115
|
+
# Replace this with your GPT model's processing function
|
|
116
|
+
process_with_gpt_model(section)
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
In this example, `process_with_gpt_model` is a placeholder for your GPT model's processing function. You would replace this with the actual function you're using to process the text with your GPT model.
|
|
120
|
+
|
|
79
121
|
## Contributing
|
|
80
122
|
|
|
81
123
|
Contributions to `split_markdown4gpt` are welcome! Please open an issue or submit a pull request on the [GitHub repository](https://github.com/twardoch/split-markdown4gpt).
|
|
@@ -84,5 +126,4 @@ Contributions to `split_markdown4gpt` are welcome! Please open an issue or submi
|
|
|
84
126
|
|
|
85
127
|
- Copyright (c) 2023 Adam Twardoch
|
|
86
128
|
- Written with assistance from ChatGPT
|
|
87
|
-
- Licensed under the [Apache License 2.0](./LICENSE.txt)
|
|
88
|
-
|
|
129
|
+
- Licensed under the [Apache License 2.0](./LICENSE.txt)
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
from typing import Union
|
|
4
|
+
|
|
5
|
+
import fire
|
|
6
|
+
|
|
7
|
+
from .splitter import MarkdownLLMSplitter
|
|
8
|
+
|
|
9
|
+
def split_md_file(
|
|
10
|
+
md_path: Union[str, Path],
|
|
11
|
+
model: str = "gpt-3.5-turbo",
|
|
12
|
+
limit: int = None,
|
|
13
|
+
separator: str = "=== SPLIT ===",
|
|
14
|
+
) -> str:
|
|
15
|
+
"""
|
|
16
|
+
Splits a Markdown file into sections according to GPT token size limits.
|
|
17
|
+
|
|
18
|
+
This tool loads a Markdown file, and splits its content into sections
|
|
19
|
+
that are within the specified token size limit using the desired GPT tokenizing model. The resulting
|
|
20
|
+
sections are then concatenated using the specified separator and returned as a single string.
|
|
21
|
+
|
|
22
|
+
Args:
|
|
23
|
+
md_path: The path of the source Markdown file to be split.
|
|
24
|
+
model: The GPT tokenizer model to use for calculating token sizes. Defaults to "gpt-3.5-turbo".
|
|
25
|
+
limit: The maximum number of GPT tokens allowed per section. Defaults to the model's maximum tokens.
|
|
26
|
+
separator: The string used to separate sections in the output. Defaults to "=== SPLIT ===".
|
|
27
|
+
|
|
28
|
+
Returns:
|
|
29
|
+
A single string containing the Markdown content of the file, split into sections and separated
|
|
30
|
+
by the specified separator.
|
|
31
|
+
"""
|
|
32
|
+
md_path = Path(md_path).resolve()
|
|
33
|
+
md_splitter = MarkdownLLMSplitter(gptok_model=model, gptok_limit=limit)
|
|
34
|
+
return f"\n{separator}\n".join(md_splitter.split(md_path))
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def cli():
|
|
38
|
+
# Set the display function for 'fire' to print output without extra formatting.
|
|
39
|
+
fire.core.Display = lambda lines, out: print(*lines, file=out)
|
|
40
|
+
fire.Fire(split_md_file)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
if __name__ == "__main__":
|
|
44
|
+
cli()
|
|
@@ -33,6 +33,24 @@ Section = namedtuple("Section", ("md", "size"))
|
|
|
33
33
|
|
|
34
34
|
|
|
35
35
|
class MarkdownLLMSplitter:
|
|
36
|
+
"""
|
|
37
|
+
A class to split Markdown files into sections according to GPT token size limits.
|
|
38
|
+
|
|
39
|
+
Attributes:
|
|
40
|
+
gptoker: GPT tokenizer instance used to calculate token sizes.
|
|
41
|
+
gptok_limit: The maximum number of GPT tokens allowed per section.
|
|
42
|
+
md_meta: Metadata found in the source Markdown file.
|
|
43
|
+
md_str: The source Markdown string.
|
|
44
|
+
md_doc: The parsed source Markdown document as a mistletoe Document instance.
|
|
45
|
+
md_dict: A dictionary representing the structure of the Markdown document.
|
|
46
|
+
md_path: The absolute path to the source Markdown file.
|
|
47
|
+
md_sections: List of sections (named tuples) containing the Markdown content and its size in tokens.
|
|
48
|
+
|
|
49
|
+
Args:
|
|
50
|
+
gptok_model: The GPT tokenizer model to use for calculating token sizes, defaults to "gpt-3.5-turbo".
|
|
51
|
+
gptok_limit: The maximum number of GPT tokens allowed per section, defaults to the model's maximum tokens.
|
|
52
|
+
"""
|
|
53
|
+
|
|
36
54
|
def __init__(
|
|
37
55
|
self, gptok_model: str = "gpt-3.5-turbo", gptok_limit: int = None
|
|
38
56
|
) -> None:
|
|
@@ -46,20 +64,44 @@ class MarkdownLLMSplitter:
|
|
|
46
64
|
self.md_sections = []
|
|
47
65
|
|
|
48
66
|
def load_md_path(self, md_path: Union[Path, str]) -> None:
|
|
67
|
+
"""
|
|
68
|
+
Load a Markdown file from a file path.
|
|
69
|
+
|
|
70
|
+
Args:
|
|
71
|
+
md_path: The file path to the source Markdown file.
|
|
72
|
+
"""
|
|
49
73
|
self.md_path = Path(md_path).resolve()
|
|
50
74
|
with open(self.md_path, "r") as md_file:
|
|
51
75
|
self.load_md_file(md_file)
|
|
52
76
|
|
|
53
77
|
def load_md_file(self, md_file: TextIOWrapper) -> None:
|
|
78
|
+
"""
|
|
79
|
+
Load a Markdown file from a file-like object.
|
|
80
|
+
|
|
81
|
+
Args:
|
|
82
|
+
md_file: The file-like object containing the source Markdown content.
|
|
83
|
+
"""
|
|
54
84
|
file_content = md_file.read()
|
|
55
85
|
parsed = frontmatter.loads(file_content)
|
|
56
86
|
self.md_meta = parsed.metadata
|
|
57
87
|
self.md_str = parsed.content
|
|
58
88
|
|
|
59
89
|
def load_md_str(self, md_str: str) -> None:
|
|
90
|
+
"""
|
|
91
|
+
Load a Markdown file from a string.
|
|
92
|
+
|
|
93
|
+
Args:
|
|
94
|
+
md_str: The source Markdown content as a string.
|
|
95
|
+
"""
|
|
60
96
|
self.md_str = md_str
|
|
61
97
|
|
|
62
98
|
def load_md(self, md: Union[str, Path, TextIOWrapper]) -> None:
|
|
99
|
+
"""
|
|
100
|
+
Load a Markdown file from a string, file path, or file-like object.
|
|
101
|
+
|
|
102
|
+
Args:
|
|
103
|
+
md: The source Markdown content, can be a string, file path or file-like object.
|
|
104
|
+
"""
|
|
63
105
|
if hasattr(md, "read"):
|
|
64
106
|
self.load_md_file(md)
|
|
65
107
|
elif isinstance(md, Path):
|
|
@@ -71,9 +113,21 @@ class MarkdownLLMSplitter:
|
|
|
71
113
|
|
|
72
114
|
@lru_cache(maxsize=None)
|
|
73
115
|
def gpttok_size(self, text: str) -> int:
|
|
116
|
+
"""
|
|
117
|
+
Calculates the number of GPT tokens in a text string.
|
|
118
|
+
|
|
119
|
+
Args:
|
|
120
|
+
text: The text string to calculate the token size.
|
|
121
|
+
|
|
122
|
+
Returns:
|
|
123
|
+
The number of GPT tokens in the text string.
|
|
124
|
+
"""
|
|
74
125
|
return len(list(self.gptoker.encode(text)))
|
|
75
126
|
|
|
76
127
|
def build_md_dict(self) -> None:
|
|
128
|
+
"""
|
|
129
|
+
Builds a dictionary representing the structure of the source Markdown document.
|
|
130
|
+
"""
|
|
77
131
|
self.md_doc = Document(self.md_str)
|
|
78
132
|
current_level = 0
|
|
79
133
|
current_dict = self.md_dict
|
|
@@ -101,6 +155,15 @@ class MarkdownLLMSplitter:
|
|
|
101
155
|
current_dict[current_level].append(fragment_dict)
|
|
102
156
|
|
|
103
157
|
def calculate_sizes(self, md_dict: Dict) -> int:
|
|
158
|
+
"""
|
|
159
|
+
Recursively calculates the total size of GPT tokens in the provided Markdown dictionary.
|
|
160
|
+
|
|
161
|
+
Args:
|
|
162
|
+
md_dict: The Markdown dictionary to calculate token sizes.
|
|
163
|
+
|
|
164
|
+
Returns:
|
|
165
|
+
The total number of GPT tokens in the Markdown dictionary.
|
|
166
|
+
"""
|
|
104
167
|
total_size = 0
|
|
105
168
|
for key, value in md_dict.items():
|
|
106
169
|
if isinstance(value, list):
|
|
@@ -119,6 +182,18 @@ class MarkdownLLMSplitter:
|
|
|
119
182
|
current_size: int,
|
|
120
183
|
md_sections: List[Section],
|
|
121
184
|
) -> Tuple[List[str], int]:
|
|
185
|
+
"""
|
|
186
|
+
Processes an item in the Markdown dictionary and adds it to the appropriate section.
|
|
187
|
+
|
|
188
|
+
Args:
|
|
189
|
+
item: The Markdown item to process.
|
|
190
|
+
current_section: The current section being built as a list of strings.
|
|
191
|
+
current_size: The current size of the section in GPT tokens.
|
|
192
|
+
md_sections: The list of sections (named tuples) being built.
|
|
193
|
+
|
|
194
|
+
Returns:
|
|
195
|
+
A tuple containing the updated current_section and the current_size.
|
|
196
|
+
"""
|
|
122
197
|
if "children" in item:
|
|
123
198
|
for child in item["children"].values():
|
|
124
199
|
for subitem in child:
|
|
@@ -132,6 +207,16 @@ class MarkdownLLMSplitter:
|
|
|
132
207
|
return current_section, current_size
|
|
133
208
|
|
|
134
209
|
def prep_section(self, section_text: str, size: int = None) -> Section:
|
|
210
|
+
"""
|
|
211
|
+
Prepares a section by removing excessive newlines and calculating the section size if not provided.
|
|
212
|
+
|
|
213
|
+
Args:
|
|
214
|
+
section_text: The Markdown content of the section.
|
|
215
|
+
size: The size of the section in GPT tokens, defaults to None (automatically calculated).
|
|
216
|
+
|
|
217
|
+
Returns:
|
|
218
|
+
A Section named tuple containing the prepared Markdown content and its size in tokens.
|
|
219
|
+
"""
|
|
135
220
|
if not size:
|
|
136
221
|
size = self.gpttok_size(section_text)
|
|
137
222
|
return Section(RE_NEWLINES.sub("\n\n", section_text), size)
|
|
@@ -143,6 +228,18 @@ class MarkdownLLMSplitter:
|
|
|
143
228
|
current_size: int,
|
|
144
229
|
md_sections: List[Section],
|
|
145
230
|
) -> Tuple[List[str], int]:
|
|
231
|
+
"""
|
|
232
|
+
Processes a Markdown item and adds it to the appropriate section.
|
|
233
|
+
|
|
234
|
+
Args:
|
|
235
|
+
item: The Markdown item to process.
|
|
236
|
+
current_section: The current section being built as a list of strings.
|
|
237
|
+
current_size: The current size of the section in GPT tokens.
|
|
238
|
+
md_sections: The list of sections (named tuples) being built.
|
|
239
|
+
|
|
240
|
+
Returns:
|
|
241
|
+
A tuple containing the updated current_section and the current_size.
|
|
242
|
+
"""
|
|
146
243
|
if item["size"] > self.gptok_limit:
|
|
147
244
|
for syntok_paragraph in syntok_analyze(item["md"]):
|
|
148
245
|
for syntok_sentence in syntok_paragraph:
|
|
@@ -159,7 +256,7 @@ class MarkdownLLMSplitter:
|
|
|
159
256
|
else:
|
|
160
257
|
md_sections.append(
|
|
161
258
|
self.prep_section("".join(current_section), current_size)
|
|
162
|
-
|
|
259
|
+
)
|
|
163
260
|
current_section = [sentence]
|
|
164
261
|
current_size = sentence_size
|
|
165
262
|
current_section.append("\n\n")
|
|
@@ -175,6 +272,15 @@ class MarkdownLLMSplitter:
|
|
|
175
272
|
return current_section, current_size
|
|
176
273
|
|
|
177
274
|
def get_sections_from_md_dict_by_limit(self, md_dict: Dict) -> List[Section]:
|
|
275
|
+
"""
|
|
276
|
+
Builds the sections from the provided Markdown dictionary by fitting the content within token limits.
|
|
277
|
+
|
|
278
|
+
Args:
|
|
279
|
+
md_dict: The Markdown dictionary to build sections from.
|
|
280
|
+
|
|
281
|
+
Returns:
|
|
282
|
+
A list of sections (named tuples) containing the Markdown content and its size in tokens.
|
|
283
|
+
"""
|
|
178
284
|
md_sections = []
|
|
179
285
|
current_section = []
|
|
180
286
|
current_size = 0
|
|
@@ -193,48 +299,73 @@ class MarkdownLLMSplitter:
|
|
|
193
299
|
return md_sections
|
|
194
300
|
|
|
195
301
|
def build(self) -> None:
|
|
302
|
+
"""
|
|
303
|
+
Builds the sections by processing the loaded Markdown document.
|
|
304
|
+
"""
|
|
196
305
|
self.build_md_dict()
|
|
197
306
|
self.calculate_sizes(self.md_dict)
|
|
198
307
|
self.md_sections = self.get_sections_from_md_dict_by_limit(md_dict=self.md_dict)
|
|
199
308
|
|
|
200
309
|
def list_section_dicts(self) -> List[Dict[str, Union[str, int]]]:
|
|
310
|
+
"""
|
|
311
|
+
Returns a list of section dictionaries containing the Markdown content and its size.
|
|
312
|
+
|
|
313
|
+
Returns:
|
|
314
|
+
A list of dictionaries with keys 'md' and 'size'.
|
|
315
|
+
"""
|
|
201
316
|
return list(self.gen_section_dicts())
|
|
202
317
|
|
|
203
318
|
def gen_section_dicts(self) -> Generator[Dict[str, Union[str, int]], None, None]:
|
|
319
|
+
"""
|
|
320
|
+
Generator that yields section dictionaries containing the Markdown content and its size.
|
|
321
|
+
"""
|
|
204
322
|
return (
|
|
205
323
|
{"md": section.md, "size": section.size} for section in self.md_sections
|
|
206
324
|
)
|
|
207
325
|
|
|
208
326
|
def list_section_texts(self) -> List[str]:
|
|
327
|
+
"""
|
|
328
|
+
Returns a list of section texts containing the Markdown content.
|
|
329
|
+
|
|
330
|
+
Returns:
|
|
331
|
+
A list of strings with the Markdown content of each section.
|
|
332
|
+
"""
|
|
209
333
|
return list(self.gen_section_texts())
|
|
210
334
|
|
|
211
335
|
def gen_section_texts(self) -> Generator[str, None, None]:
|
|
336
|
+
"""
|
|
337
|
+
Generator that yields the Markdown content of each section.
|
|
338
|
+
"""
|
|
212
339
|
return (section.md for section in self.md_sections)
|
|
213
340
|
|
|
214
|
-
def summarize(self):
|
|
215
|
-
for section in self.md_sections:
|
|
216
|
-
section_size = self.gpttok_size(section)
|
|
217
|
-
original_summary = summarize_markdown(markdown=section).replace(
|
|
218
|
-
"[NEW SUMMARY:] ", ""
|
|
219
|
-
)
|
|
220
|
-
original_summary_size = self.gpttok_size(original_summary)
|
|
221
|
-
percentage = int(original_summary_size / section_size * 100)
|
|
222
|
-
reviewed_summary = review_summary_of_reference_markdown(
|
|
223
|
-
provided_summary=original_summary, reference_markdown=section
|
|
224
|
-
).replace("[NEW SUMMARY:] ", "")
|
|
225
|
-
print(
|
|
226
|
-
f"""\n\n<!-- {original_summary_size} / {section_size} = {percentage}% {original_summary} -->\n"""
|
|
227
|
-
)
|
|
228
|
-
print(f"""\n??? optional-class "Summary">""")
|
|
229
|
-
for line in reviewed_summary.splitlines():
|
|
230
|
-
print(f""" {line}""")
|
|
231
|
-
print(f"\n{section}\n")
|
|
232
|
-
|
|
233
341
|
def split(self, md: Union[str, Path, TextIOWrapper]) -> List[str]:
|
|
342
|
+
"""
|
|
343
|
+
Splits the loaded Markdown document into sections according to the GPT token size limits.
|
|
344
|
+
|
|
345
|
+
Args:
|
|
346
|
+
md: The source Markdown content, can be a string, file path or file-like object.
|
|
347
|
+
|
|
348
|
+
Returns:
|
|
349
|
+
A list of strings with the Markdown content of each section.
|
|
350
|
+
"""
|
|
234
351
|
self.load_md(md)
|
|
235
352
|
self.build()
|
|
236
353
|
return self.list_section_texts()
|
|
237
354
|
|
|
238
|
-
|
|
355
|
+
|
|
356
|
+
def split(
|
|
357
|
+
md: Union[str, Path, TextIOWrapper], model: str = "gpt-3.5-turbo", limit: int = None
|
|
358
|
+
) -> List[str]:
|
|
359
|
+
"""
|
|
360
|
+
A utility function to split a Markdown document into sections according to GPT token size limits.
|
|
361
|
+
|
|
362
|
+
Args:
|
|
363
|
+
md: The source Markdown content, can be a string, file path or file-like object.
|
|
364
|
+
model: The GPT tokenizer model to use for calculating token sizes, defaults to "gpt-3.5-turbo".
|
|
365
|
+
limit: The maximum number of GPT tokens allowed per section, defaults to the model's maximum tokens.
|
|
366
|
+
|
|
367
|
+
Returns:
|
|
368
|
+
A list of strings with the Markdown content of each section.
|
|
369
|
+
"""
|
|
239
370
|
md_splitter = MarkdownLLMSplitter(gptok_model=model, gptok_limit=limit)
|
|
240
371
|
return md_splitter.split(md)
|
{split_markdown4gpt-1.0.1 → split_markdown4gpt-1.0.2}/src/split_markdown4gpt.egg-info/PKG-INFO
RENAMED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: split-markdown4gpt
|
|
3
|
-
Version: 1.0.
|
|
3
|
+
Version: 1.0.2
|
|
4
4
|
Summary: A Python tool for splitting large Markdown files into smaller sections based on a specified token limit. This is particularly useful for processing large Markdown files with GPT models, as it allows the models to handle the data in manageable chunks.
|
|
5
5
|
Home-page: https://github.com/twardoch/split-markdown4gpt
|
|
6
6
|
Author: Adam Twardoch
|
|
@@ -28,7 +28,7 @@ License-File: LICENSE.txt
|
|
|
28
28
|
|
|
29
29
|
`split_markdown4gpt` is a Python tool designed to split large Markdown files into smaller sections based on a specified token limit. This is particularly useful for processing large Markdown files with GPT models, as it allows the models to handle the data in manageable chunks.
|
|
30
30
|
|
|
31
|
-
_Version 1.0.
|
|
31
|
+
_Version 1.0.2_ (2023-06-18)
|
|
32
32
|
|
|
33
33
|
## Installation
|
|
34
34
|
|
|
@@ -38,7 +38,7 @@ You can install `split_markdown4gpt` via pip:
|
|
|
38
38
|
pip install split_markdown4gpt
|
|
39
39
|
```
|
|
40
40
|
|
|
41
|
-
## CLI
|
|
41
|
+
## CLI Usage
|
|
42
42
|
|
|
43
43
|
After installation, you can use the `mdsplit4gpt` command to split a Markdown file. Here's the basic syntax:
|
|
44
44
|
|
|
@@ -48,32 +48,41 @@ mdsplit4gpt path_to_your_file.md --model gpt-3.5-turbo --limit 4096 --separator
|
|
|
48
48
|
|
|
49
49
|
This command will split the Markdown file at `path_to_your_file.md` into sections, each containing no more than 4096 tokens (as counted by the `gpt-3.5-turbo` model). The sections will be separated by `=== SPLIT ===`.
|
|
50
50
|
|
|
51
|
-
All CLI options:
|
|
51
|
+
All CLI options:
|
|
52
52
|
|
|
53
53
|
```
|
|
54
54
|
NAME
|
|
55
|
-
mdsplit4gpt
|
|
55
|
+
mdsplit4gpt - Splits a Markdown file into sections according to GPT token size limits.
|
|
56
56
|
|
|
57
57
|
SYNOPSIS
|
|
58
58
|
mdsplit4gpt MD_PATH <flags>
|
|
59
59
|
|
|
60
|
+
DESCRIPTION
|
|
61
|
+
This tool loads a Markdown file, and splits its content into sections
|
|
62
|
+
that are within the specified token size limit using the desired GPT tokenizing model. The resulting
|
|
63
|
+
sections are then concatenated using the specified separator and returned as a single string.
|
|
64
|
+
|
|
60
65
|
POSITIONAL ARGUMENTS
|
|
61
66
|
MD_PATH
|
|
62
67
|
Type: Union
|
|
68
|
+
The path of the source Markdown file to be split.
|
|
63
69
|
|
|
64
70
|
FLAGS
|
|
65
71
|
-m, --model=MODEL
|
|
66
72
|
Type: str
|
|
67
73
|
Default: 'gpt-3.5-turbo'
|
|
74
|
+
The GPT tokenizer model to use for calculating token sizes. Defaults to "gpt-3.5-turbo".
|
|
68
75
|
-l, --limit=LIMIT
|
|
69
76
|
Type: Optional[int]
|
|
70
77
|
Default: None
|
|
78
|
+
The maximum number of GPT tokens allowed per section. Defaults to the model's maximum tokens.
|
|
71
79
|
-s, --separator=SEPARATOR
|
|
72
80
|
Type: str
|
|
73
81
|
Default: '=== SPLIT ==='
|
|
82
|
+
The string used to separate sections in the output. Defaults to "=== SPLIT ===".
|
|
74
83
|
```
|
|
75
84
|
|
|
76
|
-
## Python
|
|
85
|
+
## Python Usage
|
|
77
86
|
|
|
78
87
|
You can also use `split_markdown4gpt` in your Python code. Here's a basic example:
|
|
79
88
|
|
|
@@ -87,11 +96,13 @@ for section in sections:
|
|
|
87
96
|
|
|
88
97
|
This code does the same thing as the CLI command above, but in Python.
|
|
89
98
|
|
|
90
|
-
## How it
|
|
99
|
+
## How it Works
|
|
91
100
|
|
|
92
101
|
`split_markdown4gpt` works by tokenizing the input Markdown file using the specified GPT model's tokenizer (default is `gpt-3.5-turbo`). It then splits the file into sections, each containing no more than the specified token limit.
|
|
93
102
|
|
|
94
|
-
The splitting process respects the structure of the Markdown file. It will not split a section (as defined by Markdown headings) across multiple output sections unless the section is longer than the token limit. In that case, it
|
|
103
|
+
The splitting process respects the structure of the Markdown file. It will not split a section (as defined by Markdown headings) across multiple output sections unless the section is longer than the token limit. In that case, it
|
|
104
|
+
|
|
105
|
+
will split the section at the sentence level.
|
|
95
106
|
|
|
96
107
|
The tool uses several libraries to accomplish this:
|
|
97
108
|
|
|
@@ -102,6 +113,37 @@ The tool uses several libraries to accomplish this:
|
|
|
102
113
|
- `syntok` for splitting the text into sentences.
|
|
103
114
|
- `regex` and `PyYAML` for various utility functions.
|
|
104
115
|
|
|
116
|
+
### Use Cases
|
|
117
|
+
|
|
118
|
+
`split_markdown4gpt` is particularly useful in scenarios where you need to process large Markdown files with GPT models. For instance:
|
|
119
|
+
|
|
120
|
+
- **Text Generation**: If you're using a GPT model to generate text based on a large Markdown file, you can use `split_markdown4gpt` to split the file into manageable sections. This allows the GPT model to process the file in chunks, preventing memory overflow errors.
|
|
121
|
+
|
|
122
|
+
- **Data Preprocessing**: In machine learning projects, you often need to preprocess your data before feeding it into your model. If your data is in the form of large Markdown files, `split_markdown4gpt` can help you split these files into smaller sections based on the token limit of your model.
|
|
123
|
+
|
|
124
|
+
- **Document Analysis**: If you're analyzing large Markdown documents (e.g., extracting keywords, summarizing content), you can use `split_markdown4gpt` to break down the documents into smaller sections. This makes the analysis more manageable and efficient.
|
|
125
|
+
|
|
126
|
+
### Usage Examples
|
|
127
|
+
|
|
128
|
+
Here's an example of how you can use `split_markdown4gpt` in your Python code:
|
|
129
|
+
|
|
130
|
+
```python
|
|
131
|
+
from split_markdown4gpt import split
|
|
132
|
+
|
|
133
|
+
# Path to your large Markdown file
|
|
134
|
+
md_file_path = "path_to_your_file.md"
|
|
135
|
+
|
|
136
|
+
# Split the file into sections
|
|
137
|
+
sections = split(md_file_path, model="gpt-3.5-turbo", limit=4096)
|
|
138
|
+
|
|
139
|
+
# Process each section with your GPT model
|
|
140
|
+
for section in sections:
|
|
141
|
+
# Replace this with your GPT model's processing function
|
|
142
|
+
process_with_gpt_model(section)
|
|
143
|
+
```
|
|
144
|
+
|
|
145
|
+
In this example, `process_with_gpt_model` is a placeholder for your GPT model's processing function. You would replace this with the actual function you're using to process the text with your GPT model.
|
|
146
|
+
|
|
105
147
|
## Contributing
|
|
106
148
|
|
|
107
149
|
Contributions to `split_markdown4gpt` are welcome! Please open an issue or submit a pull request on the [GitHub repository](https://github.com/twardoch/split-markdown4gpt).
|
|
@@ -111,4 +153,3 @@ Contributions to `split_markdown4gpt` are welcome! Please open an issue or submi
|
|
|
111
153
|
- Copyright (c) 2023 Adam Twardoch
|
|
112
154
|
- Written with assistance from ChatGPT
|
|
113
155
|
- Licensed under the [Apache License 2.0](./LICENSE.txt)
|
|
114
|
-
|
|
@@ -1,27 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env python3
|
|
2
|
-
from pathlib import Path
|
|
3
|
-
from typing import Union
|
|
4
|
-
|
|
5
|
-
import fire
|
|
6
|
-
|
|
7
|
-
from .splitter import MarkdownLLMSplitter
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
def split_md_file(
|
|
11
|
-
md_path: Union[str, Path],
|
|
12
|
-
model: str = "gpt-3.5-turbo",
|
|
13
|
-
limit: int = None,
|
|
14
|
-
separator: str = "=== SPLIT ===",
|
|
15
|
-
) -> str:
|
|
16
|
-
md_path = Path(md_path).resolve()
|
|
17
|
-
md_splitter = MarkdownLLMSplitter(gptok_model=model, gptok_limit=limit)
|
|
18
|
-
return f"\n{separator}\n".join(md_splitter.split(md_path))
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
def cli():
|
|
22
|
-
fire.core.Display = lambda lines, out: print(*lines, file=out)
|
|
23
|
-
fire.Fire(split_md_file)
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
if __name__ == "__main__":
|
|
27
|
-
cli()
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{split_markdown4gpt-1.0.1 → split_markdown4gpt-1.0.2}/src/split_markdown4gpt.egg-info/SOURCES.txt
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{split_markdown4gpt-1.0.1 → split_markdown4gpt-1.0.2}/src/split_markdown4gpt.egg-info/not-zip-safe
RENAMED
|
File without changes
|
{split_markdown4gpt-1.0.1 → split_markdown4gpt-1.0.2}/src/split_markdown4gpt.egg-info/requires.txt
RENAMED
|
File without changes
|
{split_markdown4gpt-1.0.1 → split_markdown4gpt-1.0.2}/src/split_markdown4gpt.egg-info/top_level.txt
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|