cerebrate-file 1.0.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
__pycache__/
|
|
2
|
+
__pypackages__/
|
|
3
|
+
__version__.py
|
|
4
|
+
_version.py
|
|
5
|
+
._*
|
|
6
|
+
.cache
|
|
7
|
+
.coverage
|
|
8
|
+
.coverage.*
|
|
9
|
+
.dmypy.json
|
|
10
|
+
.DS_Store
|
|
11
|
+
.DS_Store?
|
|
12
|
+
.eggs/
|
|
13
|
+
.env
|
|
14
|
+
.hatch_build/
|
|
15
|
+
.hypothesis/
|
|
16
|
+
.idea/
|
|
17
|
+
.installed.cfg
|
|
18
|
+
.ipynb_checkpoints
|
|
19
|
+
.mypy_cache/
|
|
20
|
+
.nox/
|
|
21
|
+
.pdm.toml
|
|
22
|
+
.pybuilder/
|
|
23
|
+
.pyre/
|
|
24
|
+
.pytest_cache/
|
|
25
|
+
.Python
|
|
26
|
+
.pytype/
|
|
27
|
+
.ropeproject
|
|
28
|
+
.ruff_cache/
|
|
29
|
+
.scrapy
|
|
30
|
+
.Spotlight-V100
|
|
31
|
+
.spyderproject
|
|
32
|
+
.spyproject
|
|
33
|
+
.tox/
|
|
34
|
+
.Trashes
|
|
35
|
+
.uv
|
|
36
|
+
.venv
|
|
37
|
+
.vscode/
|
|
38
|
+
.webassets-cache
|
|
39
|
+
*.cover
|
|
40
|
+
*.egg
|
|
41
|
+
*.egg-info/
|
|
42
|
+
*.log
|
|
43
|
+
*.manifest
|
|
44
|
+
*.mo
|
|
45
|
+
*.pot
|
|
46
|
+
*.py,cover
|
|
47
|
+
*.py[cod]
|
|
48
|
+
*.sage.py
|
|
49
|
+
*.so
|
|
50
|
+
*.spec
|
|
51
|
+
*$py.class
|
|
52
|
+
/site
|
|
53
|
+
build/
|
|
54
|
+
celerybeat-schedule
|
|
55
|
+
celerybeat.pid
|
|
56
|
+
cover/
|
|
57
|
+
coverage.xml
|
|
58
|
+
cython_debug/
|
|
59
|
+
db.sqlite3
|
|
60
|
+
db.sqlite3-journal
|
|
61
|
+
develop-eggs/
|
|
62
|
+
dist/
|
|
63
|
+
dmypy.json
|
|
64
|
+
docs/_build/
|
|
65
|
+
downloads/
|
|
66
|
+
eggs/
|
|
67
|
+
ehthumbs.db
|
|
68
|
+
env.bak/
|
|
69
|
+
env/
|
|
70
|
+
ENV/
|
|
71
|
+
external/
|
|
72
|
+
htmlcov/
|
|
73
|
+
instance/
|
|
74
|
+
ipython_config.py
|
|
75
|
+
lib/
|
|
76
|
+
lib64/
|
|
77
|
+
local_settings.py
|
|
78
|
+
MANIFEST
|
|
79
|
+
nosetests.xml
|
|
80
|
+
parts/
|
|
81
|
+
pip-delete-this-directory.txt
|
|
82
|
+
pip-log.txt
|
|
83
|
+
profile_default/
|
|
84
|
+
sdist/
|
|
85
|
+
share/python-wheels/
|
|
86
|
+
target/
|
|
87
|
+
testdata
|
|
88
|
+
Thumbs.db
|
|
89
|
+
var/
|
|
90
|
+
venv.bak/
|
|
91
|
+
venv/
|
|
92
|
+
wheels/
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 Adam Twardoch
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: cerebrate-file
|
|
3
|
+
Version: 1.0.4
|
|
4
|
+
Project-URL: Documentation, https://github.com/twardoch/cerebrate-file#readme
|
|
5
|
+
Project-URL: Issues, https://github.com/twardoch/cerebrate-file/issues
|
|
6
|
+
Project-URL: Source, https://github.com/twardoch/cerebrate-file
|
|
7
|
+
Author-email: Adam Twardoch <adam+github@twardoch.com>
|
|
8
|
+
License: MIT
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Classifier: Development Status :: 4 - Beta
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
13
|
+
Classifier: Operating System :: OS Independent
|
|
14
|
+
Classifier: Programming Language :: Python
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Programming Language :: Python :: Implementation :: CPython
|
|
19
|
+
Classifier: Programming Language :: Python :: Implementation :: PyPy
|
|
20
|
+
Requires-Python: >=3.10
|
|
21
|
+
Requires-Dist: cerebras-cloud-sdk>=1.0.0
|
|
22
|
+
Requires-Dist: fire>=0.6.0
|
|
23
|
+
Requires-Dist: loguru>=0.7.0
|
|
24
|
+
Requires-Dist: python-dotenv>=1.0.0
|
|
25
|
+
Requires-Dist: python-frontmatter>=1.1.0
|
|
26
|
+
Requires-Dist: qwen-tokenizer>=0.0.8
|
|
27
|
+
Requires-Dist: semantic-text-splitter>=0.13.0
|
|
28
|
+
Requires-Dist: tenacity>=8.2.0
|
|
29
|
+
Requires-Dist: tqdm>=4.66.0
|
|
30
|
+
Provides-Extra: all
|
|
31
|
+
Provides-Extra: dev
|
|
32
|
+
Requires-Dist: absolufy-imports>=0.3.1; extra == 'dev'
|
|
33
|
+
Requires-Dist: isort>=6.0.1; extra == 'dev'
|
|
34
|
+
Requires-Dist: mypy>=1.15.0; extra == 'dev'
|
|
35
|
+
Requires-Dist: pre-commit>=4.1.0; extra == 'dev'
|
|
36
|
+
Requires-Dist: pyupgrade>=3.19.1; extra == 'dev'
|
|
37
|
+
Requires-Dist: ruff>=0.9.7; extra == 'dev'
|
|
38
|
+
Provides-Extra: docs
|
|
39
|
+
Requires-Dist: myst-parser>=3.0.0; extra == 'docs'
|
|
40
|
+
Requires-Dist: sphinx-autodoc-typehints>=2.0.0; extra == 'docs'
|
|
41
|
+
Requires-Dist: sphinx-rtd-theme>=2.0.0; extra == 'docs'
|
|
42
|
+
Requires-Dist: sphinx>=7.2.6; extra == 'docs'
|
|
43
|
+
Provides-Extra: test
|
|
44
|
+
Requires-Dist: coverage[toml]>=7.6.12; extra == 'test'
|
|
45
|
+
Requires-Dist: pytest-asyncio>=0.25.3; extra == 'test'
|
|
46
|
+
Requires-Dist: pytest-benchmark[histogram]>=5.1.0; extra == 'test'
|
|
47
|
+
Requires-Dist: pytest-cov>=6.0.0; extra == 'test'
|
|
48
|
+
Requires-Dist: pytest-xdist>=3.6.1; extra == 'test'
|
|
49
|
+
Requires-Dist: pytest>=8.3.4; extra == 'test'
|
|
50
|
+
Description-Content-Type: text/markdown
|
|
51
|
+
|
|
52
|
+
---
|
|
53
|
+
this_file: README.md
|
|
54
|
+
---
|
|
55
|
+
# cereproc.py
|
|
56
|
+
|
|
57
|
+
`old/cereproc.py` is a single-file utility that splits oversized documents into
|
|
58
|
+
Cerebras-friendly chunks, calls the `qwen-3-coder-480b` chat completion model
|
|
59
|
+
for each chunk, and stitches the results back together while keeping context
|
|
60
|
+
intact.
|
|
61
|
+
|
|
62
|
+
## Quick Start
|
|
63
|
+
|
|
64
|
+
```bash
|
|
65
|
+
export CEREBRAS_API_KEY="csk-..."
|
|
66
|
+
uv run old/cereproc.py --input_data document.md --output_data document.out.md
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
Add optional guidance by supplying an inline prompt or a separate instructions
|
|
70
|
+
file:
|
|
71
|
+
|
|
72
|
+
```bash
|
|
73
|
+
uv run old/cereproc.py \
|
|
74
|
+
--input_data huge.md \
|
|
75
|
+
--file_prompt prompts/style.md \
|
|
76
|
+
--prompt "Write concise technical summaries." \
|
|
77
|
+
--data_format code \
|
|
78
|
+
--chunk_size 28000 \
|
|
79
|
+
--sample_size 256 \
|
|
80
|
+
--verbose
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
## CLI Flags
|
|
84
|
+
|
|
85
|
+
- `--input_data PATH` (required) Text/Markdown/code file to process.
|
|
86
|
+
- `--output_data PATH` Destination file (defaults to the input path).
|
|
87
|
+
- `--file_prompt PATH` Load reusable instructions; appended before the inline prompt.
|
|
88
|
+
- `--prompt TEXT` Freeform instructions appended after the file prompt.
|
|
89
|
+
- `--chunk_size INT` Target chunk size in tokens (default `32000`).
|
|
90
|
+
- `--data_format text|semantic|markdown|code` Chunking strategy (default `markdown`).
|
|
91
|
+
- `--sample_size INT` Continuity example size in tokens (default `200`, use `0` to disable).
|
|
92
|
+
- `--max_tokens_ratio INT` Completion budget as `%` of chunk tokens (default `100`).
|
|
93
|
+
- `--temp FLOAT` and `--top_p FLOAT` Sampling controls (defaults `0.7` / `0.8`).
|
|
94
|
+
- `--model TEXT` Cerebras model name override (default `qwen-3-coder-480b`).
|
|
95
|
+
- `--verbose` Enable detailed logging and chunk previews.
|
|
96
|
+
- `--dry_run` Inspect chunking and request envelopes without calling the API.
|
|
97
|
+
- `--explain` Parse Markdown frontmatter, ensure required metadata fields, and
|
|
98
|
+
ask the model to fill gaps before processing.
|
|
99
|
+
|
|
100
|
+
## Processing Pipeline
|
|
101
|
+
|
|
102
|
+
1. Load `.env` values and validate `CEREBRAS_API_KEY` plus CLI arguments.
|
|
103
|
+
2. Build a base prompt from `--file_prompt` and `--prompt` (always separated by
|
|
104
|
+
two newlines) and count its tokens.
|
|
105
|
+
3. Read the input file (frontmatter preserved) and optionally parse metadata
|
|
106
|
+
when `--explain` is active.
|
|
107
|
+
4. Chunk the body using the selected strategy:
|
|
108
|
+
- `text`: greedy line-based splitting.
|
|
109
|
+
- `semantic`: paragraph-aware via `semantic-text-splitter`.
|
|
110
|
+
- `markdown`: structure-aware Markdown splitter.
|
|
111
|
+
- `code`: regex-guided boundaries for source files.
|
|
112
|
+
5. For each chunk, optionally blend in continuity examples drawn from the
|
|
113
|
+
previous request/response pair (`--sample_size` tokens each way), truncated to
|
|
114
|
+
stay within the 131K-token context budget.
|
|
115
|
+
6. Stream completions from Cerebras with adaptive rate-limit backoff and retry
|
|
116
|
+
(`tenacity`) on transient failures.
|
|
117
|
+
7. Write the concatenated result atomically, preserving or updating frontmatter
|
|
118
|
+
when `--explain` metadata is present.
|
|
119
|
+
|
|
120
|
+
## Explain Mode Metadata
|
|
121
|
+
|
|
122
|
+
When `--explain` is set, the script expects frontmatter containing
|
|
123
|
+
`title`, `author`, `id`, `type`, and `date`. Missing keys trigger a structured
|
|
124
|
+
JSON request to the model that fills only the absent values. Dry-run mode skips
|
|
125
|
+
this network call while still showing parsed metadata.
|
|
126
|
+
|
|
127
|
+
## Dry-Run Workflow
|
|
128
|
+
|
|
129
|
+
Use `--dry_run` to sanity-check chunk sizes, token budgets, and message shapes
|
|
130
|
+
without spending quota. The script prints the first two chunk envelopes, token
|
|
131
|
+
counts, and previews, then exits before creating the Cerebras client.
|
|
132
|
+
|
|
133
|
+
## Dependencies
|
|
134
|
+
|
|
135
|
+
Install requirements with `uv` (or your preferred tool):
|
|
136
|
+
|
|
137
|
+
- `fire`
|
|
138
|
+
- `loguru`
|
|
139
|
+
- `python-dotenv`
|
|
140
|
+
- `tenacity`
|
|
141
|
+
- `cerebras-cloud-sdk`
|
|
142
|
+
- `semantic-text-splitter`
|
|
143
|
+
- `qwen-tokenizer`
|
|
144
|
+
- `tqdm`
|
|
145
|
+
- `python-frontmatter`
|
|
146
|
+
|
|
147
|
+
## Environment
|
|
148
|
+
|
|
149
|
+
Set `CEREBRAS_API_KEY` before running. The utility warns on placeholder keys
|
|
150
|
+
and gently validates formatting. Use `--verbose` to surface additional runtime
|
|
151
|
+
information and rate-limit headers.
|
|
152
|
+
|
|
153
|
+
## Testing Tips
|
|
154
|
+
|
|
155
|
+
Run with `--dry_run` for fast validation, then process a short sample file in
|
|
156
|
+
`--verbose` mode to observe continuity handling and output statistics before you
|
|
157
|
+
launch against larger documents.
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
---
|
|
2
|
+
this_file: README.md
|
|
3
|
+
---
|
|
4
|
+
# cereproc.py
|
|
5
|
+
|
|
6
|
+
`old/cereproc.py` is a single-file utility that splits oversized documents into
|
|
7
|
+
Cerebras-friendly chunks, calls the `qwen-3-coder-480b` chat completion model
|
|
8
|
+
for each chunk, and stitches the results back together while keeping context
|
|
9
|
+
intact.
|
|
10
|
+
|
|
11
|
+
## Quick Start
|
|
12
|
+
|
|
13
|
+
```bash
|
|
14
|
+
export CEREBRAS_API_KEY="csk-..."
|
|
15
|
+
uv run old/cereproc.py --input_data document.md --output_data document.out.md
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
Add optional guidance by supplying an inline prompt or a separate instructions
|
|
19
|
+
file:
|
|
20
|
+
|
|
21
|
+
```bash
|
|
22
|
+
uv run old/cereproc.py \
|
|
23
|
+
--input_data huge.md \
|
|
24
|
+
--file_prompt prompts/style.md \
|
|
25
|
+
--prompt "Write concise technical summaries." \
|
|
26
|
+
--data_format code \
|
|
27
|
+
--chunk_size 28000 \
|
|
28
|
+
--sample_size 256 \
|
|
29
|
+
--verbose
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
## CLI Flags
|
|
33
|
+
|
|
34
|
+
- `--input_data PATH` (required) Text/Markdown/code file to process.
|
|
35
|
+
- `--output_data PATH` Destination file (defaults to the input path).
|
|
36
|
+
- `--file_prompt PATH` Load reusable instructions; appended before the inline prompt.
|
|
37
|
+
- `--prompt TEXT` Freeform instructions appended after the file prompt.
|
|
38
|
+
- `--chunk_size INT` Target chunk size in tokens (default `32000`).
|
|
39
|
+
- `--data_format text|semantic|markdown|code` Chunking strategy (default `markdown`).
|
|
40
|
+
- `--sample_size INT` Continuity example size in tokens (default `200`, use `0` to disable).
|
|
41
|
+
- `--max_tokens_ratio INT` Completion budget as `%` of chunk tokens (default `100`).
|
|
42
|
+
- `--temp FLOAT` and `--top_p FLOAT` Sampling controls (defaults `0.7` / `0.8`).
|
|
43
|
+
- `--model TEXT` Cerebras model name override (default `qwen-3-coder-480b`).
|
|
44
|
+
- `--verbose` Enable detailed logging and chunk previews.
|
|
45
|
+
- `--dry_run` Inspect chunking and request envelopes without calling the API.
|
|
46
|
+
- `--explain` Parse Markdown frontmatter, ensure required metadata fields, and
|
|
47
|
+
ask the model to fill gaps before processing.
|
|
48
|
+
|
|
49
|
+
## Processing Pipeline
|
|
50
|
+
|
|
51
|
+
1. Load `.env` values and validate `CEREBRAS_API_KEY` plus CLI arguments.
|
|
52
|
+
2. Build a base prompt from `--file_prompt` and `--prompt` (always separated by
|
|
53
|
+
two newlines) and count its tokens.
|
|
54
|
+
3. Read the input file (frontmatter preserved) and optionally parse metadata
|
|
55
|
+
when `--explain` is active.
|
|
56
|
+
4. Chunk the body using the selected strategy:
|
|
57
|
+
- `text`: greedy line-based splitting.
|
|
58
|
+
- `semantic`: paragraph-aware via `semantic-text-splitter`.
|
|
59
|
+
- `markdown`: structure-aware Markdown splitter.
|
|
60
|
+
- `code`: regex-guided boundaries for source files.
|
|
61
|
+
5. For each chunk, optionally blend in continuity examples drawn from the
|
|
62
|
+
previous request/response pair (`--sample_size` tokens each way), truncated to
|
|
63
|
+
stay within the 131K-token context budget.
|
|
64
|
+
6. Stream completions from Cerebras with adaptive rate-limit backoff and retry
|
|
65
|
+
(`tenacity`) on transient failures.
|
|
66
|
+
7. Write the concatenated result atomically, preserving or updating frontmatter
|
|
67
|
+
when `--explain` metadata is present.
|
|
68
|
+
|
|
69
|
+
## Explain Mode Metadata
|
|
70
|
+
|
|
71
|
+
When `--explain` is set, the script expects frontmatter containing
|
|
72
|
+
`title`, `author`, `id`, `type`, and `date`. Missing keys trigger a structured
|
|
73
|
+
JSON request to the model that fills only the absent values. Dry-run mode skips
|
|
74
|
+
this network call while still showing parsed metadata.
|
|
75
|
+
|
|
76
|
+
## Dry-Run Workflow
|
|
77
|
+
|
|
78
|
+
Use `--dry_run` to sanity-check chunk sizes, token budgets, and message shapes
|
|
79
|
+
without spending quota. The script prints the first two chunk envelopes, token
|
|
80
|
+
counts, and previews, then exits before creating the Cerebras client.
|
|
81
|
+
|
|
82
|
+
## Dependencies
|
|
83
|
+
|
|
84
|
+
Install requirements with `uv` (or your preferred tool):
|
|
85
|
+
|
|
86
|
+
- `fire`
|
|
87
|
+
- `loguru`
|
|
88
|
+
- `python-dotenv`
|
|
89
|
+
- `tenacity`
|
|
90
|
+
- `cerebras-cloud-sdk`
|
|
91
|
+
- `semantic-text-splitter`
|
|
92
|
+
- `qwen-tokenizer`
|
|
93
|
+
- `tqdm`
|
|
94
|
+
- `python-frontmatter`
|
|
95
|
+
|
|
96
|
+
## Environment
|
|
97
|
+
|
|
98
|
+
Set `CEREBRAS_API_KEY` before running. The utility warns on placeholder keys
|
|
99
|
+
and gently validates formatting. Use `--verbose` to surface additional runtime
|
|
100
|
+
information and rate-limit headers.
|
|
101
|
+
|
|
102
|
+
## Testing Tips
|
|
103
|
+
|
|
104
|
+
Run with `--dry_run` for fast validation, then process a short sample file in
|
|
105
|
+
`--verbose` mode to observe continuity handling and output statistics before you
|
|
106
|
+
launch against larger documents.
|
|
@@ -0,0 +1,423 @@
|
|
|
1
|
+
# this_file: pyproject.toml
|
|
2
|
+
#==============================================================================
|
|
3
|
+
# CEREBRATE-FILE PACKAGE CONFIGURATION
|
|
4
|
+
# This pyproject.toml defines the package metadata, dependencies, build system,
|
|
5
|
+
# and development environment for the cerebrate-file package.
|
|
6
|
+
#==============================================================================
|
|
7
|
+
|
|
8
|
+
#------------------------------------------------------------------------------
|
|
9
|
+
# PROJECT METADATA
|
|
10
|
+
# Core package information used by PyPI and package managers.
|
|
11
|
+
#------------------------------------------------------------------------------
|
|
12
|
+
[project]
|
|
13
|
+
name = 'cerebrate-file' # Package name on PyPI
|
|
14
|
+
description = '' # Short description
|
|
15
|
+
readme = 'README.md' # Path to README file
|
|
16
|
+
requires-python = '>=3.10' # Minimum Python version
|
|
17
|
+
keywords = [
|
|
18
|
+
] # Keywords for PyPI search
|
|
19
|
+
dynamic = ["version"] # Fields set dynamically at build time
|
|
20
|
+
|
|
21
|
+
# PyPI classifiers for package categorization
|
|
22
|
+
classifiers = [
|
|
23
|
+
'Development Status :: 4 - Beta', # Package maturity level
|
|
24
|
+
'Programming Language :: Python',
|
|
25
|
+
'Programming Language :: Python :: 3.10',
|
|
26
|
+
'Programming Language :: Python :: 3.11',
|
|
27
|
+
'Programming Language :: Python :: 3.12',
|
|
28
|
+
'Programming Language :: Python :: Implementation :: CPython',
|
|
29
|
+
'Programming Language :: Python :: Implementation :: PyPy',
|
|
30
|
+
'Operating System :: OS Independent',
|
|
31
|
+
'License :: OSI Approved :: MIT License',
|
|
32
|
+
'Intended Audience :: Developers',
|
|
33
|
+
]
|
|
34
|
+
|
|
35
|
+
dependencies = [
|
|
36
|
+
"fire>=0.6.0",
|
|
37
|
+
"loguru>=0.7.0",
|
|
38
|
+
"python-dotenv>=1.0.0",
|
|
39
|
+
"tenacity>=8.2.0",
|
|
40
|
+
"cerebras-cloud-sdk>=1.0.0",
|
|
41
|
+
"semantic-text-splitter>=0.13.0",
|
|
42
|
+
"qwen-tokenizer>=0.0.8",
|
|
43
|
+
"tqdm>=4.66.0",
|
|
44
|
+
"python-frontmatter>=1.1.0",
|
|
45
|
+
]
|
|
46
|
+
|
|
47
|
+
# Author information
|
|
48
|
+
[[project.authors]]
|
|
49
|
+
name = 'Adam Twardoch'
|
|
50
|
+
email = 'adam+github@twardoch.com'
|
|
51
|
+
|
|
52
|
+
# License information
|
|
53
|
+
[project.license]
|
|
54
|
+
text = 'MIT'
|
|
55
|
+
|
|
56
|
+
# Project URLs
|
|
57
|
+
[project.urls]
|
|
58
|
+
Documentation = 'https://github.com/twardoch/cerebrate-file#readme'
|
|
59
|
+
Issues = 'https://github.com/twardoch/cerebrate-file/issues'
|
|
60
|
+
Source = 'https://github.com/twardoch/cerebrate-file'
|
|
61
|
+
|
|
62
|
+
#------------------------------------------------------------------------------
|
|
63
|
+
# OPTIONAL DEPENDENCIES
|
|
64
|
+
# Additional dependencies for optional features, development, and testing.
|
|
65
|
+
#------------------------------------------------------------------------------
|
|
66
|
+
[project.optional-dependencies]
|
|
67
|
+
|
|
68
|
+
# Development tools
|
|
69
|
+
dev = [
|
|
70
|
+
'pre-commit>=4.1.0', # Pre-commit hook manager - Keep pre-commit as is, update if newer pre-commit version is required
|
|
71
|
+
'ruff>=0.9.7', # Linting and formatting - Keep ruff as is, update if newer ruff version is required
|
|
72
|
+
'mypy>=1.15.0', # Type checking - Keep mypy as is, update if newer mypy version is required
|
|
73
|
+
'absolufy-imports>=0.3.1', # Convert relative imports to absolute - Keep absolufy-imports as is, update if newer absolufy-imports version is required
|
|
74
|
+
'pyupgrade>=3.19.1', # Upgrade Python syntax - Keep pyupgrade as is, update if newer pyupgrade version is required
|
|
75
|
+
'isort>=6.0.1', # Sort imports - Keep isort as is, update if newer isort version is required
|
|
76
|
+
]
|
|
77
|
+
|
|
78
|
+
# Testing tools and frameworks
|
|
79
|
+
test = [
|
|
80
|
+
'pytest>=8.3.4', # Testing framework - Keep pytest as is, update if newer pytest version is required
|
|
81
|
+
'pytest-cov>=6.0.0', # Coverage plugin for pytest - Keep pytest-cov as is, update if newer pytest-cov version is required
|
|
82
|
+
'pytest-xdist>=3.6.1', # Parallel test execution - Keep pytest-xdist as is, update if newer pytest-xdist version is required
|
|
83
|
+
'pytest-benchmark[histogram]>=5.1.0', # Benchmarking plugin - Keep pytest-benchmark as is, update if newer pytest-benchmark version is required
|
|
84
|
+
'pytest-asyncio>=0.25.3', # Async test support - Keep pytest-asyncio as is, update if newer pytest-asyncio version is required
|
|
85
|
+
'coverage[toml]>=7.6.12',
|
|
86
|
+
]
|
|
87
|
+
|
|
88
|
+
docs = [
|
|
89
|
+
"sphinx>=7.2.6",
|
|
90
|
+
"sphinx-rtd-theme>=2.0.0",
|
|
91
|
+
"sphinx-autodoc-typehints>=2.0.0",
|
|
92
|
+
"myst-parser>=3.0.0", # Markdown support in Sphinx
|
|
93
|
+
]
|
|
94
|
+
|
|
95
|
+
# All optional dependencies combined
|
|
96
|
+
all = [
|
|
97
|
+
]
|
|
98
|
+
|
|
99
|
+
#------------------------------------------------------------------------------
|
|
100
|
+
# COMMAND-LINE SCRIPTS
|
|
101
|
+
# Entry points for command-line executables installed with the package.
|
|
102
|
+
#------------------------------------------------------------------------------
|
|
103
|
+
[project.scripts]
|
|
104
|
+
cerebrate-file = "cerebrate_file.__main__:main"
|
|
105
|
+
|
|
106
|
+
#------------------------------------------------------------------------------
|
|
107
|
+
# BUILD SYSTEM CONFIGURATION
|
|
108
|
+
# Defines the tools required to build the package and the build backend.
|
|
109
|
+
#------------------------------------------------------------------------------
|
|
110
|
+
[build-system]
|
|
111
|
+
# Hatchling is a modern build backend for Python packaging
|
|
112
|
+
# hatch-vcs integrates with version control systems for versioning
|
|
113
|
+
requires = [
|
|
114
|
+
'hatchling>=1.27.0', # Keep hatchling as is, update if newer hatchling version is required
|
|
115
|
+
'hatch-vcs>=0.4.0', # Keep hatch-vcs as is, update if newer hatch-vcs version is required
|
|
116
|
+
]
|
|
117
|
+
build-backend = 'hatchling.build' # Specifies Hatchling as the build backend
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
#------------------------------------------------------------------------------
|
|
121
|
+
# HATCH BUILD CONFIGURATION
|
|
122
|
+
# Configures the build process, specifying which packages to include and
|
|
123
|
+
# how to handle versioning.
|
|
124
|
+
#------------------------------------------------------------------------------
|
|
125
|
+
[tool.hatch.build]
|
|
126
|
+
# Include package data files
|
|
127
|
+
include = [
|
|
128
|
+
"src/cerebrate_file/py.typed", # For better type checking support
|
|
129
|
+
"src/cerebrate_file/data/**/*", # Include data files if any
|
|
130
|
+
|
|
131
|
+
]
|
|
132
|
+
exclude = ["**/__pycache__", "**/.pytest_cache", "**/.mypy_cache"]
|
|
133
|
+
|
|
134
|
+
[tool.hatch.build.targets.wheel]
|
|
135
|
+
packages = ["src/cerebrate_file"]
|
|
136
|
+
reproducible = true
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
# Version control system hook configuration
|
|
140
|
+
# Automatically updates the version file from git tags
|
|
141
|
+
[tool.hatch.build.hooks.vcs]
|
|
142
|
+
version-file = "src/cerebrate_file/__version__.py"
|
|
143
|
+
|
|
144
|
+
# Version source configuration for git-tag-based semver
|
|
145
|
+
[tool.hatch.version]
|
|
146
|
+
source = 'vcs' # Get version from git tags or other VCS info
|
|
147
|
+
raw-options = { local_scheme = "no-local-version" }
|
|
148
|
+
|
|
149
|
+
# Metadata handling configuration
|
|
150
|
+
[tool.hatch.metadata]
|
|
151
|
+
allow-direct-references = true # Allow direct references in metadata (useful for local dependencies)
|
|
152
|
+
|
|
153
|
+
#------------------------------------------------------------------------------
|
|
154
|
+
# UV PACKAGE MANAGER CONFIGURATION
|
|
155
|
+
# Configuration for uv package manager - provides faster installs and better resolution
|
|
156
|
+
#------------------------------------------------------------------------------
|
|
157
|
+
[tool.uv]
|
|
158
|
+
dev-dependencies = [
|
|
159
|
+
"pre-commit>=4.1.0",
|
|
160
|
+
"ruff>=0.9.7",
|
|
161
|
+
"mypy>=1.15.0",
|
|
162
|
+
"pytest>=8.3.4",
|
|
163
|
+
"pytest-cov>=6.0.0",
|
|
164
|
+
]
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
#------------------------------------------------------------------------------
|
|
168
|
+
# DEVELOPMENT ENVIRONMENTS
|
|
169
|
+
|
|
170
|
+
[tool.hatch.envs.default]
|
|
171
|
+
features = ['dev', 'test', 'all']
|
|
172
|
+
dependencies = [
|
|
173
|
+
]
|
|
174
|
+
|
|
175
|
+
# Commands available in the default environment
|
|
176
|
+
[tool.hatch.envs.default.scripts]
|
|
177
|
+
# Run tests with optional arguments
|
|
178
|
+
test = 'pytest {args:tests}'
|
|
179
|
+
# Run tests with coverage reporting
|
|
180
|
+
test-cov = "pytest --cov-report=term-missing --cov-config=pyproject.toml --cov=src/cerebrate_file --cov=tests {args:tests}"
|
|
181
|
+
# Run type checking
|
|
182
|
+
type-check = "mypy src/cerebrate_file tests"
|
|
183
|
+
# Run linting and formatting
|
|
184
|
+
lint = ["ruff check src/cerebrate_file tests", "ruff format --respect-gitignore src/cerebrate_file tests"]
|
|
185
|
+
# Format and fix style issues
|
|
186
|
+
fmt = ["ruff format --respect-gitignore src/cerebrate_file tests", "ruff check --fix src/cerebrate_file tests"]
|
|
187
|
+
fix = ["ruff check --fix --unsafe-fixes src/cerebrate_file tests", "ruff format --respect-gitignore src/cerebrate_file tests"]
|
|
188
|
+
|
|
189
|
+
# Matrix configuration to test across multiple Python versions
|
|
190
|
+
|
|
191
|
+
[[tool.hatch.envs.all.matrix]]
|
|
192
|
+
python = ["3.10", "3.11", "3.12"]
|
|
193
|
+
|
|
194
|
+
#------------------------------------------------------------------------------
|
|
195
|
+
# SPECIALIZED ENVIRONMENTS
|
|
196
|
+
# Additional environments for specific development tasks.
|
|
197
|
+
#------------------------------------------------------------------------------
|
|
198
|
+
|
|
199
|
+
# Dedicated environment for linting and code quality checks
|
|
200
|
+
[tool.hatch.envs.lint]
|
|
201
|
+
detached = true # Create a separate, isolated environment
|
|
202
|
+
features = ['dev'] # Use dev extras dependencies
|
|
203
|
+
|
|
204
|
+
# Linting environment commands
|
|
205
|
+
[tool.hatch.envs.lint.scripts]
|
|
206
|
+
# Type checking with automatic type installation
|
|
207
|
+
typing = "mypy --install-types --non-interactive {args:src/cerebrate_file tests}"
|
|
208
|
+
# Check style and format code
|
|
209
|
+
style = ["ruff check {args:.}", "ruff format --respect-gitignore {args:.}"]
|
|
210
|
+
# Format and fix style issues
|
|
211
|
+
fmt = ["ruff format --respect-gitignore {args:.}", "ruff check --fix {args:.}"]
|
|
212
|
+
fix = ["ruff check --fix --unsafe-fixes {args:.}", "ruff format --respect-gitignore {args:.}"]
|
|
213
|
+
# Run all ops
|
|
214
|
+
all = ["style", "typing", "fix"]
|
|
215
|
+
|
|
216
|
+
# Dedicated environment for testing
|
|
217
|
+
[tool.hatch.envs.test]
|
|
218
|
+
features = ['test'] # Use test extras as dependencies
|
|
219
|
+
|
|
220
|
+
# Testing environment commands
|
|
221
|
+
[tool.hatch.envs.test.scripts]
|
|
222
|
+
# Run tests in parallel
|
|
223
|
+
test = "python -m pytest -n auto {args:tests}"
|
|
224
|
+
# Run tests with coverage in parallel
|
|
225
|
+
test-cov = "python -m pytest -n auto --cov-report=term-missing --cov-config=pyproject.toml --cov=src/cerebrate_file --cov=tests {args:tests}"
|
|
226
|
+
# Run benchmarks
|
|
227
|
+
bench = "python -m pytest -v -p no:briefcase tests/test_benchmark.py --benchmark-only"
|
|
228
|
+
# Run benchmarks and save results
|
|
229
|
+
bench-save = "python -m pytest -v -p no:briefcase tests/test_benchmark.py --benchmark-only --benchmark-json=benchmark/results.json"
|
|
230
|
+
|
|
231
|
+
# Documentation environment
|
|
232
|
+
[tool.hatch.envs.docs]
|
|
233
|
+
features = ['docs']
|
|
234
|
+
|
|
235
|
+
# Documentation environment commands
|
|
236
|
+
[tool.hatch.envs.docs.scripts]
|
|
237
|
+
build = "sphinx-build -b html docs/source docs/build"
|
|
238
|
+
|
|
239
|
+
# GitHub Actions workflow configuration
|
|
240
|
+
[tool.hatch.envs.ci]
|
|
241
|
+
features = ['test']
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
[tool.hatch.envs.ci.scripts]
|
|
245
|
+
test = "pytest --cov=src/cerebrate_file --cov-report=xml"
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
#------------------------------------------------------------------------------
|
|
249
|
+
# CODE QUALITY TOOLS
|
|
250
|
+
# Configuration for linting, formatting, and code quality enforcement.
|
|
251
|
+
#------------------------------------------------------------------------------
|
|
252
|
+
|
|
253
|
+
#------------------------------------------------------------------------------
|
|
254
|
+
# COVERAGE CONFIGURATION
|
|
255
|
+
# Settings for test coverage measurement and reporting.
|
|
256
|
+
#------------------------------------------------------------------------------
|
|
257
|
+
|
|
258
|
+
# Path mapping for coverage in different environments
|
|
259
|
+
[tool.coverage.paths]
|
|
260
|
+
cerebrate_file = ["src/cerebrate_file", "*/cerebrate-file/src/cerebrate_file"]
|
|
261
|
+
tests = ["tests", "*/cerebrate-file/tests"]
|
|
262
|
+
|
|
263
|
+
# Coverage report configuration
|
|
264
|
+
[tool.coverage.report]
|
|
265
|
+
# Lines to exclude from coverage reporting
|
|
266
|
+
exclude_lines = [
|
|
267
|
+
'no cov', # Custom marker to skip coverage
|
|
268
|
+
'if __name__ == .__main__.:', # Script execution guard
|
|
269
|
+
'if TYPE_CHECKING:', # Type checking imports and code
|
|
270
|
+
'pass', # Empty pass statements
|
|
271
|
+
'raise NotImplementedError', # Unimplemented method placeholders
|
|
272
|
+
'raise ImportError', # Import error handling
|
|
273
|
+
'except ImportError', # Import error handling
|
|
274
|
+
'except KeyError', # Common error handling
|
|
275
|
+
'except AttributeError', # Common error handling
|
|
276
|
+
'except NotImplementedError', # Common error handling
|
|
277
|
+
]
|
|
278
|
+
|
|
279
|
+
[tool.coverage.run]
|
|
280
|
+
source_pkgs = ["cerebrate_file", "tests"]
|
|
281
|
+
branch = true # Measure branch coverage (if/else statements)
|
|
282
|
+
parallel = true # Support parallel test execution
|
|
283
|
+
omit = [
|
|
284
|
+
"src/cerebrate_file/__about__.py",
|
|
285
|
+
]
|
|
286
|
+
|
|
287
|
+
#------------------------------------------------------------------------------
|
|
288
|
+
# MYPY CONFIGURATION
|
|
289
|
+
# Configuration for type checking with mypy.
|
|
290
|
+
#------------------------------------------------------------------------------
|
|
291
|
+
|
|
292
|
+
[tool.mypy]
|
|
293
|
+
python_version = "3.10"
|
|
294
|
+
warn_return_any = true
|
|
295
|
+
warn_unused_configs = true
|
|
296
|
+
disallow_untyped_defs = true
|
|
297
|
+
disallow_incomplete_defs = true
|
|
298
|
+
check_untyped_defs = true
|
|
299
|
+
disallow_untyped_decorators = true
|
|
300
|
+
no_implicit_optional = true
|
|
301
|
+
warn_redundant_casts = true
|
|
302
|
+
warn_unused_ignores = true
|
|
303
|
+
warn_no_return = true
|
|
304
|
+
warn_unreachable = true
|
|
305
|
+
|
|
306
|
+
[[tool.mypy.overrides]]
|
|
307
|
+
module = ["tests.*"]
|
|
308
|
+
disallow_untyped_defs = false
|
|
309
|
+
disallow_incomplete_defs = false
|
|
310
|
+
|
|
311
|
+
#------------------------------------------------------------------------------
|
|
312
|
+
# PYTEST CONFIGURATION
|
|
313
|
+
# Configuration for pytest, including markers, options, and benchmark settings.
|
|
314
|
+
#------------------------------------------------------------------------------
|
|
315
|
+
|
|
316
|
+
[tool.pytest.ini_options]
|
|
317
|
+
addopts = "-v --durations=10 -p no:briefcase"
|
|
318
|
+
asyncio_mode = "auto"
|
|
319
|
+
asyncio_default_fixture_loop_scope = "function"
|
|
320
|
+
console_output_style = "progress"
|
|
321
|
+
filterwarnings = ["ignore::DeprecationWarning", "ignore::UserWarning"]
|
|
322
|
+
log_cli = true
|
|
323
|
+
log_cli_level = "INFO"
|
|
324
|
+
markers = [
|
|
325
|
+
"benchmark: marks tests as benchmarks (select with '-m benchmark')",
|
|
326
|
+
"unit: mark a test as a unit test",
|
|
327
|
+
"integration: mark a test as an integration test",
|
|
328
|
+
"permutation: tests for permutation functionality",
|
|
329
|
+
"parameter: tests for parameter parsing",
|
|
330
|
+
"prompt: tests for prompt parsing",
|
|
331
|
+
]
|
|
332
|
+
norecursedirs = [
|
|
333
|
+
".*",
|
|
334
|
+
"build",
|
|
335
|
+
"dist",
|
|
336
|
+
"venv",
|
|
337
|
+
"__pycache__",
|
|
338
|
+
"*.egg-info",
|
|
339
|
+
"_private",
|
|
340
|
+
]
|
|
341
|
+
python_classes = ["Test*"]
|
|
342
|
+
python_files = ["test_*.py"]
|
|
343
|
+
python_functions = ["test_*"]
|
|
344
|
+
testpaths = ["tests"]
|
|
345
|
+
|
|
346
|
+
[tool.pytest-benchmark]
|
|
347
|
+
min_rounds = 100
|
|
348
|
+
min_time = 0.1
|
|
349
|
+
histogram = true
|
|
350
|
+
storage = "file"
|
|
351
|
+
save-data = true
|
|
352
|
+
compare = [
|
|
353
|
+
"min", # Minimum time
|
|
354
|
+
"max", # Maximum time
|
|
355
|
+
"mean", # Mean time
|
|
356
|
+
"stddev", # Standard deviation
|
|
357
|
+
"median", # Median time
|
|
358
|
+
"iqr", # Inter-quartile range
|
|
359
|
+
"ops", # Operations per second
|
|
360
|
+
"rounds", # Number of rounds
|
|
361
|
+
]
|
|
362
|
+
|
|
363
|
+
#------------------------------------------------------------------------------
|
|
364
|
+
# RUFF CONFIGURATION
|
|
365
|
+
# Configuration for Ruff, including linter and formatter settings.
|
|
366
|
+
#------------------------------------------------------------------------------
|
|
367
|
+
|
|
368
|
+
# Ruff linter and formatter configuration
|
|
369
|
+
[tool.ruff]
|
|
370
|
+
target-version = "py310"
|
|
371
|
+
line-length = 100
|
|
372
|
+
|
|
373
|
+
# Linting rules configuration
|
|
374
|
+
[tool.ruff.lint]
|
|
375
|
+
# Rule sets to enable - sensible defaults for most projects
|
|
376
|
+
select = [
|
|
377
|
+
'E', # pycodestyle errors
|
|
378
|
+
'W', # pycodestyle warnings
|
|
379
|
+
'F', # pyflakes
|
|
380
|
+
'I', # isort
|
|
381
|
+
'N', # pep8-naming
|
|
382
|
+
'UP', # pyupgrade
|
|
383
|
+
'B', # flake8-bugbear
|
|
384
|
+
'C4', # flake8-comprehensions
|
|
385
|
+
'SIM', # flake8-simplify
|
|
386
|
+
'RUF', # Ruff-specific rules
|
|
387
|
+
'PT', # flake8-pytest-style
|
|
388
|
+
'PTH', # flake8-use-pathlib
|
|
389
|
+
'ARG', # flake8-unused-arguments
|
|
390
|
+
'PLE', # pylint errors
|
|
391
|
+
'PLW', # pylint warnings
|
|
392
|
+
]
|
|
393
|
+
# Rules to ignore (with reasons)
|
|
394
|
+
ignore = [
|
|
395
|
+
'E501', # Line too long - handled by formatter
|
|
396
|
+
'W503', # Line break before binary operator - conflicts with formatter
|
|
397
|
+
'PLW0603', # Using the global statement - sometimes necessary
|
|
398
|
+
'SIM102', # Nested if statements - sometimes more readable
|
|
399
|
+
]
|
|
400
|
+
# Rules that should not be automatically fixed
|
|
401
|
+
unfixable = [
|
|
402
|
+
'F401', # Don't automatically remove unused imports - may be needed later
|
|
403
|
+
]
|
|
404
|
+
# Configure extend-exclude to ignore specific directories
|
|
405
|
+
extend-exclude = [".git", ".venv", "venv", "dist", "build"]
|
|
406
|
+
|
|
407
|
+
# isort configuration within Ruff
|
|
408
|
+
[tool.ruff.lint.isort]
|
|
409
|
+
known-first-party = ['cerebrate_file'] # Treat as first-party imports for sorting
|
|
410
|
+
|
|
411
|
+
# flake8-tidy-imports configuration within Ruff
|
|
412
|
+
[tool.ruff.lint.flake8-tidy-imports]
|
|
413
|
+
ban-relative-imports = 'all' # Ban all relative imports for consistency
|
|
414
|
+
|
|
415
|
+
# Per-file rule exceptions
|
|
416
|
+
[tool.ruff.lint.per-file-ignores]
|
|
417
|
+
# Tests can use magic values, assertions, and relative imports
|
|
418
|
+
'tests/**/*' = [
|
|
419
|
+
'PLR2004', # Allow magic values in tests for readability
|
|
420
|
+
'S101', # Allow assertions in tests
|
|
421
|
+
'TID252'
|
|
422
|
+
# Allow relative imports in tests for convenience
|
|
423
|
+
]
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
# file generated by setuptools-scm
|
|
2
|
+
# don't change, don't track in version control
|
|
3
|
+
|
|
4
|
+
__all__ = [
|
|
5
|
+
"__version__",
|
|
6
|
+
"__version_tuple__",
|
|
7
|
+
"version",
|
|
8
|
+
"version_tuple",
|
|
9
|
+
"__commit_id__",
|
|
10
|
+
"commit_id",
|
|
11
|
+
]
|
|
12
|
+
|
|
13
|
+
TYPE_CHECKING = False
|
|
14
|
+
if TYPE_CHECKING:
|
|
15
|
+
from typing import Tuple
|
|
16
|
+
from typing import Union
|
|
17
|
+
|
|
18
|
+
VERSION_TUPLE = Tuple[Union[int, str], ...]
|
|
19
|
+
COMMIT_ID = Union[str, None]
|
|
20
|
+
else:
|
|
21
|
+
VERSION_TUPLE = object
|
|
22
|
+
COMMIT_ID = object
|
|
23
|
+
|
|
24
|
+
version: str
|
|
25
|
+
__version__: str
|
|
26
|
+
__version_tuple__: VERSION_TUPLE
|
|
27
|
+
version_tuple: VERSION_TUPLE
|
|
28
|
+
commit_id: COMMIT_ID
|
|
29
|
+
__commit_id__: COMMIT_ID
|
|
30
|
+
|
|
31
|
+
__version__ = version = '1.0.4'
|
|
32
|
+
__version_tuple__ = version_tuple = (1, 0, 4)
|
|
33
|
+
|
|
34
|
+
__commit_id__ = commit_id = None
|