pdf-to-markdown-cli 0.2.1__tar.gz → 0.5.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. {pdf_to_markdown_cli-0.2.1 → pdf_to_markdown_cli-0.5.0}/MANIFEST.in +2 -2
  2. {pdf_to_markdown_cli-0.2.1 → pdf_to_markdown_cli-0.5.0}/PKG-INFO +66 -17
  3. pdf_to_markdown_cli-0.2.1/pdf_to_markdown_cli.egg-info/PKG-INFO → pdf_to_markdown_cli-0.5.0/README.md +41 -47
  4. pdf_to_markdown_cli-0.5.0/pyproject.toml +54 -0
  5. pdf_to_markdown_cli-0.5.0/src/docs_to_md/api/client.py +183 -0
  6. {pdf_to_markdown_cli-0.2.1 → pdf_to_markdown_cli-0.5.0/src}/docs_to_md/api/models.py +29 -1
  7. {pdf_to_markdown_cli-0.2.1 → pdf_to_markdown_cli-0.5.0/src}/docs_to_md/config/cli.py +32 -31
  8. pdf_to_markdown_cli-0.5.0/src/docs_to_md/config/settings.py +54 -0
  9. pdf_to_markdown_cli-0.5.0/src/docs_to_md/core/paths.py +80 -0
  10. pdf_to_markdown_cli-0.5.0/src/docs_to_md/core/processor.py +347 -0
  11. pdf_to_markdown_cli-0.5.0/src/docs_to_md/core/result_handler.py +601 -0
  12. {pdf_to_markdown_cli-0.2.1 → pdf_to_markdown_cli-0.5.0/src}/docs_to_md/main.py +20 -11
  13. {pdf_to_markdown_cli-0.2.1 → pdf_to_markdown_cli-0.5.0/src}/docs_to_md/storage/cache.py +2 -2
  14. {pdf_to_markdown_cli-0.2.1 → pdf_to_markdown_cli-0.5.0/src}/docs_to_md/storage/models.py +7 -3
  15. pdf_to_markdown_cli-0.5.0/src/docs_to_md/utils/file_utils.py +219 -0
  16. {pdf_to_markdown_cli-0.2.1 → pdf_to_markdown_cli-0.5.0/src}/docs_to_md/utils/logging.py +51 -39
  17. pdf_to_markdown_cli-0.2.1/docs_to_md/pdf/splitter.py → pdf_to_markdown_cli-0.5.0/src/docs_to_md/utils/pdf_splitter.py +3 -35
  18. pdf_to_markdown_cli-0.5.0/src/pdf_to_markdown_cli.egg-info/PKG-INFO +227 -0
  19. pdf_to_markdown_cli-0.5.0/src/pdf_to_markdown_cli.egg-info/SOURCES.txt +35 -0
  20. {pdf_to_markdown_cli-0.2.1 → pdf_to_markdown_cli-0.5.0/src}/pdf_to_markdown_cli.egg-info/requires.txt +2 -0
  21. pdf_to_markdown_cli-0.5.0/tests/test_cli.py +37 -0
  22. pdf_to_markdown_cli-0.5.0/tests/test_paths.py +25 -0
  23. pdf_to_markdown_cli-0.5.0/tests/test_settings.py +65 -0
  24. pdf_to_markdown_cli-0.5.0/tests/test_utils.py +22 -0
  25. pdf_to_markdown_cli-0.2.1/README.md +0 -136
  26. pdf_to_markdown_cli-0.2.1/docs_to_md/api/client.py +0 -218
  27. pdf_to_markdown_cli-0.2.1/docs_to_md/config/settings.py +0 -108
  28. pdf_to_markdown_cli-0.2.1/docs_to_md/core/processor.py +0 -229
  29. pdf_to_markdown_cli-0.2.1/docs_to_md/core/result_handler.py +0 -496
  30. pdf_to_markdown_cli-0.2.1/docs_to_md/utils/__init__.py +0 -0
  31. pdf_to_markdown_cli-0.2.1/docs_to_md/utils/file_utils.py +0 -189
  32. pdf_to_markdown_cli-0.2.1/pdf_to_markdown_cli.egg-info/SOURCES.txt +0 -31
  33. pdf_to_markdown_cli-0.2.1/setup.py +0 -67
  34. {pdf_to_markdown_cli-0.2.1 → pdf_to_markdown_cli-0.5.0}/LICENSE +0 -0
  35. {pdf_to_markdown_cli-0.2.1 → pdf_to_markdown_cli-0.5.0}/setup.cfg +0 -0
  36. {pdf_to_markdown_cli-0.2.1 → pdf_to_markdown_cli-0.5.0/src}/docs_to_md/__init__.py +0 -0
  37. {pdf_to_markdown_cli-0.2.1 → pdf_to_markdown_cli-0.5.0/src}/docs_to_md/__main__.py +0 -0
  38. {pdf_to_markdown_cli-0.2.1 → pdf_to_markdown_cli-0.5.0/src}/docs_to_md/api/__init__.py +0 -0
  39. {pdf_to_markdown_cli-0.2.1 → pdf_to_markdown_cli-0.5.0/src}/docs_to_md/config/__init__.py +0 -0
  40. {pdf_to_markdown_cli-0.2.1 → pdf_to_markdown_cli-0.5.0/src}/docs_to_md/core/__init__.py +0 -0
  41. {pdf_to_markdown_cli-0.2.1/docs_to_md/pdf → pdf_to_markdown_cli-0.5.0/src/docs_to_md/storage}/__init__.py +0 -0
  42. {pdf_to_markdown_cli-0.2.1/docs_to_md/storage → pdf_to_markdown_cli-0.5.0/src/docs_to_md/utils}/__init__.py +0 -0
  43. {pdf_to_markdown_cli-0.2.1 → pdf_to_markdown_cli-0.5.0/src}/docs_to_md/utils/exceptions.py +0 -0
  44. {pdf_to_markdown_cli-0.2.1 → pdf_to_markdown_cli-0.5.0/src}/pdf_to_markdown_cli.egg-info/dependency_links.txt +0 -0
  45. {pdf_to_markdown_cli-0.2.1 → pdf_to_markdown_cli-0.5.0/src}/pdf_to_markdown_cli.egg-info/entry_points.txt +0 -0
  46. {pdf_to_markdown_cli-0.2.1 → pdf_to_markdown_cli-0.5.0/src}/pdf_to_markdown_cli.egg-info/top_level.txt +0 -0
@@ -2,8 +2,8 @@
2
2
  include README.md
3
3
  include LICENSE
4
4
 
5
- # Include the package source directory
6
- recursive-include docs_to_md *
5
+ # Include the package source directory (now within src/)
6
+ recursive-include src/docs_to_md *
7
7
 
8
8
  # Exclude development/cache/temporary files and directories
9
9
  global-exclude *.py[cod]
@@ -1,9 +1,31 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: pdf-to-markdown-cli
3
- Version: 0.2.1
3
+ Version: 0.5.0
4
4
  Summary: CLI tool to convert PDF files (and other documents) to markdown using the Marker API.
5
- Home-page: https://github.com/SokolskyNikita/pdf-to-markdown-cli
6
5
  Author: Nikita Sokolsky
6
+ License: MIT License
7
+
8
+ Copyright (c) 2025 SokolskyNikita
9
+
10
+ Permission is hereby granted, free of charge, to any person obtaining a copy
11
+ of this software and associated documentation files (the "Software"), to deal
12
+ in the Software without restriction, including without limitation the rights
13
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
14
+ copies of the Software, and to permit persons to whom the Software is
15
+ furnished to do so, subject to the following conditions:
16
+
17
+ The above copyright notice and this permission notice shall be included in all
18
+ copies or substantial portions of the Software.
19
+
20
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
21
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
23
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
24
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
25
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
26
+ SOFTWARE.
27
+
28
+ Project-URL: Homepage, https://github.com/SokolskyNikita/pdf-to-markdown-cli
7
29
  Project-URL: Bug Tracker, https://github.com/SokolskyNikita/pdf-to-markdown-cli/issues
8
30
  Keywords: pdf,markdown,converter,cli,document,marker,md
9
31
  Classifier: Development Status :: 4 - Beta
@@ -28,17 +50,8 @@ Requires-Dist: pydantic>=2.0
28
50
  Requires-Dist: ratelimit>=2.0
29
51
  Requires-Dist: requests>=2.0
30
52
  Requires-Dist: tqdm>=4.0
31
- Dynamic: author
32
- Dynamic: classifier
33
- Dynamic: description
34
- Dynamic: description-content-type
35
- Dynamic: home-page
36
- Dynamic: keywords
53
+ Provides-Extra: test
37
54
  Dynamic: license-file
38
- Dynamic: project-url
39
- Dynamic: requires-dist
40
- Dynamic: requires-python
41
- Dynamic: summary
42
55
 
43
56
  # PDF to Markdown CLI (via the Datalab Marker API)
44
57
 
@@ -95,6 +108,9 @@ pdf-to-md /path/to/file.pdf --langs "English,French,German"
95
108
 
96
109
  # Use all Marker OCR enhancements
97
110
  pdf-to-md /path/to/file.pdf --max
111
+
112
+ # Display all available options
113
+ pdf-to-md --help
98
114
  ```
99
115
 
100
116
  ### Full list of CLI options
@@ -108,11 +124,31 @@ pdf-to-md /path/to/file.pdf --max
108
124
  - `--force`: Force OCR on all pages
109
125
  - `--pages`: Add page delimiters
110
126
  - `--max`: Enable all OCR enhancements (equivalent to --llm --strip --force)
111
- - `--max-pages`: Maximum number of pages to process from the start of the file
127
+ - `-mp`, `--max-pages`: Maximum number of pages to process from the start of the file
112
128
  - `--no-chunk`: Disable PDF chunking
113
- - `--chunk-size`: Set PDF chunk size in pages (default: 25)
114
- - `--output-dir`: Output directory (default: "converted")
115
- - `--cache-dir`: Cache directory (default: ".marker_cache")
129
+ - `-cs`, `--chunk-size`: Set PDF chunk size in pages (default: 25)
130
+ - `-o`, `--output-dir`: Absolute path to the output directory. If not provided, output files will be saved in the same directory as their corresponding input files.
131
+ - `-v`, `--verbose`: Enable verbose (DEBUG level) logging
132
+ - `--version`: Show the installed `pdf-to-markdown-cli` version and exit
133
+
134
+ ### Output Structure
135
+
136
+ By default, output files are saved in the same directory as the input file with the format `[input-filename].[format]`. For example, converting `/data/report.pdf` to markdown will result in `/data/report.md`.
137
+
138
+ If an output file with the same name already exists, the new file will be automatically renamed using a numeric suffix (e.g., `[input-filename]_1.[format]`, `[input-filename]_2.[format]`, etc.) to avoid overwriting.
139
+
140
+ If you specify `--output-dir /path/to/output`, the output file will be placed in that directory (e.g., `/path/to/output/report.md`). The same automatic renaming logic applies if a file with the same name exists in the target directory.
141
+
142
+ ### Core Workflow
143
+
144
+ `MarkerProcessor` drives the conversion process:
145
+
146
+ 1. Discover files with `FileDiscovery`.
147
+ 2. Determine unique output paths for each file and chunk.
148
+ 3. Submit jobs to the Marker API via `BatchProcessor`.
149
+ 4. Poll for results and combine chunk outputs while moving extracted images.
150
+
151
+ Settings are validated in `Config.validate()` and request data is cached with `CacheManager` so interrupted runs can resume.
116
152
 
117
153
  ### API
118
154
 
@@ -152,6 +188,8 @@ pdf-to-markdown-cli/ (Project Root)
152
188
  └── ... (other config files)
153
189
  ```
154
190
 
191
+ `pyproject.toml` defines `pdf-to-md` as a console script pointing to `docs_to_md.main:main`, so installing the project exposes the `pdf-to-md` command.
192
+
155
193
  ## Requirements
156
194
 
157
195
  - Python 3.10 or higher
@@ -168,11 +206,22 @@ To run the code directly from the source tree without installation:
168
206
  # pdf-to-md /path/to/file.pdf
169
207
 
170
208
  # Or running the module directly:
171
- python -m docs_to_md /path/to/file.pdf
209
+ python -m docs_to_md /path/to/file.pdf
172
210
  ```
173
211
 
174
212
  For regular use after installation, use the `pdf-to-md` command.
175
213
 
214
+ ## Getting Started for Contributors
215
+
216
+ - Run `pdf-to-md --help` to explore all CLI flags. The `examples/` directory contains sample documents for testing.
217
+ - Follow `MarkerProcessor.process()` in `docs_to_md/core/processor.py` to see how jobs are prepared and results combined.
218
+ - Consult `datalab_marker_api_docs.md` for full details on authentication and Marker API parameters.
219
+ - Utilities in `docs_to_md/utils` and the caching layer in `docs_to_md/storage` are useful starting points for deeper customization.
220
+
176
221
  ## License
177
222
 
178
223
  This project is licensed under the MIT License - see the LICENSE file for details.
224
+
225
+ ## Future Considerations
226
+
227
+ While this project currently uses a modern structure (`pyproject.toml`, `src` layout), future development might involve migrating to a standardized project template, such as [simonw/python-lib](https://github.com/simonw/python-lib), to further align with community best practices and potentially simplify workflows like automated PyPI publishing via Trusted Publishers.
@@ -1,45 +1,3 @@
1
- Metadata-Version: 2.4
2
- Name: pdf-to-markdown-cli
3
- Version: 0.2.1
4
- Summary: CLI tool to convert PDF files (and other documents) to markdown using the Marker API.
5
- Home-page: https://github.com/SokolskyNikita/pdf-to-markdown-cli
6
- Author: Nikita Sokolsky
7
- Project-URL: Bug Tracker, https://github.com/SokolskyNikita/pdf-to-markdown-cli/issues
8
- Keywords: pdf,markdown,converter,cli,document,marker,md
9
- Classifier: Development Status :: 4 - Beta
10
- Classifier: Intended Audience :: Developers
11
- Classifier: Intended Audience :: End Users/Desktop
12
- Classifier: License :: OSI Approved :: MIT License
13
- Classifier: Programming Language :: Python :: 3
14
- Classifier: Programming Language :: Python :: 3.10
15
- Classifier: Programming Language :: Python :: 3.11
16
- Classifier: Programming Language :: Python :: 3.12
17
- Classifier: Operating System :: OS Independent
18
- Classifier: Topic :: Text Processing
19
- Classifier: Topic :: Utilities
20
- Requires-Python: >=3.10
21
- Description-Content-Type: text/markdown
22
- License-File: LICENSE
23
- Requires-Dist: backoff>=2.0
24
- Requires-Dist: diskcache>=5.0
25
- Requires-Dist: filetype>=1.0
26
- Requires-Dist: pikepdf>=8.0
27
- Requires-Dist: pydantic>=2.0
28
- Requires-Dist: ratelimit>=2.0
29
- Requires-Dist: requests>=2.0
30
- Requires-Dist: tqdm>=4.0
31
- Dynamic: author
32
- Dynamic: classifier
33
- Dynamic: description
34
- Dynamic: description-content-type
35
- Dynamic: home-page
36
- Dynamic: keywords
37
- Dynamic: license-file
38
- Dynamic: project-url
39
- Dynamic: requires-dist
40
- Dynamic: requires-python
41
- Dynamic: summary
42
-
43
1
  # PDF to Markdown CLI (via the Datalab Marker API)
44
2
 
45
3
  Convert PDF files (and other documents) to Markdown using the [Marker API](https://www.datalab.to/marker) via a convenient CLI tool.
@@ -95,6 +53,9 @@ pdf-to-md /path/to/file.pdf --langs "English,French,German"
95
53
 
96
54
  # Use all Marker OCR enhancements
97
55
  pdf-to-md /path/to/file.pdf --max
56
+
57
+ # Display all available options
58
+ pdf-to-md --help
98
59
  ```
99
60
 
100
61
  ### Full list of CLI options
@@ -108,11 +69,31 @@ pdf-to-md /path/to/file.pdf --max
108
69
  - `--force`: Force OCR on all pages
109
70
  - `--pages`: Add page delimiters
110
71
  - `--max`: Enable all OCR enhancements (equivalent to --llm --strip --force)
111
- - `--max-pages`: Maximum number of pages to process from the start of the file
72
+ - `-mp`, `--max-pages`: Maximum number of pages to process from the start of the file
112
73
  - `--no-chunk`: Disable PDF chunking
113
- - `--chunk-size`: Set PDF chunk size in pages (default: 25)
114
- - `--output-dir`: Output directory (default: "converted")
115
- - `--cache-dir`: Cache directory (default: ".marker_cache")
74
+ - `-cs`, `--chunk-size`: Set PDF chunk size in pages (default: 25)
75
+ - `-o`, `--output-dir`: Absolute path to the output directory. If not provided, output files will be saved in the same directory as their corresponding input files.
76
+ - `-v`, `--verbose`: Enable verbose (DEBUG level) logging
77
+ - `--version`: Show the installed `pdf-to-markdown-cli` version and exit
78
+
79
+ ### Output Structure
80
+
81
+ By default, output files are saved in the same directory as the input file with the format `[input-filename].[format]`. For example, converting `/data/report.pdf` to markdown will result in `/data/report.md`.
82
+
83
+ If an output file with the same name already exists, the new file will be automatically renamed using a numeric suffix (e.g., `[input-filename]_1.[format]`, `[input-filename]_2.[format]`, etc.) to avoid overwriting.
84
+
85
+ If you specify `--output-dir /path/to/output`, the output file will be placed in that directory (e.g., `/path/to/output/report.md`). The same automatic renaming logic applies if a file with the same name exists in the target directory.
86
+
87
+ ### Core Workflow
88
+
89
+ `MarkerProcessor` drives the conversion process:
90
+
91
+ 1. Discover files with `FileDiscovery`.
92
+ 2. Determine unique output paths for each file and chunk.
93
+ 3. Submit jobs to the Marker API via `BatchProcessor`.
94
+ 4. Poll for results and combine chunk outputs while moving extracted images.
95
+
96
+ Settings are validated in `Config.validate()` and request data is cached with `CacheManager` so interrupted runs can resume.
116
97
 
117
98
  ### API
118
99
 
@@ -152,6 +133,8 @@ pdf-to-markdown-cli/ (Project Root)
152
133
  └── ... (other config files)
153
134
  ```
154
135
 
136
+ `pyproject.toml` defines `pdf-to-md` as a console script pointing to `docs_to_md.main:main`, so installing the project exposes the `pdf-to-md` command.
137
+
155
138
  ## Requirements
156
139
 
157
140
  - Python 3.10 or higher
@@ -168,11 +151,22 @@ To run the code directly from the source tree without installation:
168
151
  # pdf-to-md /path/to/file.pdf
169
152
 
170
153
  # Or running the module directly:
171
- python -m docs_to_md /path/to/file.pdf
154
+ python -m docs_to_md /path/to/file.pdf
172
155
  ```
173
156
 
174
157
  For regular use after installation, use the `pdf-to-md` command.
175
158
 
159
+ ## Getting Started for Contributors
160
+
161
+ - Run `pdf-to-md --help` to explore all CLI flags. The `examples/` directory contains sample documents for testing.
162
+ - Follow `MarkerProcessor.process()` in `docs_to_md/core/processor.py` to see how jobs are prepared and results combined.
163
+ - Consult `datalab_marker_api_docs.md` for full details on authentication and Marker API parameters.
164
+ - Utilities in `docs_to_md/utils` and the caching layer in `docs_to_md/storage` are useful starting points for deeper customization.
165
+
176
166
  ## License
177
167
 
178
168
  This project is licensed under the MIT License - see the LICENSE file for details.
169
+
170
+ ## Future Considerations
171
+
172
+ While this project currently uses a modern structure (`pyproject.toml`, `src` layout), future development might involve migrating to a standardized project template, such as [simonw/python-lib](https://github.com/simonw/python-lib), to further align with community best practices and potentially simplify workflows like automated PyPI publishing via Trusted Publishers.
@@ -0,0 +1,54 @@
1
+ [build-system]
2
+ requires = ["setuptools>=61.0", "wheel"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "pdf-to-markdown-cli"
7
+ version = "0.5.0"
8
+ description = "CLI tool to convert PDF files (and other documents) to markdown using the Marker API."
9
+ readme = "README.md"
10
+ authors = [
11
+ { name = "Nikita Sokolsky" },
12
+ ]
13
+ license = { file = "LICENSE" }
14
+ requires-python = ">=3.10"
15
+ classifiers = [
16
+ "Development Status :: 4 - Beta",
17
+ "Intended Audience :: Developers",
18
+ "Intended Audience :: End Users/Desktop",
19
+ "License :: OSI Approved :: MIT License",
20
+ "Programming Language :: Python :: 3",
21
+ "Programming Language :: Python :: 3.10",
22
+ "Programming Language :: Python :: 3.11",
23
+ "Programming Language :: Python :: 3.12",
24
+ "Operating System :: OS Independent",
25
+ "Topic :: Text Processing",
26
+ "Topic :: Utilities",
27
+ ]
28
+ keywords = ["pdf", "markdown", "converter", "cli", "document", "marker", "md"]
29
+ dependencies = [
30
+ "backoff>=2.0",
31
+ "diskcache>=5.0",
32
+ "filetype>=1.0",
33
+ "pikepdf>=8.0",
34
+ "pydantic>=2.0",
35
+ "ratelimit>=2.0",
36
+ "requests>=2.0",
37
+ "tqdm>=4.0",
38
+ ]
39
+
40
+ # Define project URLs
41
+ [project.urls]
42
+ "Homepage" = "https://github.com/SokolskyNikita/pdf-to-markdown-cli"
43
+ "Bug Tracker" = "https://github.com/SokolskyNikita/pdf-to-markdown-cli/issues"
44
+
45
+ # Define console script entry point
46
+ [project.scripts]
47
+ pdf-to-md = "docs_to_md.main:main"
48
+
49
+ # Just in case: Configure setuptools to find packages in src/
50
+ [tool.setuptools.packages.find]
51
+ where = ["src"]
52
+
53
+ [project.optional-dependencies]
54
+ test = []
@@ -0,0 +1,183 @@
1
+ import json
2
+ import logging
3
+ from pathlib import Path
4
+ from typing import Optional
5
+
6
+ import backoff
7
+ import filetype
8
+ import requests
9
+ from ratelimit import limits, sleep_and_retry
10
+
11
+ from docs_to_md.api.models import (
12
+ MarkerStatus,
13
+ StatusEnum,
14
+ SubmitResponse,
15
+ SUPPORTED_MIME_TYPES,
16
+ )
17
+ from docs_to_md.utils.exceptions import APIError
18
+ from docs_to_md.utils.file_utils import FileIO
19
+
20
+ # Client-side constants
21
+ MAX_REQUESTS_PER_MINUTE = 150
22
+ REQUEST_TIMEOUT_SECONDS = 30
23
+ MAX_RETRIES = 3
24
+
25
+ logger = logging.getLogger(__name__)
26
+
27
+
28
+ class MarkerClient:
29
+ BASE_MARKER_API_ENDPOINT = "https://www.datalab.to/api/v1/marker"
30
+
31
+ # See datalab_marker_api_docs.md#authentication for API key details
32
+ def __init__(self, api_key: str):
33
+ if not api_key or not api_key.strip():
34
+ raise APIError("API key is required")
35
+
36
+ self.headers = {"X-Api-Key": api_key.strip()}
37
+
38
+ @sleep_and_retry
39
+ @limits(calls=MAX_REQUESTS_PER_MINUTE, period=60)
40
+ @backoff.on_exception(
41
+ backoff.expo,
42
+ (requests.exceptions.RequestException, json.JSONDecodeError),
43
+ max_tries=MAX_RETRIES,
44
+ )
45
+ def submit_file(
46
+ self,
47
+ file_path: Path,
48
+ output_format: str = "markdown",
49
+ langs: str = "English",
50
+ use_llm: bool = False,
51
+ strip_existing_ocr: bool = False,
52
+ disable_image_extraction: bool = False,
53
+ force_ocr: bool = False,
54
+ paginate: bool = False,
55
+ max_pages: Optional[int] = None,
56
+ ) -> Optional[str]:
57
+ """Submit a file for conversion via the Marker API.
58
+ See datalab_marker_api_docs.md#marker for API parameter details.
59
+ """
60
+ try:
61
+ if not file_path.exists():
62
+ raise APIError(f"File not found: {file_path}")
63
+
64
+ file_data = FileIO.read_file(file_path)
65
+ kind = filetype.guess(file_data)
66
+
67
+ # Supported types listed in datalab_marker_api_docs.md#supported-file-types
68
+ if not kind or kind.mime not in SUPPORTED_MIME_TYPES:
69
+ raise APIError(
70
+ f"Unsupported file type: {kind.mime if kind else 'unknown'}"
71
+ )
72
+
73
+ form_data = {
74
+ "file": (file_path.name, file_data, kind.mime),
75
+ "langs": (None, langs),
76
+ "force_ocr": (None, force_ocr),
77
+ "paginate": (None, paginate),
78
+ "strip_existing_ocr": (None, strip_existing_ocr),
79
+ "disable_image_extraction": (None, disable_image_extraction),
80
+ "use_llm": (None, use_llm),
81
+ "output_format": (None, output_format),
82
+ "max_pages": (None, max_pages),
83
+ }
84
+
85
+ response = requests.post(
86
+ self.BASE_MARKER_API_ENDPOINT,
87
+ files=form_data,
88
+ headers=self.headers,
89
+ timeout=REQUEST_TIMEOUT_SECONDS,
90
+ )
91
+ response.raise_for_status() # Default handling for HTTP errors,
92
+ submit_response = SubmitResponse.model_validate(response.json())
93
+
94
+ if not submit_response.success:
95
+ logger.error(
96
+ f"API request failed: {submit_response.error or 'Unknown error'}"
97
+ )
98
+ return None
99
+
100
+ logger.info(
101
+ f"Successfully submitted file {file_path.name}. Request ID: {submit_response.request_id}"
102
+ )
103
+ return submit_response.request_id
104
+
105
+ except Exception as e:
106
+ logger.error(f"Error submitting file {file_path}: {e}")
107
+ return None
108
+
109
+ def _handle_status_error(
110
+ self, status_code: int, request_id: str
111
+ ) -> Optional[MarkerStatus]:
112
+ """Handle non-200 status codes from the check_status endpoint."""
113
+ logger.error(f"API returned status code {status_code} for request {request_id}")
114
+ # Specific handling for non-fatal polling errors
115
+ if status_code == 404:
116
+ # Treat not found as still processing, might appear later
117
+ return MarkerStatus(status=StatusEnum.PROCESSING, error="Request not found")
118
+ elif status_code == 401:
119
+ return MarkerStatus(status=StatusEnum.FAILED, error="Authentication failed")
120
+ elif status_code == 429:
121
+ # Treat rate limit as still processing, should retry later
122
+ return MarkerStatus(
123
+ status=StatusEnum.PROCESSING, error="Rate limit exceeded"
124
+ )
125
+ # For other non-200 errors, return None to indicate failure to get status
126
+ return None
127
+
128
+ @sleep_and_retry
129
+ @limits(calls=MAX_REQUESTS_PER_MINUTE, period=60)
130
+ @backoff.on_exception(
131
+ backoff.expo,
132
+ (requests.exceptions.RequestException, json.JSONDecodeError),
133
+ max_tries=MAX_RETRIES,
134
+ )
135
+ def check_status(self, request_id: str) -> Optional[MarkerStatus]:
136
+ """
137
+ Check the status of a conversion request.
138
+ See datalab_marker_api_docs.md#marker for polling details.
139
+
140
+ Returns:
141
+ MarkerStatus object with current status, or None if the check fails.
142
+ """
143
+ if not request_id:
144
+ logger.error("Empty Marker request ID provided, skipping status check")
145
+ return None
146
+
147
+ try:
148
+ response = requests.get(
149
+ f"{self.BASE_MARKER_API_ENDPOINT}/{request_id}",
150
+ headers=self.headers,
151
+ timeout=REQUEST_TIMEOUT_SECONDS,
152
+ )
153
+
154
+ if response.status_code != 200:
155
+ return self._handle_status_error(response.status_code, request_id)
156
+
157
+ data = response.json()
158
+ # Handle potential empty response from API
159
+ if not data:
160
+ logger.error(f"Empty response for request {request_id}")
161
+ return None
162
+
163
+ status = MarkerStatus.model_validate(data)
164
+ return status
165
+
166
+ except json.JSONDecodeError as e:
167
+ logger.error(f"Invalid JSON response for request {request_id}: {e}")
168
+ return None
169
+ except (
170
+ requests.exceptions.RequestException
171
+ ) as e: # Consolidated network/request errors
172
+ logger.error(f"Request error checking status for {request_id}: {e}")
173
+ return None
174
+ except Exception as e: # Catch-all for validation or other unexpected errors
175
+ logger.error(f"Unexpected error checking status for {request_id}: {e}")
176
+ return None
177
+
178
+ def __enter__(self):
179
+ return self
180
+
181
+ def __exit__(self, exc_type, exc_val, exc_tb):
182
+ # No specific cleanup needed for this client
183
+ pass
@@ -44,9 +44,37 @@ class ApiParams:
44
44
  force_ocr: bool = False
45
45
  paginate: bool = False
46
46
  max_pages: Optional[int] = None
47
+
48
+ # Map of supported output formats to their extensions
49
+ SUPPORTED_FORMAT_EXTENSIONS = {
50
+ "markdown": ".md",
51
+ "json": ".json",
52
+ "html": ".html",
53
+ "txt": ".txt"
54
+ }
47
55
 
56
+ SUPPORTED_IMAGE_EXTENSIONS: Set[str] = {
57
+ "jpg",
58
+ "jpeg",
59
+ "png",
60
+ "gif",
61
+ "tiff"
62
+ }
48
63
 
49
- # Supported mime types according to API docs
64
+ SUPPORTED_INPUT_EXTENSIONS: Set[str] = {
65
+ "pdf",
66
+ "docx",
67
+ "doc",
68
+ "pptx",
69
+ "ppt",
70
+ "jpg",
71
+ "jpeg",
72
+ "png",
73
+ "gif",
74
+ "tiff"
75
+ }
76
+
77
+ # Supported mime types according to datalab_marker_api_docs.md#supported-file-types
50
78
  SUPPORTED_MIME_TYPES: Set[str] = {
51
79
  # PDF
52
80
  'application/pdf',
@@ -1,61 +1,57 @@
1
1
  import argparse
2
- from pathlib import Path
2
+ from pathlib import Path
3
+ import os
4
+ from typing import Optional
5
+ import importlib.metadata
3
6
 
4
7
  from docs_to_md.config.settings import Config
5
- from docs_to_md.utils.exceptions import ConfigurationError
6
- from docs_to_md.utils.file_utils import get_env_var
8
+ from docs_to_md.utils.exceptions import ConfigurationError, FileError
7
9
 
8
10
 
9
11
  def parse_args() -> argparse.Namespace:
10
- """
11
- Parse command line arguments.
12
-
13
- Returns:
14
- Parsed arguments namespace
15
- """
12
+ # Get package version dynamically
13
+ try:
14
+ __version__ = importlib.metadata.version('pdf-to-markdown-cli')
15
+ except importlib.metadata.PackageNotFoundError:
16
+ __version__ = 'unknown' # Fallback if package not installed
17
+
16
18
  parser = argparse.ArgumentParser(
17
19
  description="Process PDF files using Marker API.",
18
20
  formatter_class=argparse.ArgumentDefaultsHelpFormatter
19
21
  )
20
22
 
21
- # Required arguments
23
+ # Add version argument
24
+ parser.add_argument(
25
+ '--version',
26
+ action='version',
27
+ version=f'pdf-to-markdown-cli version: {__version__}'
28
+ )
29
+
22
30
  parser.add_argument("input", help="Input file or directory path")
23
31
 
24
- # Output format
25
32
  parser.add_argument("--json", action="store_true", help="Output in JSON format")
26
33
 
27
- # OCR settings
28
- parser.add_argument("--langs", default="English", help="Comma-separated OCR languages")
34
+ parser.add_argument("-l", "--langs", default="English", help="Comma-separated OCR languages")
29
35
  parser.add_argument("--llm", action="store_true", help="Use LLM for enhanced processing")
30
36
  parser.add_argument("--strip", action="store_true", help="Redo OCR processing")
31
37
  parser.add_argument("--noimg", action="store_true", help="Disable image extraction")
32
38
  parser.add_argument("--force", action="store_true", help="Force OCR on all pages")
33
39
  parser.add_argument("--pages", action="store_true", help="Add page delimiters")
34
- parser.add_argument("--max-pages", type=int, help="Maximum number of pages to process from the start of the file")
40
+ parser.add_argument("-mp", "--max-pages", type=int, help="Maximum number of pages to process from the start of the file")
35
41
 
36
- # Advanced settings
37
42
  parser.add_argument("--max", action="store_true", help="Enable all OCR enhancements (LLM, strip OCR, force OCR)")
38
43
  parser.add_argument("--no-chunk", action="store_true", help="Disable PDF chunking (sets chunk size to 1 million)")
39
44
  parser.add_argument("-cs", "--chunk-size", type=int, help="Set PDF chunk size in pages", default=25)
40
- parser.add_argument("--output-dir", help="Output directory", default="converted")
41
- parser.add_argument("--cache-dir", help="Cache directory", default=".docs_to_md_cache")
45
+ parser.add_argument("-o", "--output-dir", help="Absolute path to the output directory (default: same directory as input file)", default=None)
46
+
47
+ parser.add_argument("-v", "--verbose", action="store_true", help="Enable verbose (DEBUG level) logging")
42
48
 
43
49
  return parser.parse_args()
44
50
 
45
51
 
46
52
  def create_config_from_args() -> Config:
47
- """
48
- Create configuration from command line arguments.
49
-
50
- Returns:
51
- Config object with settings from command line
52
-
53
- Raises:
54
- ConfigurationError: If required arguments are missing
55
- """
56
53
  args = parse_args()
57
54
 
58
- # Get API key from environment
59
55
  try:
60
56
  api_key = get_env_var("MARKER_PDF_KEY")
61
57
  except Exception as e:
@@ -67,8 +63,7 @@ def create_config_from_args() -> Config:
67
63
  config = Config(
68
64
  api_key=api_key,
69
65
  input_path=args.input,
70
- output_dir=Path(args.output_dir),
71
- cache_dir=Path(args.cache_dir),
66
+ output_dir=Path(args.output_dir) if args.output_dir else None,
72
67
  output_format="json" if args.json else "markdown",
73
68
  langs=args.langs,
74
69
  use_llm=args.llm or args.max,
@@ -80,7 +75,13 @@ def create_config_from_args() -> Config:
80
75
  max_pages=args.max_pages
81
76
  )
82
77
 
83
- # Validate the configuration
84
78
  config.validate()
85
79
 
86
- return config
80
+ return config
81
+
82
+ def get_env_var(name: str, required: bool = True) -> Optional[str]:
83
+ """Get environment variable with optional requirement."""
84
+ value = os.getenv(name)
85
+ if required and not value:
86
+ raise FileError(f"Required environment variable {name} is not set")
87
+ return value