pdf-to-markdown-cli 1.1.0__tar.gz → 1.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/CHANGELOG.md +10 -0
- {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/CONTRIBUTING.md +6 -4
- {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/PKG-INFO +2 -1
- {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/README.md +1 -0
- {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/pyproject.toml +1 -1
- {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/src/docs_to_md/assemble.py +15 -2
- pdf_to_markdown_cli-1.2.0/src/docs_to_md/backends/__init__.py +31 -0
- pdf_to_markdown_cli-1.2.0/src/docs_to_md/backends/base.py +72 -0
- pdf_to_markdown_cli-1.2.0/src/docs_to_md/backends/datalab/__init__.py +5 -0
- pdf_to_markdown_cli-1.2.0/src/docs_to_md/backends/datalab/backend.py +119 -0
- {pdf_to_markdown_cli-1.1.0/src/docs_to_md → pdf_to_markdown_cli-1.2.0/src/docs_to_md/backends/datalab}/client.py +1 -1
- {pdf_to_markdown_cli-1.1.0/src/docs_to_md → pdf_to_markdown_cli-1.2.0/src/docs_to_md/backends/datalab}/models.py +2 -11
- {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/src/docs_to_md/cli.py +28 -15
- {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/src/docs_to_md/config.py +16 -27
- {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/src/docs_to_md/console.py +10 -0
- {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/src/docs_to_md/discovery.py +28 -16
- {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/src/docs_to_md/errors.py +1 -1
- {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/src/docs_to_md/pdf.py +12 -7
- {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/src/docs_to_md/pipeline.py +30 -84
- {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/src/pdf_to_markdown_cli.egg-info/PKG-INFO +2 -1
- {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/src/pdf_to_markdown_cli.egg-info/SOURCES.txt +9 -4
- {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/tests/conftest.py +49 -1
- {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/tests/test_cli.py +55 -12
- {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/tests/test_console.py +13 -1
- pdf_to_markdown_cli-1.2.0/tests/test_datalab_backend.py +206 -0
- pdf_to_markdown_cli-1.1.0/tests/test_client.py → pdf_to_markdown_cli-1.2.0/tests/test_datalab_client.py +2 -2
- pdf_to_markdown_cli-1.1.0/tests/test_models.py → pdf_to_markdown_cli-1.2.0/tests/test_datalab_models.py +1 -11
- {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/tests/test_discovery.py +17 -1
- pdf_to_markdown_cli-1.2.0/tests/test_pipeline.py +251 -0
- {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/tests/test_samples.py +5 -4
- pdf_to_markdown_cli-1.1.0/tests/test_pipeline.py +0 -297
- {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/LICENSE +0 -0
- {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/MANIFEST.in +0 -0
- {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/examples/README.md +0 -0
- {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/examples/alice_in_wonderland_sample.md +0 -0
- {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/examples/alice_in_wonderland_sample.pdf +0 -0
- {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/examples/alice_in_wonderland_sample_images/61972112770d9c8fb00883057954f885_img.jpg +0 -0
- {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/examples/census_1880_tables_en.md +0 -0
- {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/examples/census_1880_tables_en.pdf +0 -0
- {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/examples/census_1980_ancestry_en.md +0 -0
- {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/examples/census_1980_ancestry_en.pdf +0 -0
- {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/examples/darwin_origin_of_species_en.md +0 -0
- {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/examples/darwin_origin_of_species_en.pdf +0 -0
- {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/examples/darwin_origin_of_species_en_images/68ac34ff111db52afaa786afcb8346c3_img.jpg +0 -0
- {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/examples/equations.md +0 -0
- {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/examples/equations.pdf +0 -0
- {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/examples/equations_images/30a26f2d17ca95672702bf50fb4f0242_img.jpg +0 -0
- {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/examples/grimm_fairy_tales_de.md +0 -0
- {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/examples/grimm_fairy_tales_de.pdf +0 -0
- {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/examples/shijing_gupu_zh.md +0 -0
- {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/examples/shijing_gupu_zh.pdf +0 -0
- {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/examples/shijing_gupu_zh_images/13f8cc7f38c868748825f3a80b201b57_img.jpg +0 -0
- {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/examples/shijing_gupu_zh_images/8c378a184b5ae4d1605cb74d7b7a7e3f_img.jpg +0 -0
- {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/examples/shijing_gupu_zh_images/d5fc881e4328d6a2e76c9576408ced49_img.jpg +0 -0
- {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/examples/shijing_gupu_zh_images/eb70ac3e793aea24b49d2cd9f7b0b269_img.jpg +0 -0
- {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/examples/tolstoy_war_and_peace_ru.md +0 -0
- {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/examples/tolstoy_war_and_peace_ru.pdf +0 -0
- {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/setup.cfg +0 -0
- {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/src/docs_to_md/__init__.py +0 -0
- {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/src/docs_to_md/__main__.py +0 -0
- {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/src/docs_to_md/markdown.py +0 -0
- {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/src/pdf_to_markdown_cli.egg-info/dependency_links.txt +0 -0
- {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/src/pdf_to_markdown_cli.egg-info/entry_points.txt +0 -0
- {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/src/pdf_to_markdown_cli.egg-info/requires.txt +0 -0
- {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/src/pdf_to_markdown_cli.egg-info/top_level.txt +0 -0
- {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/tests/__init__.py +0 -0
- {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/tests/samples.py +0 -0
- {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/tests/test_assemble.py +0 -0
- {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/tests/test_live.py +0 -0
- {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/tests/test_markdown.py +0 -0
- {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/tests/test_pdf.py +0 -0
|
@@ -8,6 +8,16 @@ This project follows [Semantic Versioning](https://semver.org/spec/v2.0.0.html),
|
|
|
8
8
|
|
|
9
9
|
## [Unreleased]
|
|
10
10
|
|
|
11
|
+
## [1.2.0] - 2026-09-29
|
|
12
|
+
|
|
13
|
+
### Added
|
|
14
|
+
|
|
15
|
+
- `--backend` option for choosing the conversion service. `datalab` is the default and, for now, the only choice.
|
|
16
|
+
|
|
17
|
+
### Changed
|
|
18
|
+
|
|
19
|
+
- Conversion now goes through a pluggable backend interface (`docs_to_md.backends`), so other services or local engines can be added without touching the pipeline. Datalab behavior is unchanged. The Datalab client and API types moved from `docs_to_md.client` and `docs_to_md.models` to `docs_to_md.backends.datalab`.
|
|
20
|
+
|
|
11
21
|
## [1.1.0] - 2026-09-29
|
|
12
22
|
|
|
13
23
|
### Added
|
|
@@ -45,9 +45,10 @@ src/docs_to_md/
|
|
|
45
45
|
├── cli.py arguments, entry point, exit codes, summary
|
|
46
46
|
├── config.py validated runtime configuration
|
|
47
47
|
├── discovery.py input discovery, deterministic output names
|
|
48
|
-
├── pipeline.py split,
|
|
49
|
-
├──
|
|
50
|
-
├──
|
|
48
|
+
├── pipeline.py split, convert chunks concurrently, report
|
|
49
|
+
├── backends/ conversion backends, selected with --backend
|
|
50
|
+
│ ├── base.py Backend contract and BackendInfo capabilities
|
|
51
|
+
│ └── datalab/ Datalab Convert API: submit/poll, HTTP client, types
|
|
51
52
|
├── pdf.py page ranges and PDF splitting (pikepdf)
|
|
52
53
|
├── assemble.py merge chunks, renumber pages, write files
|
|
53
54
|
├── markdown.py Markdown clean-up (reflow, captions)
|
|
@@ -61,7 +62,8 @@ examples/ sample documents with reference outputs
|
|
|
61
62
|
|
|
62
63
|
- Python 3.11+, type-hinted, with `from __future__ import annotations`.
|
|
63
64
|
- Raise exceptions from `errors.py`. Use `FatalAPIError` only for problems that affect every request (bad key, no credits). It aborts the whole run.
|
|
64
|
-
- Keep
|
|
65
|
+
- Keep orchestration in `pipeline.py` and everything service-specific in its backend under `backends/`. The pipeline must not know which backend it runs. Pipeline tests use `FakeBackend` from `tests/conftest.py`. Datalab behavior is tested with `FakeClient` in `tests/test_datalab_backend.py`, and HTTP behavior with the fake session in `tests/test_datalab_client.py`.
|
|
66
|
+
- A new backend subclasses `Backend` (`backends/base.py`), declares what it accepts in a `BackendInfo`, and is registered in `BACKENDS` (`backends/__init__.py`).
|
|
65
67
|
- Add tests with every behavior change, in the module's test file. The suite is fast (under a second), so keep it that way.
|
|
66
68
|
- Update the docs in the same pull request: `README.md` for user-facing changes, `CHANGELOG.md` under **Unreleased**, and `AGENTS.md` when the module map or conventions change.
|
|
67
69
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: pdf-to-markdown-cli
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.2.0
|
|
4
4
|
Summary: Convert PDFs, Office documents, ebooks, and images to Markdown, HTML, or JSON with the Datalab API.
|
|
5
5
|
Author-email: Nikita Sokolsky <sokolx@gmail.com>
|
|
6
6
|
Maintainer-email: Nikita Sokolsky <sokolx@gmail.com>
|
|
@@ -218,6 +218,7 @@ pdf-to-md INPUT [INPUT ...] [options]
|
|
|
218
218
|
|
|
219
219
|
| Option | Description |
|
|
220
220
|
| --- | --- |
|
|
221
|
+
| `--backend NAME` | Conversion service. Default and currently only choice: `datalab`. |
|
|
221
222
|
| `-m`, `--mode MODE` | `fast` (the API default), `balanced`, or `accurate`. |
|
|
222
223
|
| `--paginate` | Insert page separators. |
|
|
223
224
|
| `--no-images` | Don't extract images. |
|
|
@@ -175,6 +175,7 @@ pdf-to-md INPUT [INPUT ...] [options]
|
|
|
175
175
|
|
|
176
176
|
| Option | Description |
|
|
177
177
|
| --- | --- |
|
|
178
|
+
| `--backend NAME` | Conversion service. Default and currently only choice: `datalab`. |
|
|
178
179
|
| `-m`, `--mode MODE` | `fast` (the API default), `balanced`, or `accurate`. |
|
|
179
180
|
| `--paginate` | Insert page separators. |
|
|
180
181
|
| `--no-images` | Don't extract images. |
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "pdf-to-markdown-cli"
|
|
7
|
-
version = "1.
|
|
7
|
+
version = "1.2.0"
|
|
8
8
|
description = "Convert PDFs, Office documents, ebooks, and images to Markdown, HTML, or JSON with the Datalab API."
|
|
9
9
|
readme = { file = "README.md", content-type = "text/markdown" }
|
|
10
10
|
authors = [
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
"""Combine per-chunk
|
|
1
|
+
"""Combine per-chunk backend results into one output document plus images."""
|
|
2
2
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
@@ -20,6 +20,13 @@ from docs_to_md.markdown import normalize_line_breaks, remove_duplicate_captions
|
|
|
20
20
|
|
|
21
21
|
logger = logging.getLogger(__name__)
|
|
22
22
|
|
|
23
|
+
# Output format -> file extension
|
|
24
|
+
OUTPUT_EXTENSIONS: dict[str, str] = {
|
|
25
|
+
"markdown": ".md",
|
|
26
|
+
"html": ".html",
|
|
27
|
+
"json": ".json",
|
|
28
|
+
}
|
|
29
|
+
|
|
23
30
|
_MD_PAGE_MARKER = re.compile(r"^\{(\d+)\}(-{48})$", re.MULTILINE)
|
|
24
31
|
_HTML_PAGE_ID = re.compile(r'(data-page-id=")(\d+)(")')
|
|
25
32
|
_BLOCK_ID = re.compile(r"(^|['\"])/page/(\d+)/") # block ids and <content-ref src='/page/N/...'>
|
|
@@ -30,7 +37,13 @@ _HTML_HEAD = re.compile(r"<head[^>]*>.*?</head>", re.DOTALL | re.IGNORECASE)
|
|
|
30
37
|
|
|
31
38
|
@dataclass
|
|
32
39
|
class ChunkOutput:
|
|
33
|
-
"""The result of converting one chunk.
|
|
40
|
+
"""The result of converting one chunk.
|
|
41
|
+
|
|
42
|
+
``content`` follows Marker's output conventions, with page numbers local to
|
|
43
|
+
the chunk (0 is its first page): Markdown ``{N}`` + 48 dashes separators and
|
|
44
|
+
``page-N-M`` anchors, HTML ``data-page-id``, and JSON ``/page/N/`` block ids
|
|
45
|
+
and ``page`` fields. Images are referenced by their key in ``images``.
|
|
46
|
+
"""
|
|
34
47
|
|
|
35
48
|
pages: Sequence[int] # source page numbers covered by this chunk, in order
|
|
36
49
|
content: Any # str for markdown/html, dict for json
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
"""Conversion backends, selected with ``--backend``.
|
|
2
|
+
|
|
3
|
+
To add one, subclass ``Backend`` (see ``base.py``) and list it in ``BACKENDS``.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
from docs_to_md.backends.base import Backend, BackendInfo, CancelCheck
|
|
9
|
+
from docs_to_md.backends.datalab import DatalabBackend
|
|
10
|
+
from docs_to_md.errors import ConfigurationError
|
|
11
|
+
|
|
12
|
+
DEFAULT_BACKEND = "datalab"
|
|
13
|
+
|
|
14
|
+
BACKENDS: dict[str, type[Backend]] = {backend.info.name: backend for backend in (DatalabBackend,)}
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def get_backend(name: str) -> type[Backend]:
|
|
18
|
+
try:
|
|
19
|
+
return BACKENDS[name]
|
|
20
|
+
except KeyError:
|
|
21
|
+
raise ConfigurationError(f"Unknown backend: {name} (available: {', '.join(BACKENDS)})") from None
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
__all__ = [
|
|
25
|
+
"BACKENDS",
|
|
26
|
+
"DEFAULT_BACKEND",
|
|
27
|
+
"Backend",
|
|
28
|
+
"BackendInfo",
|
|
29
|
+
"CancelCheck",
|
|
30
|
+
"get_backend",
|
|
31
|
+
]
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
"""The contract between the pipeline and a conversion backend.
|
|
2
|
+
|
|
3
|
+
The pipeline plans outputs, splits PDFs, schedules chunks on worker threads,
|
|
4
|
+
merges the results, and writes files. A backend only turns one chunk into
|
|
5
|
+
Markdown, HTML, or JSON.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import threading
|
|
11
|
+
from abc import ABC, abstractmethod
|
|
12
|
+
from collections.abc import Callable
|
|
13
|
+
from dataclasses import dataclass
|
|
14
|
+
from typing import TYPE_CHECKING, ClassVar
|
|
15
|
+
|
|
16
|
+
from docs_to_md.assemble import ChunkOutput
|
|
17
|
+
from docs_to_md.pdf import Chunk
|
|
18
|
+
|
|
19
|
+
if TYPE_CHECKING:
|
|
20
|
+
from docs_to_md.config import Config
|
|
21
|
+
|
|
22
|
+
CancelCheck = Callable[[], None]
|
|
23
|
+
"""Raises ``Cancelled`` once the run is interrupted or another chunk of the file failed."""
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@dataclass(frozen=True)
|
|
27
|
+
class BackendInfo:
|
|
28
|
+
"""What a backend accepts, used to validate the config before any work starts."""
|
|
29
|
+
|
|
30
|
+
name: str # the --backend value
|
|
31
|
+
label: str # human-readable name for messages
|
|
32
|
+
input_extensions: frozenset[str] # lowercase, without the dot
|
|
33
|
+
output_formats: frozenset[str] # keys of assemble.OUTPUT_EXTENSIONS
|
|
34
|
+
modes: tuple[str, ...] = () # accepted --mode values
|
|
35
|
+
max_upload_bytes: int | None = None # limit for a document sent whole
|
|
36
|
+
api_key_env_vars: tuple[str, ...] = () # lookup order; empty if no key is needed
|
|
37
|
+
api_key_url: str | None = None # where to get a key
|
|
38
|
+
|
|
39
|
+
@property
|
|
40
|
+
def requires_api_key(self) -> bool:
|
|
41
|
+
return bool(self.api_key_env_vars)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
class Backend(ABC):
|
|
45
|
+
"""Converts one chunk at a time.
|
|
46
|
+
|
|
47
|
+
``convert`` runs concurrently on up to ``--concurrency`` worker threads, so
|
|
48
|
+
implementations must be thread-safe. Errors decide how far a failure spreads:
|
|
49
|
+
|
|
50
|
+
- ``FatalAPIError`` affects every request (bad key, no credits) and stops the run.
|
|
51
|
+
- ``Cancelled`` comes from ``check_cancelled`` and must be allowed to propagate.
|
|
52
|
+
- Any other ``DocsToMdError`` fails only the file the chunk belongs to.
|
|
53
|
+
"""
|
|
54
|
+
|
|
55
|
+
info: ClassVar[BackendInfo]
|
|
56
|
+
|
|
57
|
+
@classmethod
|
|
58
|
+
@abstractmethod
|
|
59
|
+
def from_config(cls, config: Config, stop_event: threading.Event) -> Backend:
|
|
60
|
+
"""Build a backend for a validated ``config``.
|
|
61
|
+
|
|
62
|
+
``stop_event`` is set when the run is interrupted. Long waits should wake
|
|
63
|
+
up on it instead of sleeping through Ctrl-C.
|
|
64
|
+
"""
|
|
65
|
+
|
|
66
|
+
@abstractmethod
|
|
67
|
+
def convert(self, chunk: Chunk, check_cancelled: CancelCheck) -> ChunkOutput:
|
|
68
|
+
"""Convert ``chunk`` to ``config.output_format``, blocking until done.
|
|
69
|
+
|
|
70
|
+
Call ``check_cancelled`` between slow steps (uploads, polls, retries) so
|
|
71
|
+
an abandoned chunk stops spending time and credits.
|
|
72
|
+
"""
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
"""Convert chunks with the hosted Datalab Convert API: submit, then poll."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import logging
|
|
6
|
+
import threading
|
|
7
|
+
import time
|
|
8
|
+
from typing import TYPE_CHECKING
|
|
9
|
+
|
|
10
|
+
from docs_to_md.assemble import ChunkOutput
|
|
11
|
+
from docs_to_md.backends.base import Backend, BackendInfo, CancelCheck
|
|
12
|
+
from docs_to_md.backends.datalab.client import DatalabClient
|
|
13
|
+
from docs_to_md.backends.datalab.models import (
|
|
14
|
+
INPUT_MIME_TYPES,
|
|
15
|
+
MODES,
|
|
16
|
+
OUTPUT_FORMATS,
|
|
17
|
+
ConvertOptions,
|
|
18
|
+
ConvertResult,
|
|
19
|
+
)
|
|
20
|
+
from docs_to_md.console import format_elapsed
|
|
21
|
+
from docs_to_md.errors import APIError, RetryableAPIError
|
|
22
|
+
from docs_to_md.pdf import Chunk
|
|
23
|
+
|
|
24
|
+
if TYPE_CHECKING:
|
|
25
|
+
from docs_to_md.config import Config
|
|
26
|
+
|
|
27
|
+
logger = logging.getLogger(__name__)
|
|
28
|
+
|
|
29
|
+
INITIAL_POLL_SECONDS = 2.0
|
|
30
|
+
MAX_POLL_SECONDS = 15.0
|
|
31
|
+
MAX_RATE_LIMIT_RESUBMITS = 3
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class DatalabBackend(Backend):
|
|
35
|
+
info = BackendInfo(
|
|
36
|
+
name="datalab",
|
|
37
|
+
label="Datalab",
|
|
38
|
+
input_extensions=frozenset(INPUT_MIME_TYPES),
|
|
39
|
+
output_formats=OUTPUT_FORMATS,
|
|
40
|
+
modes=MODES,
|
|
41
|
+
max_upload_bytes=200 * 1024 * 1024,
|
|
42
|
+
api_key_env_vars=("DATALAB_API_KEY", "MARKER_PDF_KEY"),
|
|
43
|
+
api_key_url="https://www.datalab.to/app/keys",
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
def __init__(self, config: Config, client: DatalabClient):
|
|
47
|
+
self.config = config
|
|
48
|
+
self.client = client
|
|
49
|
+
|
|
50
|
+
@classmethod
|
|
51
|
+
def from_config(cls, config: Config, stop_event: threading.Event) -> DatalabBackend:
|
|
52
|
+
return cls(config, DatalabClient(config.api_key or "", stop_event=stop_event))
|
|
53
|
+
|
|
54
|
+
def options_for(self, chunk: Chunk) -> ConvertOptions:
|
|
55
|
+
"""Form fields for ``chunk``.
|
|
56
|
+
|
|
57
|
+
Split PDFs already contain only the selected pages, so ``page_range`` and
|
|
58
|
+
``max_pages`` are sent only for documents uploaded whole.
|
|
59
|
+
"""
|
|
60
|
+
config = self.config
|
|
61
|
+
select_remotely = not chunk.pages
|
|
62
|
+
return ConvertOptions(
|
|
63
|
+
output_format=config.output_format,
|
|
64
|
+
mode=config.mode,
|
|
65
|
+
paginate=config.paginate,
|
|
66
|
+
disable_image_extraction=config.disable_image_extraction,
|
|
67
|
+
disable_image_captions=config.disable_image_captions,
|
|
68
|
+
skip_cache=config.skip_cache,
|
|
69
|
+
page_range=config.page_range if select_remotely else None,
|
|
70
|
+
max_pages=config.max_pages if select_remotely else None,
|
|
71
|
+
extra=dict(config.extra_options),
|
|
72
|
+
)
|
|
73
|
+
|
|
74
|
+
def convert(self, chunk: Chunk, check_cancelled: CancelCheck) -> ChunkOutput:
|
|
75
|
+
options = self.options_for(chunk)
|
|
76
|
+
deadline = time.monotonic() + self.config.timeout
|
|
77
|
+
for attempt in range(MAX_RATE_LIMIT_RESUBMITS + 1):
|
|
78
|
+
check_cancelled()
|
|
79
|
+
request_id = self.client.submit(chunk.path, options)
|
|
80
|
+
result = self._wait_for_result(request_id, deadline, check_cancelled)
|
|
81
|
+
if result.status == "complete" and result.success is not False:
|
|
82
|
+
content = result.content_for(self.config.output_format)
|
|
83
|
+
if content is None:
|
|
84
|
+
raise APIError(f"the API returned no {self.config.output_format} output")
|
|
85
|
+
return ChunkOutput(
|
|
86
|
+
pages=chunk.pages,
|
|
87
|
+
content=content,
|
|
88
|
+
images=result.images,
|
|
89
|
+
page_count=result.page_count or len(chunk.pages),
|
|
90
|
+
cost_cents=result.cost_cents or 0.0,
|
|
91
|
+
)
|
|
92
|
+
error = result.error or f"conversion {result.status}"
|
|
93
|
+
# Page throughput limits surface as failed results, not HTTP 429.
|
|
94
|
+
if "rate limit" in error.lower() and attempt < MAX_RATE_LIMIT_RESUBMITS:
|
|
95
|
+
logger.debug("Request %s hit a rate limit; resubmitting", request_id)
|
|
96
|
+
self.client.sleep(30.0 * (attempt + 1))
|
|
97
|
+
continue
|
|
98
|
+
raise APIError(error)
|
|
99
|
+
raise AssertionError("unreachable")
|
|
100
|
+
|
|
101
|
+
def _wait_for_result(
|
|
102
|
+
self, request_id: str, deadline: float, check_cancelled: CancelCheck
|
|
103
|
+
) -> ConvertResult:
|
|
104
|
+
delay = INITIAL_POLL_SECONDS
|
|
105
|
+
while True:
|
|
106
|
+
check_cancelled()
|
|
107
|
+
try:
|
|
108
|
+
result = self.client.get_result(request_id)
|
|
109
|
+
if result.status != "processing":
|
|
110
|
+
return result
|
|
111
|
+
except RetryableAPIError as e:
|
|
112
|
+
logger.debug("Polling %s: %s", request_id, e)
|
|
113
|
+
if time.monotonic() + delay > deadline:
|
|
114
|
+
raise APIError(
|
|
115
|
+
f"timed out after {format_elapsed(self.config.timeout)} waiting for "
|
|
116
|
+
f"request {request_id} (raise --timeout for very large jobs)"
|
|
117
|
+
)
|
|
118
|
+
self.client.sleep(delay)
|
|
119
|
+
delay = min(delay * 1.5, MAX_POLL_SECONDS)
|
|
@@ -13,13 +13,13 @@ from typing import Any
|
|
|
13
13
|
|
|
14
14
|
import requests
|
|
15
15
|
|
|
16
|
+
from docs_to_md.backends.datalab.models import INPUT_MIME_TYPES, ConvertOptions, ConvertResult
|
|
16
17
|
from docs_to_md.errors import (
|
|
17
18
|
APIError,
|
|
18
19
|
Cancelled,
|
|
19
20
|
FatalAPIError,
|
|
20
21
|
RetryableAPIError,
|
|
21
22
|
)
|
|
22
|
-
from docs_to_md.models import INPUT_MIME_TYPES, ConvertOptions, ConvertResult
|
|
23
23
|
|
|
24
24
|
logger = logging.getLogger(__name__)
|
|
25
25
|
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
"""Datalab Convert API data types and supported
|
|
1
|
+
"""Datalab Convert API data types and supported formats.
|
|
2
2
|
|
|
3
3
|
Reference: https://documentation.datalab.to/api-reference/convert-document
|
|
4
4
|
"""
|
|
@@ -9,13 +9,7 @@ from dataclasses import dataclass, field
|
|
|
9
9
|
from typing import Any
|
|
10
10
|
|
|
11
11
|
MODES = ("fast", "balanced", "accurate")
|
|
12
|
-
|
|
13
|
-
# Output format -> file extension
|
|
14
|
-
SUPPORTED_FORMAT_EXTENSIONS: dict[str, str] = {
|
|
15
|
-
"markdown": ".md",
|
|
16
|
-
"json": ".json",
|
|
17
|
-
"html": ".html",
|
|
18
|
-
}
|
|
12
|
+
OUTPUT_FORMATS = frozenset({"markdown", "html", "json"})
|
|
19
13
|
|
|
20
14
|
# Input extension (without dot) -> MIME type sent to the API.
|
|
21
15
|
# https://documentation.datalab.to/docs/common/supportedfiletypes
|
|
@@ -45,9 +39,6 @@ INPUT_MIME_TYPES: dict[str, str] = {
|
|
|
45
39
|
"tif": "image/tiff",
|
|
46
40
|
}
|
|
47
41
|
|
|
48
|
-
SUPPORTED_INPUT_EXTENSIONS = frozenset(INPUT_MIME_TYPES)
|
|
49
|
-
SUPPORTED_MIME_TYPES = frozenset(INPUT_MIME_TYPES.values())
|
|
50
|
-
|
|
51
42
|
|
|
52
43
|
@dataclass
|
|
53
44
|
class ConvertOptions:
|
|
@@ -11,18 +11,18 @@ from collections.abc import Sequence
|
|
|
11
11
|
from pathlib import Path
|
|
12
12
|
|
|
13
13
|
from docs_to_md import __version__
|
|
14
|
-
from docs_to_md.
|
|
14
|
+
from docs_to_md.assemble import OUTPUT_EXTENSIONS
|
|
15
|
+
from docs_to_md.backends import BACKENDS, DEFAULT_BACKEND, get_backend
|
|
15
16
|
from docs_to_md.config import (
|
|
16
17
|
DEFAULT_CHUNK_SIZE,
|
|
17
18
|
DEFAULT_CONCURRENCY,
|
|
18
19
|
DEFAULT_TIMEOUT_SECONDS,
|
|
19
20
|
Config,
|
|
20
21
|
)
|
|
21
|
-
from docs_to_md.console import Console, setup_logging
|
|
22
|
+
from docs_to_md.console import Console, format_cost, format_elapsed, setup_logging
|
|
22
23
|
from docs_to_md.discovery import plan_jobs
|
|
23
24
|
from docs_to_md.errors import Cancelled, ConfigurationError, FatalAPIError
|
|
24
|
-
from docs_to_md.
|
|
25
|
-
from docs_to_md.pipeline import CONVERTED, FAILED, SKIPPED, Pipeline, format_cost, format_elapsed
|
|
25
|
+
from docs_to_md.pipeline import CONVERTED, FAILED, SKIPPED, Pipeline
|
|
26
26
|
|
|
27
27
|
logger = logging.getLogger(__name__)
|
|
28
28
|
|
|
@@ -31,8 +31,6 @@ EXIT_FAILURES = 1
|
|
|
31
31
|
EXIT_USAGE = 2
|
|
32
32
|
EXIT_INTERRUPTED = 130
|
|
33
33
|
|
|
34
|
-
API_KEY_ENV_VARS = ("DATALAB_API_KEY", "MARKER_PDF_KEY")
|
|
35
|
-
|
|
36
34
|
EPILOG = """\
|
|
37
35
|
examples:
|
|
38
36
|
pdf-to-md report.pdf convert to report.md (+ report_images/)
|
|
@@ -84,7 +82,7 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
84
82
|
"-f",
|
|
85
83
|
"--format",
|
|
86
84
|
dest="output_format",
|
|
87
|
-
choices=(
|
|
85
|
+
choices=tuple(OUTPUT_EXTENSIONS),
|
|
88
86
|
default="markdown",
|
|
89
87
|
help="output format (default: markdown)",
|
|
90
88
|
)
|
|
@@ -112,11 +110,17 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
112
110
|
)
|
|
113
111
|
|
|
114
112
|
conv = parser.add_argument_group("conversion")
|
|
113
|
+
conv.add_argument(
|
|
114
|
+
"--backend",
|
|
115
|
+
choices=tuple(BACKENDS),
|
|
116
|
+
default=DEFAULT_BACKEND,
|
|
117
|
+
help=f"conversion service (default: {DEFAULT_BACKEND})",
|
|
118
|
+
)
|
|
115
119
|
conv.add_argument(
|
|
116
120
|
"-m",
|
|
117
121
|
"--mode",
|
|
118
|
-
choices=
|
|
119
|
-
help="quality/speed trade-off;
|
|
122
|
+
choices=tuple(dict.fromkeys(mode for backend in BACKENDS.values() for mode in backend.info.modes)),
|
|
123
|
+
help="quality/speed trade-off; Datalab defaults to fast",
|
|
120
124
|
)
|
|
121
125
|
conv.add_argument("--paginate", action="store_true", help="insert page separators in the output")
|
|
122
126
|
conv.add_argument(
|
|
@@ -189,10 +193,11 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
189
193
|
return parser
|
|
190
194
|
|
|
191
195
|
|
|
192
|
-
def resolve_api_key(explicit: str | None) -> str | None:
|
|
196
|
+
def resolve_api_key(explicit: str | None, env_vars: Sequence[str]) -> str | None:
|
|
197
|
+
"""``--api-key`` if given, else the first non-empty variable in ``env_vars``."""
|
|
193
198
|
if explicit:
|
|
194
199
|
return explicit.strip()
|
|
195
|
-
for name in
|
|
200
|
+
for name in env_vars:
|
|
196
201
|
value = os.environ.get(name, "").strip()
|
|
197
202
|
if value:
|
|
198
203
|
return value
|
|
@@ -211,9 +216,11 @@ def config_from_args(args: argparse.Namespace, console: Console) -> Config:
|
|
|
211
216
|
if used:
|
|
212
217
|
console.warning(f"{flag} is no longer supported by the Datalab API and is ignored")
|
|
213
218
|
|
|
219
|
+
backend = get_backend(args.backend)
|
|
214
220
|
return Config(
|
|
215
221
|
inputs=list(args.inputs),
|
|
216
|
-
|
|
222
|
+
backend=backend.info.name,
|
|
223
|
+
api_key=resolve_api_key(args.api_key, backend.info.api_key_env_vars),
|
|
217
224
|
output_dir=args.output_dir,
|
|
218
225
|
output_format=args.output_format,
|
|
219
226
|
mode=mode,
|
|
@@ -265,12 +272,18 @@ def main(argv: Sequence[str] | None = None, console: Console | None = None) -> i
|
|
|
265
272
|
stop_event = threading.Event()
|
|
266
273
|
try:
|
|
267
274
|
config = config_from_args(args, console)
|
|
268
|
-
|
|
275
|
+
backend_type = get_backend(config.backend)
|
|
276
|
+
jobs = plan_jobs(
|
|
277
|
+
config.inputs,
|
|
278
|
+
config.output_format,
|
|
279
|
+
config.output_dir,
|
|
280
|
+
input_extensions=backend_type.info.input_extensions,
|
|
281
|
+
)
|
|
269
282
|
if not jobs:
|
|
270
283
|
console.error("No supported files to convert.")
|
|
271
284
|
return EXIT_FAILURES
|
|
272
|
-
|
|
273
|
-
pipeline = Pipeline(config, console,
|
|
285
|
+
backend = None if config.dry_run else backend_type.from_config(config, stop_event)
|
|
286
|
+
pipeline = Pipeline(config, console, backend=backend, stop_event=stop_event)
|
|
274
287
|
summary = pipeline.run(jobs)
|
|
275
288
|
except ConfigurationError as e:
|
|
276
289
|
console.error(str(e))
|
|
@@ -5,8 +5,9 @@ from __future__ import annotations
|
|
|
5
5
|
from dataclasses import dataclass, field
|
|
6
6
|
from pathlib import Path
|
|
7
7
|
|
|
8
|
+
from docs_to_md.assemble import OUTPUT_EXTENSIONS
|
|
9
|
+
from docs_to_md.backends import DEFAULT_BACKEND, get_backend
|
|
8
10
|
from docs_to_md.errors import ConfigurationError
|
|
9
|
-
from docs_to_md.models import MODES, SUPPORTED_FORMAT_EXTENSIONS, ConvertOptions
|
|
10
11
|
from docs_to_md.pdf import validate_page_range
|
|
11
12
|
|
|
12
13
|
DEFAULT_CHUNK_SIZE = 25
|
|
@@ -17,6 +18,7 @@ DEFAULT_TIMEOUT_SECONDS = 3600.0
|
|
|
17
18
|
@dataclass
|
|
18
19
|
class Config:
|
|
19
20
|
inputs: list[Path]
|
|
21
|
+
backend: str = DEFAULT_BACKEND
|
|
20
22
|
api_key: str | None = None
|
|
21
23
|
output_dir: Path | None = None
|
|
22
24
|
output_format: str = "markdown"
|
|
@@ -33,20 +35,24 @@ class Config:
|
|
|
33
35
|
overwrite: bool = False
|
|
34
36
|
dry_run: bool = False
|
|
35
37
|
reflow_markdown: bool = True
|
|
36
|
-
extra_options: dict = field(default_factory=dict)
|
|
38
|
+
extra_options: dict = field(default_factory=dict) # passed through to the backend
|
|
37
39
|
|
|
38
40
|
def validate(self) -> Config:
|
|
41
|
+
info = get_backend(self.backend).info
|
|
39
42
|
if not self.inputs:
|
|
40
43
|
raise ConfigurationError("At least one input path is required")
|
|
41
|
-
if not self.dry_run and not self.api_key:
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
)
|
|
46
|
-
if self.output_format not in
|
|
44
|
+
if not self.dry_run and info.requires_api_key and not self.api_key:
|
|
45
|
+
primary, *others = info.api_key_env_vars
|
|
46
|
+
also = f" (or {', '.join(others)})" if others else ""
|
|
47
|
+
where = f" Get a key at {info.api_key_url}" if info.api_key_url else ""
|
|
48
|
+
raise ConfigurationError(f"No API key found. Set {primary}{also}, or pass --api-key.{where}")
|
|
49
|
+
if self.output_format not in OUTPUT_EXTENSIONS:
|
|
47
50
|
raise ConfigurationError(f"Unsupported output format: {self.output_format}")
|
|
48
|
-
if self.
|
|
49
|
-
raise ConfigurationError(f"
|
|
51
|
+
if self.output_format not in info.output_formats:
|
|
52
|
+
raise ConfigurationError(f"The {info.label} backend cannot produce {self.output_format} output")
|
|
53
|
+
if self.mode is not None and self.mode not in info.modes:
|
|
54
|
+
choices = f" (choose from {', '.join(info.modes)})" if info.modes else ""
|
|
55
|
+
raise ConfigurationError(f"Unsupported mode for {info.label}: {self.mode}{choices}")
|
|
50
56
|
if self.chunk_size is not None and self.chunk_size < 1:
|
|
51
57
|
raise ConfigurationError("--chunk-size must be at least 1")
|
|
52
58
|
if self.max_pages is not None and self.max_pages < 1:
|
|
@@ -62,20 +68,3 @@ class Config:
|
|
|
62
68
|
if self.output_dir.exists() and not self.output_dir.is_dir():
|
|
63
69
|
raise ConfigurationError(f"Output path is not a directory: {self.output_dir}")
|
|
64
70
|
return self
|
|
65
|
-
|
|
66
|
-
def convert_options(self, *, local_page_selection: bool) -> ConvertOptions:
|
|
67
|
-
"""API options for one request.
|
|
68
|
-
|
|
69
|
-
PDFs are split locally, so their page selection is not sent to the API.
|
|
70
|
-
"""
|
|
71
|
-
return ConvertOptions(
|
|
72
|
-
output_format=self.output_format,
|
|
73
|
-
mode=self.mode,
|
|
74
|
-
paginate=self.paginate,
|
|
75
|
-
disable_image_extraction=self.disable_image_extraction,
|
|
76
|
-
disable_image_captions=self.disable_image_captions,
|
|
77
|
-
skip_cache=self.skip_cache,
|
|
78
|
-
page_range=None if local_page_selection else self.page_range,
|
|
79
|
-
max_pages=None if local_page_selection else self.max_pages,
|
|
80
|
-
extra=dict(self.extra_options),
|
|
81
|
-
)
|
|
@@ -15,6 +15,16 @@ from typing import TextIO
|
|
|
15
15
|
from tqdm import tqdm
|
|
16
16
|
|
|
17
17
|
|
|
18
|
+
def format_cost(cents: float) -> str:
|
|
19
|
+
dollars = cents / 100
|
|
20
|
+
return f"${dollars:.2f}" if dollars >= 0.01 or dollars == 0 else f"${dollars:.4f}"
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def format_elapsed(seconds: float) -> str:
|
|
24
|
+
minutes, secs = divmod(round(seconds), 60)
|
|
25
|
+
return f"{minutes}m{secs:02d}s" if minutes else f"{seconds:.1f}s"
|
|
26
|
+
|
|
27
|
+
|
|
18
28
|
def setup_logging(verbose: bool) -> None:
|
|
19
29
|
handler = logging.StreamHandler(sys.stderr)
|
|
20
30
|
if verbose:
|