pdf-to-markdown-cli 1.1.0__tar.gz → 1.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/CHANGELOG.md +10 -0
  2. {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/CONTRIBUTING.md +6 -4
  3. {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/PKG-INFO +2 -1
  4. {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/README.md +1 -0
  5. {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/pyproject.toml +1 -1
  6. {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/src/docs_to_md/assemble.py +15 -2
  7. pdf_to_markdown_cli-1.2.0/src/docs_to_md/backends/__init__.py +31 -0
  8. pdf_to_markdown_cli-1.2.0/src/docs_to_md/backends/base.py +72 -0
  9. pdf_to_markdown_cli-1.2.0/src/docs_to_md/backends/datalab/__init__.py +5 -0
  10. pdf_to_markdown_cli-1.2.0/src/docs_to_md/backends/datalab/backend.py +119 -0
  11. {pdf_to_markdown_cli-1.1.0/src/docs_to_md → pdf_to_markdown_cli-1.2.0/src/docs_to_md/backends/datalab}/client.py +1 -1
  12. {pdf_to_markdown_cli-1.1.0/src/docs_to_md → pdf_to_markdown_cli-1.2.0/src/docs_to_md/backends/datalab}/models.py +2 -11
  13. {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/src/docs_to_md/cli.py +28 -15
  14. {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/src/docs_to_md/config.py +16 -27
  15. {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/src/docs_to_md/console.py +10 -0
  16. {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/src/docs_to_md/discovery.py +28 -16
  17. {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/src/docs_to_md/errors.py +1 -1
  18. {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/src/docs_to_md/pdf.py +12 -7
  19. {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/src/docs_to_md/pipeline.py +30 -84
  20. {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/src/pdf_to_markdown_cli.egg-info/PKG-INFO +2 -1
  21. {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/src/pdf_to_markdown_cli.egg-info/SOURCES.txt +9 -4
  22. {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/tests/conftest.py +49 -1
  23. {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/tests/test_cli.py +55 -12
  24. {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/tests/test_console.py +13 -1
  25. pdf_to_markdown_cli-1.2.0/tests/test_datalab_backend.py +206 -0
  26. pdf_to_markdown_cli-1.1.0/tests/test_client.py → pdf_to_markdown_cli-1.2.0/tests/test_datalab_client.py +2 -2
  27. pdf_to_markdown_cli-1.1.0/tests/test_models.py → pdf_to_markdown_cli-1.2.0/tests/test_datalab_models.py +1 -11
  28. {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/tests/test_discovery.py +17 -1
  29. pdf_to_markdown_cli-1.2.0/tests/test_pipeline.py +251 -0
  30. {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/tests/test_samples.py +5 -4
  31. pdf_to_markdown_cli-1.1.0/tests/test_pipeline.py +0 -297
  32. {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/LICENSE +0 -0
  33. {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/MANIFEST.in +0 -0
  34. {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/examples/README.md +0 -0
  35. {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/examples/alice_in_wonderland_sample.md +0 -0
  36. {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/examples/alice_in_wonderland_sample.pdf +0 -0
  37. {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/examples/alice_in_wonderland_sample_images/61972112770d9c8fb00883057954f885_img.jpg +0 -0
  38. {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/examples/census_1880_tables_en.md +0 -0
  39. {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/examples/census_1880_tables_en.pdf +0 -0
  40. {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/examples/census_1980_ancestry_en.md +0 -0
  41. {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/examples/census_1980_ancestry_en.pdf +0 -0
  42. {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/examples/darwin_origin_of_species_en.md +0 -0
  43. {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/examples/darwin_origin_of_species_en.pdf +0 -0
  44. {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/examples/darwin_origin_of_species_en_images/68ac34ff111db52afaa786afcb8346c3_img.jpg +0 -0
  45. {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/examples/equations.md +0 -0
  46. {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/examples/equations.pdf +0 -0
  47. {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/examples/equations_images/30a26f2d17ca95672702bf50fb4f0242_img.jpg +0 -0
  48. {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/examples/grimm_fairy_tales_de.md +0 -0
  49. {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/examples/grimm_fairy_tales_de.pdf +0 -0
  50. {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/examples/shijing_gupu_zh.md +0 -0
  51. {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/examples/shijing_gupu_zh.pdf +0 -0
  52. {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/examples/shijing_gupu_zh_images/13f8cc7f38c868748825f3a80b201b57_img.jpg +0 -0
  53. {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/examples/shijing_gupu_zh_images/8c378a184b5ae4d1605cb74d7b7a7e3f_img.jpg +0 -0
  54. {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/examples/shijing_gupu_zh_images/d5fc881e4328d6a2e76c9576408ced49_img.jpg +0 -0
  55. {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/examples/shijing_gupu_zh_images/eb70ac3e793aea24b49d2cd9f7b0b269_img.jpg +0 -0
  56. {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/examples/tolstoy_war_and_peace_ru.md +0 -0
  57. {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/examples/tolstoy_war_and_peace_ru.pdf +0 -0
  58. {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/setup.cfg +0 -0
  59. {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/src/docs_to_md/__init__.py +0 -0
  60. {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/src/docs_to_md/__main__.py +0 -0
  61. {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/src/docs_to_md/markdown.py +0 -0
  62. {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/src/pdf_to_markdown_cli.egg-info/dependency_links.txt +0 -0
  63. {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/src/pdf_to_markdown_cli.egg-info/entry_points.txt +0 -0
  64. {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/src/pdf_to_markdown_cli.egg-info/requires.txt +0 -0
  65. {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/src/pdf_to_markdown_cli.egg-info/top_level.txt +0 -0
  66. {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/tests/__init__.py +0 -0
  67. {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/tests/samples.py +0 -0
  68. {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/tests/test_assemble.py +0 -0
  69. {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/tests/test_live.py +0 -0
  70. {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/tests/test_markdown.py +0 -0
  71. {pdf_to_markdown_cli-1.1.0 → pdf_to_markdown_cli-1.2.0}/tests/test_pdf.py +0 -0
@@ -8,6 +8,16 @@ This project follows [Semantic Versioning](https://semver.org/spec/v2.0.0.html),
8
8
 
9
9
  ## [Unreleased]
10
10
 
11
+ ## [1.2.0] - 2026-09-29
12
+
13
+ ### Added
14
+
15
+ - `--backend` option for choosing the conversion service. `datalab` is the default and, for now, the only choice.
16
+
17
+ ### Changed
18
+
19
+ - Conversion now goes through a pluggable backend interface (`docs_to_md.backends`), so other services or local engines can be added without touching the pipeline. Datalab behavior is unchanged. The Datalab client and API types moved from `docs_to_md.client` and `docs_to_md.models` to `docs_to_md.backends.datalab`.
20
+
11
21
  ## [1.1.0] - 2026-09-29
12
22
 
13
23
  ### Added
@@ -45,9 +45,10 @@ src/docs_to_md/
45
45
  ├── cli.py arguments, entry point, exit codes, summary
46
46
  ├── config.py validated runtime configuration
47
47
  ├── discovery.py input discovery, deterministic output names
48
- ├── pipeline.py split, submit concurrently, poll, report
49
- ├── client.py API client: retries, timeouts, error types
50
- ├── models.py API request/response types, formats
48
+ ├── pipeline.py split, convert chunks concurrently, report
49
+ ├── backends/ conversion backends, selected with --backend
50
+ │ ├── base.py Backend contract and BackendInfo capabilities
51
+ │ └── datalab/ Datalab Convert API: submit/poll, HTTP client, types
51
52
  ├── pdf.py page ranges and PDF splitting (pikepdf)
52
53
  ├── assemble.py merge chunks, renumber pages, write files
53
54
  ├── markdown.py Markdown clean-up (reflow, captions)
@@ -61,7 +62,8 @@ examples/ sample documents with reference outputs
61
62
 
62
63
  - Python 3.11+, type-hinted, with `from __future__ import annotations`.
63
64
  - Raise exceptions from `errors.py`. Use `FatalAPIError` only for problems that affect every request (bad key, no credits). It aborts the whole run.
64
- - Keep network code in `client.py` and orchestration in `pipeline.py`. Pipeline tests use `FakeClient` from `tests/conftest.py`. HTTP behavior is tested with the fake session in `tests/test_client.py`.
65
+ - Keep orchestration in `pipeline.py` and everything service-specific in its backend under `backends/`. The pipeline must not know which backend it runs. Pipeline tests use `FakeBackend` from `tests/conftest.py`. Datalab behavior is tested with `FakeClient` in `tests/test_datalab_backend.py`, and HTTP behavior with the fake session in `tests/test_datalab_client.py`.
66
+ - A new backend subclasses `Backend` (`backends/base.py`), declares what it accepts in a `BackendInfo`, and is registered in `BACKENDS` (`backends/__init__.py`).
65
67
  - Add tests with every behavior change, in the module's test file. The suite is fast (under a second), so keep it that way.
66
68
  - Update the docs in the same pull request: `README.md` for user-facing changes, `CHANGELOG.md` under **Unreleased**, and `AGENTS.md` when the module map or conventions change.
67
69
 
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: pdf-to-markdown-cli
3
- Version: 1.1.0
3
+ Version: 1.2.0
4
4
  Summary: Convert PDFs, Office documents, ebooks, and images to Markdown, HTML, or JSON with the Datalab API.
5
5
  Author-email: Nikita Sokolsky <sokolx@gmail.com>
6
6
  Maintainer-email: Nikita Sokolsky <sokolx@gmail.com>
@@ -218,6 +218,7 @@ pdf-to-md INPUT [INPUT ...] [options]
218
218
 
219
219
  | Option | Description |
220
220
  | --- | --- |
221
+ | `--backend NAME` | Conversion service. Default and currently only choice: `datalab`. |
221
222
  | `-m`, `--mode MODE` | `fast` (the API default), `balanced`, or `accurate`. |
222
223
  | `--paginate` | Insert page separators. |
223
224
  | `--no-images` | Don't extract images. |
@@ -175,6 +175,7 @@ pdf-to-md INPUT [INPUT ...] [options]
175
175
 
176
176
  | Option | Description |
177
177
  | --- | --- |
178
+ | `--backend NAME` | Conversion service. Default and currently only choice: `datalab`. |
178
179
  | `-m`, `--mode MODE` | `fast` (the API default), `balanced`, or `accurate`. |
179
180
  | `--paginate` | Insert page separators. |
180
181
  | `--no-images` | Don't extract images. |
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "pdf-to-markdown-cli"
7
- version = "1.1.0"
7
+ version = "1.2.0"
8
8
  description = "Convert PDFs, Office documents, ebooks, and images to Markdown, HTML, or JSON with the Datalab API."
9
9
  readme = { file = "README.md", content-type = "text/markdown" }
10
10
  authors = [
@@ -1,4 +1,4 @@
1
- """Combine per-chunk API results into one output document plus images."""
1
+ """Combine per-chunk backend results into one output document plus images."""
2
2
 
3
3
  from __future__ import annotations
4
4
 
@@ -20,6 +20,13 @@ from docs_to_md.markdown import normalize_line_breaks, remove_duplicate_captions
20
20
 
21
21
  logger = logging.getLogger(__name__)
22
22
 
23
+ # Output format -> file extension
24
+ OUTPUT_EXTENSIONS: dict[str, str] = {
25
+ "markdown": ".md",
26
+ "html": ".html",
27
+ "json": ".json",
28
+ }
29
+
23
30
  _MD_PAGE_MARKER = re.compile(r"^\{(\d+)\}(-{48})$", re.MULTILINE)
24
31
  _HTML_PAGE_ID = re.compile(r'(data-page-id=")(\d+)(")')
25
32
  _BLOCK_ID = re.compile(r"(^|['\"])/page/(\d+)/") # block ids and <content-ref src='/page/N/...'>
@@ -30,7 +37,13 @@ _HTML_HEAD = re.compile(r"<head[^>]*>.*?</head>", re.DOTALL | re.IGNORECASE)
30
37
 
31
38
  @dataclass
32
39
  class ChunkOutput:
33
- """The result of converting one chunk."""
40
+ """The result of converting one chunk.
41
+
42
+ ``content`` follows Marker's output conventions, with page numbers local to
43
+ the chunk (0 is its first page): Markdown ``{N}`` + 48 dashes separators and
44
+ ``page-N-M`` anchors, HTML ``data-page-id``, and JSON ``/page/N/`` block ids
45
+ and ``page`` fields. Images are referenced by their key in ``images``.
46
+ """
34
47
 
35
48
  pages: Sequence[int] # source page numbers covered by this chunk, in order
36
49
  content: Any # str for markdown/html, dict for json
@@ -0,0 +1,31 @@
1
+ """Conversion backends, selected with ``--backend``.
2
+
3
+ To add one, subclass ``Backend`` (see ``base.py``) and list it in ``BACKENDS``.
4
+ """
5
+
6
+ from __future__ import annotations
7
+
8
+ from docs_to_md.backends.base import Backend, BackendInfo, CancelCheck
9
+ from docs_to_md.backends.datalab import DatalabBackend
10
+ from docs_to_md.errors import ConfigurationError
11
+
12
+ DEFAULT_BACKEND = "datalab"
13
+
14
+ BACKENDS: dict[str, type[Backend]] = {backend.info.name: backend for backend in (DatalabBackend,)}
15
+
16
+
17
+ def get_backend(name: str) -> type[Backend]:
18
+ try:
19
+ return BACKENDS[name]
20
+ except KeyError:
21
+ raise ConfigurationError(f"Unknown backend: {name} (available: {', '.join(BACKENDS)})") from None
22
+
23
+
24
+ __all__ = [
25
+ "BACKENDS",
26
+ "DEFAULT_BACKEND",
27
+ "Backend",
28
+ "BackendInfo",
29
+ "CancelCheck",
30
+ "get_backend",
31
+ ]
@@ -0,0 +1,72 @@
1
+ """The contract between the pipeline and a conversion backend.
2
+
3
+ The pipeline plans outputs, splits PDFs, schedules chunks on worker threads,
4
+ merges the results, and writes files. A backend only turns one chunk into
5
+ Markdown, HTML, or JSON.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import threading
11
+ from abc import ABC, abstractmethod
12
+ from collections.abc import Callable
13
+ from dataclasses import dataclass
14
+ from typing import TYPE_CHECKING, ClassVar
15
+
16
+ from docs_to_md.assemble import ChunkOutput
17
+ from docs_to_md.pdf import Chunk
18
+
19
+ if TYPE_CHECKING:
20
+ from docs_to_md.config import Config
21
+
22
+ CancelCheck = Callable[[], None]
23
+ """Raises ``Cancelled`` once the run is interrupted or another chunk of the file failed."""
24
+
25
+
26
+ @dataclass(frozen=True)
27
+ class BackendInfo:
28
+ """What a backend accepts, used to validate the config before any work starts."""
29
+
30
+ name: str # the --backend value
31
+ label: str # human-readable name for messages
32
+ input_extensions: frozenset[str] # lowercase, without the dot
33
+ output_formats: frozenset[str] # keys of assemble.OUTPUT_EXTENSIONS
34
+ modes: tuple[str, ...] = () # accepted --mode values
35
+ max_upload_bytes: int | None = None # limit for a document sent whole
36
+ api_key_env_vars: tuple[str, ...] = () # lookup order; empty if no key is needed
37
+ api_key_url: str | None = None # where to get a key
38
+
39
+ @property
40
+ def requires_api_key(self) -> bool:
41
+ return bool(self.api_key_env_vars)
42
+
43
+
44
+ class Backend(ABC):
45
+ """Converts one chunk at a time.
46
+
47
+ ``convert`` runs concurrently on up to ``--concurrency`` worker threads, so
48
+ implementations must be thread-safe. Errors decide how far a failure spreads:
49
+
50
+ - ``FatalAPIError`` affects every request (bad key, no credits) and stops the run.
51
+ - ``Cancelled`` comes from ``check_cancelled`` and must be allowed to propagate.
52
+ - Any other ``DocsToMdError`` fails only the file the chunk belongs to.
53
+ """
54
+
55
+ info: ClassVar[BackendInfo]
56
+
57
+ @classmethod
58
+ @abstractmethod
59
+ def from_config(cls, config: Config, stop_event: threading.Event) -> Backend:
60
+ """Build a backend for a validated ``config``.
61
+
62
+ ``stop_event`` is set when the run is interrupted. Long waits should wake
63
+ up on it instead of sleeping through Ctrl-C.
64
+ """
65
+
66
+ @abstractmethod
67
+ def convert(self, chunk: Chunk, check_cancelled: CancelCheck) -> ChunkOutput:
68
+ """Convert ``chunk`` to ``config.output_format``, blocking until done.
69
+
70
+ Call ``check_cancelled`` between slow steps (uploads, polls, retries) so
71
+ an abandoned chunk stops spending time and credits.
72
+ """
@@ -0,0 +1,5 @@
1
+ """The hosted Datalab Convert API (https://www.datalab.to)."""
2
+
3
+ from docs_to_md.backends.datalab.backend import DatalabBackend
4
+
5
+ __all__ = ["DatalabBackend"]
@@ -0,0 +1,119 @@
1
+ """Convert chunks with the hosted Datalab Convert API: submit, then poll."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import logging
6
+ import threading
7
+ import time
8
+ from typing import TYPE_CHECKING
9
+
10
+ from docs_to_md.assemble import ChunkOutput
11
+ from docs_to_md.backends.base import Backend, BackendInfo, CancelCheck
12
+ from docs_to_md.backends.datalab.client import DatalabClient
13
+ from docs_to_md.backends.datalab.models import (
14
+ INPUT_MIME_TYPES,
15
+ MODES,
16
+ OUTPUT_FORMATS,
17
+ ConvertOptions,
18
+ ConvertResult,
19
+ )
20
+ from docs_to_md.console import format_elapsed
21
+ from docs_to_md.errors import APIError, RetryableAPIError
22
+ from docs_to_md.pdf import Chunk
23
+
24
+ if TYPE_CHECKING:
25
+ from docs_to_md.config import Config
26
+
27
+ logger = logging.getLogger(__name__)
28
+
29
+ INITIAL_POLL_SECONDS = 2.0
30
+ MAX_POLL_SECONDS = 15.0
31
+ MAX_RATE_LIMIT_RESUBMITS = 3
32
+
33
+
34
+ class DatalabBackend(Backend):
35
+ info = BackendInfo(
36
+ name="datalab",
37
+ label="Datalab",
38
+ input_extensions=frozenset(INPUT_MIME_TYPES),
39
+ output_formats=OUTPUT_FORMATS,
40
+ modes=MODES,
41
+ max_upload_bytes=200 * 1024 * 1024,
42
+ api_key_env_vars=("DATALAB_API_KEY", "MARKER_PDF_KEY"),
43
+ api_key_url="https://www.datalab.to/app/keys",
44
+ )
45
+
46
+ def __init__(self, config: Config, client: DatalabClient):
47
+ self.config = config
48
+ self.client = client
49
+
50
+ @classmethod
51
+ def from_config(cls, config: Config, stop_event: threading.Event) -> DatalabBackend:
52
+ return cls(config, DatalabClient(config.api_key or "", stop_event=stop_event))
53
+
54
+ def options_for(self, chunk: Chunk) -> ConvertOptions:
55
+ """Form fields for ``chunk``.
56
+
57
+ Split PDFs already contain only the selected pages, so ``page_range`` and
58
+ ``max_pages`` are sent only for documents uploaded whole.
59
+ """
60
+ config = self.config
61
+ select_remotely = not chunk.pages
62
+ return ConvertOptions(
63
+ output_format=config.output_format,
64
+ mode=config.mode,
65
+ paginate=config.paginate,
66
+ disable_image_extraction=config.disable_image_extraction,
67
+ disable_image_captions=config.disable_image_captions,
68
+ skip_cache=config.skip_cache,
69
+ page_range=config.page_range if select_remotely else None,
70
+ max_pages=config.max_pages if select_remotely else None,
71
+ extra=dict(config.extra_options),
72
+ )
73
+
74
+ def convert(self, chunk: Chunk, check_cancelled: CancelCheck) -> ChunkOutput:
75
+ options = self.options_for(chunk)
76
+ deadline = time.monotonic() + self.config.timeout
77
+ for attempt in range(MAX_RATE_LIMIT_RESUBMITS + 1):
78
+ check_cancelled()
79
+ request_id = self.client.submit(chunk.path, options)
80
+ result = self._wait_for_result(request_id, deadline, check_cancelled)
81
+ if result.status == "complete" and result.success is not False:
82
+ content = result.content_for(self.config.output_format)
83
+ if content is None:
84
+ raise APIError(f"the API returned no {self.config.output_format} output")
85
+ return ChunkOutput(
86
+ pages=chunk.pages,
87
+ content=content,
88
+ images=result.images,
89
+ page_count=result.page_count or len(chunk.pages),
90
+ cost_cents=result.cost_cents or 0.0,
91
+ )
92
+ error = result.error or f"conversion {result.status}"
93
+ # Page throughput limits surface as failed results, not HTTP 429.
94
+ if "rate limit" in error.lower() and attempt < MAX_RATE_LIMIT_RESUBMITS:
95
+ logger.debug("Request %s hit a rate limit; resubmitting", request_id)
96
+ self.client.sleep(30.0 * (attempt + 1))
97
+ continue
98
+ raise APIError(error)
99
+ raise AssertionError("unreachable")
100
+
101
+ def _wait_for_result(
102
+ self, request_id: str, deadline: float, check_cancelled: CancelCheck
103
+ ) -> ConvertResult:
104
+ delay = INITIAL_POLL_SECONDS
105
+ while True:
106
+ check_cancelled()
107
+ try:
108
+ result = self.client.get_result(request_id)
109
+ if result.status != "processing":
110
+ return result
111
+ except RetryableAPIError as e:
112
+ logger.debug("Polling %s: %s", request_id, e)
113
+ if time.monotonic() + delay > deadline:
114
+ raise APIError(
115
+ f"timed out after {format_elapsed(self.config.timeout)} waiting for "
116
+ f"request {request_id} (raise --timeout for very large jobs)"
117
+ )
118
+ self.client.sleep(delay)
119
+ delay = min(delay * 1.5, MAX_POLL_SECONDS)
@@ -13,13 +13,13 @@ from typing import Any
13
13
 
14
14
  import requests
15
15
 
16
+ from docs_to_md.backends.datalab.models import INPUT_MIME_TYPES, ConvertOptions, ConvertResult
16
17
  from docs_to_md.errors import (
17
18
  APIError,
18
19
  Cancelled,
19
20
  FatalAPIError,
20
21
  RetryableAPIError,
21
22
  )
22
- from docs_to_md.models import INPUT_MIME_TYPES, ConvertOptions, ConvertResult
23
23
 
24
24
  logger = logging.getLogger(__name__)
25
25
 
@@ -1,4 +1,4 @@
1
- """Datalab Convert API data types and supported format tables.
1
+ """Datalab Convert API data types and supported formats.
2
2
 
3
3
  Reference: https://documentation.datalab.to/api-reference/convert-document
4
4
  """
@@ -9,13 +9,7 @@ from dataclasses import dataclass, field
9
9
  from typing import Any
10
10
 
11
11
  MODES = ("fast", "balanced", "accurate")
12
-
13
- # Output format -> file extension
14
- SUPPORTED_FORMAT_EXTENSIONS: dict[str, str] = {
15
- "markdown": ".md",
16
- "json": ".json",
17
- "html": ".html",
18
- }
12
+ OUTPUT_FORMATS = frozenset({"markdown", "html", "json"})
19
13
 
20
14
  # Input extension (without dot) -> MIME type sent to the API.
21
15
  # https://documentation.datalab.to/docs/common/supportedfiletypes
@@ -45,9 +39,6 @@ INPUT_MIME_TYPES: dict[str, str] = {
45
39
  "tif": "image/tiff",
46
40
  }
47
41
 
48
- SUPPORTED_INPUT_EXTENSIONS = frozenset(INPUT_MIME_TYPES)
49
- SUPPORTED_MIME_TYPES = frozenset(INPUT_MIME_TYPES.values())
50
-
51
42
 
52
43
  @dataclass
53
44
  class ConvertOptions:
@@ -11,18 +11,18 @@ from collections.abc import Sequence
11
11
  from pathlib import Path
12
12
 
13
13
  from docs_to_md import __version__
14
- from docs_to_md.client import DatalabClient
14
+ from docs_to_md.assemble import OUTPUT_EXTENSIONS
15
+ from docs_to_md.backends import BACKENDS, DEFAULT_BACKEND, get_backend
15
16
  from docs_to_md.config import (
16
17
  DEFAULT_CHUNK_SIZE,
17
18
  DEFAULT_CONCURRENCY,
18
19
  DEFAULT_TIMEOUT_SECONDS,
19
20
  Config,
20
21
  )
21
- from docs_to_md.console import Console, setup_logging
22
+ from docs_to_md.console import Console, format_cost, format_elapsed, setup_logging
22
23
  from docs_to_md.discovery import plan_jobs
23
24
  from docs_to_md.errors import Cancelled, ConfigurationError, FatalAPIError
24
- from docs_to_md.models import MODES
25
- from docs_to_md.pipeline import CONVERTED, FAILED, SKIPPED, Pipeline, format_cost, format_elapsed
25
+ from docs_to_md.pipeline import CONVERTED, FAILED, SKIPPED, Pipeline
26
26
 
27
27
  logger = logging.getLogger(__name__)
28
28
 
@@ -31,8 +31,6 @@ EXIT_FAILURES = 1
31
31
  EXIT_USAGE = 2
32
32
  EXIT_INTERRUPTED = 130
33
33
 
34
- API_KEY_ENV_VARS = ("DATALAB_API_KEY", "MARKER_PDF_KEY")
35
-
36
34
  EPILOG = """\
37
35
  examples:
38
36
  pdf-to-md report.pdf convert to report.md (+ report_images/)
@@ -84,7 +82,7 @@ def build_parser() -> argparse.ArgumentParser:
84
82
  "-f",
85
83
  "--format",
86
84
  dest="output_format",
87
- choices=("markdown", "html", "json"),
85
+ choices=tuple(OUTPUT_EXTENSIONS),
88
86
  default="markdown",
89
87
  help="output format (default: markdown)",
90
88
  )
@@ -112,11 +110,17 @@ def build_parser() -> argparse.ArgumentParser:
112
110
  )
113
111
 
114
112
  conv = parser.add_argument_group("conversion")
113
+ conv.add_argument(
114
+ "--backend",
115
+ choices=tuple(BACKENDS),
116
+ default=DEFAULT_BACKEND,
117
+ help=f"conversion service (default: {DEFAULT_BACKEND})",
118
+ )
115
119
  conv.add_argument(
116
120
  "-m",
117
121
  "--mode",
118
- choices=MODES,
119
- help="quality/speed trade-off; the API defaults to fast",
122
+ choices=tuple(dict.fromkeys(mode for backend in BACKENDS.values() for mode in backend.info.modes)),
123
+ help="quality/speed trade-off; Datalab defaults to fast",
120
124
  )
121
125
  conv.add_argument("--paginate", action="store_true", help="insert page separators in the output")
122
126
  conv.add_argument(
@@ -189,10 +193,11 @@ def build_parser() -> argparse.ArgumentParser:
189
193
  return parser
190
194
 
191
195
 
192
- def resolve_api_key(explicit: str | None) -> str | None:
196
+ def resolve_api_key(explicit: str | None, env_vars: Sequence[str]) -> str | None:
197
+ """``--api-key`` if given, else the first non-empty variable in ``env_vars``."""
193
198
  if explicit:
194
199
  return explicit.strip()
195
- for name in API_KEY_ENV_VARS:
200
+ for name in env_vars:
196
201
  value = os.environ.get(name, "").strip()
197
202
  if value:
198
203
  return value
@@ -211,9 +216,11 @@ def config_from_args(args: argparse.Namespace, console: Console) -> Config:
211
216
  if used:
212
217
  console.warning(f"{flag} is no longer supported by the Datalab API and is ignored")
213
218
 
219
+ backend = get_backend(args.backend)
214
220
  return Config(
215
221
  inputs=list(args.inputs),
216
- api_key=resolve_api_key(args.api_key),
222
+ backend=backend.info.name,
223
+ api_key=resolve_api_key(args.api_key, backend.info.api_key_env_vars),
217
224
  output_dir=args.output_dir,
218
225
  output_format=args.output_format,
219
226
  mode=mode,
@@ -265,12 +272,18 @@ def main(argv: Sequence[str] | None = None, console: Console | None = None) -> i
265
272
  stop_event = threading.Event()
266
273
  try:
267
274
  config = config_from_args(args, console)
268
- jobs = plan_jobs(config.inputs, config.output_format, config.output_dir)
275
+ backend_type = get_backend(config.backend)
276
+ jobs = plan_jobs(
277
+ config.inputs,
278
+ config.output_format,
279
+ config.output_dir,
280
+ input_extensions=backend_type.info.input_extensions,
281
+ )
269
282
  if not jobs:
270
283
  console.error("No supported files to convert.")
271
284
  return EXIT_FAILURES
272
- client = None if config.dry_run else DatalabClient(config.api_key or "", stop_event=stop_event)
273
- pipeline = Pipeline(config, console, client=client, stop_event=stop_event)
285
+ backend = None if config.dry_run else backend_type.from_config(config, stop_event)
286
+ pipeline = Pipeline(config, console, backend=backend, stop_event=stop_event)
274
287
  summary = pipeline.run(jobs)
275
288
  except ConfigurationError as e:
276
289
  console.error(str(e))
@@ -5,8 +5,9 @@ from __future__ import annotations
5
5
  from dataclasses import dataclass, field
6
6
  from pathlib import Path
7
7
 
8
+ from docs_to_md.assemble import OUTPUT_EXTENSIONS
9
+ from docs_to_md.backends import DEFAULT_BACKEND, get_backend
8
10
  from docs_to_md.errors import ConfigurationError
9
- from docs_to_md.models import MODES, SUPPORTED_FORMAT_EXTENSIONS, ConvertOptions
10
11
  from docs_to_md.pdf import validate_page_range
11
12
 
12
13
  DEFAULT_CHUNK_SIZE = 25
@@ -17,6 +18,7 @@ DEFAULT_TIMEOUT_SECONDS = 3600.0
17
18
  @dataclass
18
19
  class Config:
19
20
  inputs: list[Path]
21
+ backend: str = DEFAULT_BACKEND
20
22
  api_key: str | None = None
21
23
  output_dir: Path | None = None
22
24
  output_format: str = "markdown"
@@ -33,20 +35,24 @@ class Config:
33
35
  overwrite: bool = False
34
36
  dry_run: bool = False
35
37
  reflow_markdown: bool = True
36
- extra_options: dict = field(default_factory=dict)
38
+ extra_options: dict = field(default_factory=dict) # passed through to the backend
37
39
 
38
40
  def validate(self) -> Config:
41
+ info = get_backend(self.backend).info
39
42
  if not self.inputs:
40
43
  raise ConfigurationError("At least one input path is required")
41
- if not self.dry_run and not self.api_key:
42
- raise ConfigurationError(
43
- "No API key found. Set DATALAB_API_KEY (or MARKER_PDF_KEY), or pass "
44
- "--api-key. Get a key at https://www.datalab.to/app/keys"
45
- )
46
- if self.output_format not in SUPPORTED_FORMAT_EXTENSIONS:
44
+ if not self.dry_run and info.requires_api_key and not self.api_key:
45
+ primary, *others = info.api_key_env_vars
46
+ also = f" (or {', '.join(others)})" if others else ""
47
+ where = f" Get a key at {info.api_key_url}" if info.api_key_url else ""
48
+ raise ConfigurationError(f"No API key found. Set {primary}{also}, or pass --api-key.{where}")
49
+ if self.output_format not in OUTPUT_EXTENSIONS:
47
50
  raise ConfigurationError(f"Unsupported output format: {self.output_format}")
48
- if self.mode is not None and self.mode not in MODES:
49
- raise ConfigurationError(f"Unsupported mode: {self.mode}")
51
+ if self.output_format not in info.output_formats:
52
+ raise ConfigurationError(f"The {info.label} backend cannot produce {self.output_format} output")
53
+ if self.mode is not None and self.mode not in info.modes:
54
+ choices = f" (choose from {', '.join(info.modes)})" if info.modes else ""
55
+ raise ConfigurationError(f"Unsupported mode for {info.label}: {self.mode}{choices}")
50
56
  if self.chunk_size is not None and self.chunk_size < 1:
51
57
  raise ConfigurationError("--chunk-size must be at least 1")
52
58
  if self.max_pages is not None and self.max_pages < 1:
@@ -62,20 +68,3 @@ class Config:
62
68
  if self.output_dir.exists() and not self.output_dir.is_dir():
63
69
  raise ConfigurationError(f"Output path is not a directory: {self.output_dir}")
64
70
  return self
65
-
66
- def convert_options(self, *, local_page_selection: bool) -> ConvertOptions:
67
- """API options for one request.
68
-
69
- PDFs are split locally, so their page selection is not sent to the API.
70
- """
71
- return ConvertOptions(
72
- output_format=self.output_format,
73
- mode=self.mode,
74
- paginate=self.paginate,
75
- disable_image_extraction=self.disable_image_extraction,
76
- disable_image_captions=self.disable_image_captions,
77
- skip_cache=self.skip_cache,
78
- page_range=None if local_page_selection else self.page_range,
79
- max_pages=None if local_page_selection else self.max_pages,
80
- extra=dict(self.extra_options),
81
- )
@@ -15,6 +15,16 @@ from typing import TextIO
15
15
  from tqdm import tqdm
16
16
 
17
17
 
18
+ def format_cost(cents: float) -> str:
19
+ dollars = cents / 100
20
+ return f"${dollars:.2f}" if dollars >= 0.01 or dollars == 0 else f"${dollars:.4f}"
21
+
22
+
23
+ def format_elapsed(seconds: float) -> str:
24
+ minutes, secs = divmod(round(seconds), 60)
25
+ return f"{minutes}m{secs:02d}s" if minutes else f"{seconds:.1f}s"
26
+
27
+
18
28
  def setup_logging(verbose: bool) -> None:
19
29
  handler = logging.StreamHandler(sys.stderr)
20
30
  if verbose: