pdf-to-markdown-cli 0.4.0__tar.gz → 0.5.0__tar.gz

{pdf_to_markdown_cli-0.4.0/src/pdf_to_markdown_cli.egg-info → pdf_to_markdown_cli-0.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pdf-to-markdown-cli
-Version: 0.4.0
+Version: 0.5.0
 Summary: CLI tool to convert PDF files (and other documents) to markdown using the Marker API.
 Author: Nikita Sokolsky
 License: MIT License
@@ -50,6 +50,7 @@ Requires-Dist: pydantic>=2.0
 Requires-Dist: ratelimit>=2.0
 Requires-Dist: requests>=2.0
 Requires-Dist: tqdm>=4.0
+Provides-Extra: test
 Dynamic: license-file
 
 # PDF to Markdown CLI (via the Datalab Marker API)
@@ -107,6 +108,9 @@ pdf-to-md /path/to/file.pdf --langs "English,French,German"
 
 # Use all Marker OCR enhancements
 pdf-to-md /path/to/file.pdf --max
+
+# Display all available options
+pdf-to-md --help
 ```
 
 ### Full list of CLI options
@@ -125,6 +129,7 @@ pdf-to-md /path/to/file.pdf --max
 - `-cs`, `--chunk-size`: Set PDF chunk size in pages (default: 25)
 - `-o`, `--output-dir`: Absolute path to the output directory. If not provided, output files will be saved in the same directory as their corresponding input files.
 - `-v`, `--verbose`: Enable verbose (DEBUG level) logging
+- `--version`: Show the installed `pdf-to-markdown-cli` version and exit
 
 ### Output Structure
 
@@ -134,6 +139,17 @@ If an output file with the same name already exists, the new file will be automa
 
 If you specify `--output-dir /path/to/output`, the output file will be placed in that directory (e.g., `/path/to/output/report.md`). The same automatic renaming logic applies if a file with the same name exists in the target directory.
 
+### Core Workflow
+
+`MarkerProcessor` drives the conversion process:
+
+1. Discover files with `FileDiscovery`.
+2. Determine unique output paths for each file and chunk.
+3. Submit jobs to the Marker API via `BatchProcessor`.
+4. Poll for results and combine chunk outputs while moving extracted images.
+
+Settings are validated in `Config.validate()` and request data is cached with `CacheManager` so interrupted runs can resume.
+
 ### API
 
 ```python
@@ -172,6 +188,8 @@ pdf-to-markdown-cli/ (Project Root)
 └── ... (other config files)
 ```
 
+`pyproject.toml` defines `pdf-to-md` as a console script pointing to `docs_to_md.main:main`, so installing the project exposes the `pdf-to-md` command.
+
 ## Requirements
 
 - Python 3.10 or higher
@@ -188,11 +206,18 @@ To run the code directly from the source tree without installation:
 # pdf-to-md /path/to/file.pdf
 
 # Or running the module directly:
-python -m docs_to_md /path/to/file.pdf 
+python -m docs_to_md /path/to/file.pdf
 ```
 
 For regular use after installation, use the `pdf-to-md` command.
 
+## Getting Started for Contributors
+
+- Run `pdf-to-md --help` to explore all CLI flags. The `examples/` directory contains sample documents for testing.
+- Follow `MarkerProcessor.process()` in `docs_to_md/core/processor.py` to see how jobs are prepared and results combined.
+- Consult `datalab_marker_api_docs.md` for full details on authentication and Marker API parameters.
+- Utilities in `docs_to_md/utils` and the caching layer in `docs_to_md/storage` are useful starting points for deeper customization.
+
 ## License
 
 This project is licensed under the MIT License - see the LICENSE file for details.

{pdf_to_markdown_cli-0.4.0 → pdf_to_markdown_cli-0.5.0}/README.md

@@ -53,6 +53,9 @@ pdf-to-md /path/to/file.pdf --langs "English,French,German"
 
 # Use all Marker OCR enhancements
 pdf-to-md /path/to/file.pdf --max
+
+# Display all available options
+pdf-to-md --help
 ```
 
 ### Full list of CLI options
@@ -71,6 +74,7 @@ pdf-to-md /path/to/file.pdf --max
 - `-cs`, `--chunk-size`: Set PDF chunk size in pages (default: 25)
 - `-o`, `--output-dir`: Absolute path to the output directory. If not provided, output files will be saved in the same directory as their corresponding input files.
 - `-v`, `--verbose`: Enable verbose (DEBUG level) logging
+- `--version`: Show the installed `pdf-to-markdown-cli` version and exit
 
 ### Output Structure
 
@@ -80,6 +84,17 @@ If an output file with the same name already exists, the new file will be automa
 
 If you specify `--output-dir /path/to/output`, the output file will be placed in that directory (e.g., `/path/to/output/report.md`). The same automatic renaming logic applies if a file with the same name exists in the target directory.
 
+### Core Workflow
+
+`MarkerProcessor` drives the conversion process:
+
+1. Discover files with `FileDiscovery`.
+2. Determine unique output paths for each file and chunk.
+3. Submit jobs to the Marker API via `BatchProcessor`.
+4. Poll for results and combine chunk outputs while moving extracted images.
+
+Settings are validated in `Config.validate()` and request data is cached with `CacheManager` so interrupted runs can resume.
+
 ### API
 
 ```python
@@ -118,6 +133,8 @@ pdf-to-markdown-cli/ (Project Root)
 └── ... (other config files)
 ```
 
+`pyproject.toml` defines `pdf-to-md` as a console script pointing to `docs_to_md.main:main`, so installing the project exposes the `pdf-to-md` command.
+
 ## Requirements
 
 - Python 3.10 or higher
@@ -134,11 +151,18 @@ To run the code directly from the source tree without installation:
 # pdf-to-md /path/to/file.pdf
 
 # Or running the module directly:
-python -m docs_to_md /path/to/file.pdf 
+python -m docs_to_md /path/to/file.pdf
 ```
 
 For regular use after installation, use the `pdf-to-md` command.
 
+## Getting Started for Contributors
+
+- Run `pdf-to-md --help` to explore all CLI flags. The `examples/` directory contains sample documents for testing.
+- Follow `MarkerProcessor.process()` in `docs_to_md/core/processor.py` to see how jobs are prepared and results combined.
+- Consult `datalab_marker_api_docs.md` for full details on authentication and Marker API parameters.
+- Utilities in `docs_to_md/utils` and the caching layer in `docs_to_md/storage` are useful starting points for deeper customization.
+
 ## License
 
 This project is licensed under the MIT License - see the LICENSE file for details.

{pdf_to_markdown_cli-0.4.0 → pdf_to_markdown_cli-0.5.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "pdf-to-markdown-cli"
-version = "0.4.0"
+version = "0.5.0"
 description = "CLI tool to convert PDF files (and other documents) to markdown using the Marker API."
 readme = "README.md"
 authors = [
@@ -48,4 +48,7 @@ pdf-to-md = "docs_to_md.main:main"
 
 # Just in case: Configure setuptools to find packages in src/
 [tool.setuptools.packages.find]
-where = ["src"]
+where = ["src"]
+
+[project.optional-dependencies]
+test = []

{pdf_to_markdown_cli-0.4.0 → pdf_to_markdown_cli-0.5.0}/src/docs_to_md/core/processor.py

@@ -53,6 +53,75 @@ class BatchProcessor:
     def should_chunk(self, file_path: Path) -> bool:
         return file_path.suffix.lower() == ".pdf"
 
+    def _chunk_file(
+        self, file_path: Path, tmp_dir: Path, request: ConversionRequest
+    ) -> bool:
+        """Chunk the provided PDF and populate the request object.
+
+        Returns ``True`` if chunking fails and the request should be marked
+        as failed.
+        """
+        try:
+            chunk_result = chunk_pdf_to_temp(str(file_path), self.chunk_size, tmp_dir)
+            if chunk_result:
+                for chunk_info in chunk_result.chunks:
+                    request.add_chunk(Path(chunk_info.path), chunk_info.index)
+                logger.debug(
+                    f"Created {len(request.chunks)} chunks in {tmp_dir}"
+                )
+            else:
+                logger.debug(
+                    f"No chunking needed for {file_path} (<= {self.chunk_size} pages)"
+                )
+            return False
+        except (PDFProcessingError, Exception) as e:
+            logger.error(f"Error chunking PDF {file_path}: {e}", exc_info=True)
+            request.set_status(Status.FAILED, f"Error chunking PDF: {e}")
+            self.cache.save(request)
+            return True
+
+    def _submit_chunks(
+        self, request: ConversionRequest, api_params: ApiParams
+    ) -> bool:
+        """Submit all chunks to the API.
+
+        Returns ``True`` if any submission fails.
+        """
+        submission_failed = False
+        with ProgressTracker(len(request.chunks), "Submitting to API", "chunk") as progress:
+            for chunk in request.ordered_chunks:
+                try:
+                    chunk_request_id = self.client.submit_file(
+                        chunk.path,
+                        output_format=api_params.output_format,
+                        langs=api_params.langs,
+                        use_llm=api_params.use_llm,
+                        strip_existing_ocr=api_params.strip_existing_ocr,
+                        disable_image_extraction=api_params.disable_image_extraction,
+                        force_ocr=api_params.force_ocr,
+                        paginate=api_params.paginate,
+                        max_pages=api_params.max_pages,
+                    )
+                    if chunk_request_id:
+                        chunk.mark_processing(chunk_request_id)
+                    else:
+                        chunk.mark_failed(
+                            f"API submission failed for {chunk.path.name}"
+                        )
+                        submission_failed = True
+                        break
+                except Exception as submit_e:
+                    logger.error(
+                        f"Unexpected error submitting chunk {chunk.path.name}: {submit_e}",
+                        exc_info=True,
+                    )
+                    chunk.mark_failed(f"Error submitting chunk: {submit_e}")
+                    submission_failed = True
+                    break
+                finally:
+                    progress.update()
+        return submission_failed or request.has_failed
+
     def process_file(
         self,
         file_path: Path,
@@ -60,20 +129,11 @@ class BatchProcessor:
         api_params: ApiParams,
         output_paths_obj: OutputPaths,
     ) -> Optional[str]:
-        """
-        Process a single file: chunk if needed, submit to API.
-
-        Args:
-            file_path: Path to the input file.
-            final_output_path: Path where the final output markdown file should be saved.
-            api_params: Parameters for the Marker API call.
-            output_paths_obj: The OutputPaths object containing final markdown and image paths.
+        """Process a single file and submit it to the Marker API.
 
-        Returns:
-            Request ID for tracking if submission is initiated, otherwise None.
-
-        Raises:
-            FileError: Potentially raised by underlying operations (though many are caught).
+        The method now delegates chunking and submission to smaller helpers to
+        keep the logic readable.  It returns the created request ID, which can
+        later be used to poll for results.
         """
         with TemporaryDirectory(self.root_tmp_dir, file_path.stem) as tmp_dir:
             request = ConversionRequest(
@@ -93,30 +153,7 @@ class BatchProcessor:
 
             try:
                 if self.should_chunk(file_path):
-                    try:
-                        chunk_result = chunk_pdf_to_temp(
-                            str(file_path), self.chunk_size, tmp_dir
-                        )
-                        if chunk_result:
-                            for chunk_info in chunk_result.chunks:
-                                request.add_chunk(
-                                    Path(chunk_info.path), chunk_info.index
-                                )
-                            logger.debug(
-                                f"Created {len(request.chunks)} chunks in {tmp_dir}"
-                            )
-                        else:
-                            logger.debug(
-                                f"No chunking needed for {file_path} (<= {self.chunk_size} pages)"
-                            )
-                    except (PDFProcessingError, Exception) as e:
-                        logger.error(
-                            f"Error chunking PDF {file_path}: {e}", exc_info=True
-                        )
-                        request.set_status(
-                            Status.FAILED, f"Error chunking PDF: {str(e)}"
-                        )
-                        self.cache.save(request)
+                    if self._chunk_file(file_path, tmp_dir, request):
                         return request.request_id
 
                 if not request.chunks:
@@ -126,43 +163,9 @@ class BatchProcessor:
                     f"Submitting {len(request.chunks)} chunk(s) to API for {request.original_file.name}..."
                 )
 
-                submission_failed = False
-                with ProgressTracker(
-                    len(request.chunks), "Submitting to API", "chunk"
-                ) as progress:
-                    for chunk in request.ordered_chunks:
-                        try:
-                            chunk_request_id = self.client.submit_file(
-                                chunk.path,
-                                output_format=api_params.output_format,
-                                langs=api_params.langs,
-                                use_llm=api_params.use_llm,
-                                strip_existing_ocr=api_params.strip_existing_ocr,
-                                disable_image_extraction=api_params.disable_image_extraction,
-                                force_ocr=api_params.force_ocr,
-                                paginate=api_params.paginate,
-                                max_pages=api_params.max_pages,
-                            )
-                            if chunk_request_id:
-                                chunk.mark_processing(chunk_request_id)
-                            else:
-                                chunk.mark_failed(
-                                    f"API submission failed for {chunk.path.name}"
-                                )
-                                submission_failed = True
-                                break
-                        except Exception as submit_e:
-                            logger.error(
-                                f"Unexpected error submitting chunk {chunk.path.name}: {submit_e}",
-                                exc_info=True,
-                            )
-                            chunk.mark_failed(f"Error submitting chunk: {submit_e}")
-                            submission_failed = True
-                            break
-                        finally:
-                            progress.update()
-
-                if submission_failed or request.has_failed:
+                submission_failed = self._submit_chunks(request, api_params)
+
+                if submission_failed:
                     request.set_status(
                         Status.FAILED,
                         request.error or "One or more chunk submissions failed.",

{pdf_to_markdown_cli-0.4.0 → pdf_to_markdown_cli-0.5.0}/src/docs_to_md/storage/models.py

@@ -48,7 +48,8 @@ class ConversionRequest(BaseModel):
     output_format: str = "markdown"
     status: Status = Status.PENDING
     error: Optional[str] = None
-    chunks: List[ChunkInfo] = []
+    # Use default_factory to avoid shared mutable list across instances
+    chunks: List[ChunkInfo] = Field(default_factory=list)
     chunk_size: int
     tmp_dir: Optional[Path] = None  # Directory for temporary files for this conversion
     images_dir: Optional[Path] = None  # Added to store determined image path

{pdf_to_markdown_cli-0.4.0 → pdf_to_markdown_cli-0.5.0/src/pdf_to_markdown_cli.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pdf-to-markdown-cli
-Version: 0.4.0
+Version: 0.5.0
 Summary: CLI tool to convert PDF files (and other documents) to markdown using the Marker API.
 Author: Nikita Sokolsky
 License: MIT License
@@ -50,6 +50,7 @@ Requires-Dist: pydantic>=2.0
 Requires-Dist: ratelimit>=2.0
 Requires-Dist: requests>=2.0
 Requires-Dist: tqdm>=4.0
+Provides-Extra: test
 Dynamic: license-file
 
 # PDF to Markdown CLI (via the Datalab Marker API)
@@ -107,6 +108,9 @@ pdf-to-md /path/to/file.pdf --langs "English,French,German"
 
 # Use all Marker OCR enhancements
 pdf-to-md /path/to/file.pdf --max
+
+# Display all available options
+pdf-to-md --help
 ```
 
 ### Full list of CLI options
@@ -125,6 +129,7 @@ pdf-to-md /path/to/file.pdf --max
 - `-cs`, `--chunk-size`: Set PDF chunk size in pages (default: 25)
 - `-o`, `--output-dir`: Absolute path to the output directory. If not provided, output files will be saved in the same directory as their corresponding input files.
 - `-v`, `--verbose`: Enable verbose (DEBUG level) logging
+- `--version`: Show the installed `pdf-to-markdown-cli` version and exit
 
 ### Output Structure
 
@@ -134,6 +139,17 @@ If an output file with the same name already exists, the new file will be automa
 
 If you specify `--output-dir /path/to/output`, the output file will be placed in that directory (e.g., `/path/to/output/report.md`). The same automatic renaming logic applies if a file with the same name exists in the target directory.
 
+### Core Workflow
+
+`MarkerProcessor` drives the conversion process:
+
+1. Discover files with `FileDiscovery`.
+2. Determine unique output paths for each file and chunk.
+3. Submit jobs to the Marker API via `BatchProcessor`.
+4. Poll for results and combine chunk outputs while moving extracted images.
+
+Settings are validated in `Config.validate()` and request data is cached with `CacheManager` so interrupted runs can resume.
+
 ### API
 
 ```python
@@ -172,6 +188,8 @@ pdf-to-markdown-cli/ (Project Root)
 └── ... (other config files)
 ```
 
+`pyproject.toml` defines `pdf-to-md` as a console script pointing to `docs_to_md.main:main`, so installing the project exposes the `pdf-to-md` command.
+
 ## Requirements
 
 - Python 3.10 or higher
@@ -188,11 +206,18 @@ To run the code directly from the source tree without installation:
 # pdf-to-md /path/to/file.pdf
 
 # Or running the module directly:
-python -m docs_to_md /path/to/file.pdf 
+python -m docs_to_md /path/to/file.pdf
 ```
 
 For regular use after installation, use the `pdf-to-md` command.
 
+## Getting Started for Contributors
+
+- Run `pdf-to-md --help` to explore all CLI flags. The `examples/` directory contains sample documents for testing.
+- Follow `MarkerProcessor.process()` in `docs_to_md/core/processor.py` to see how jobs are prepared and results combined.
+- Consult `datalab_marker_api_docs.md` for full details on authentication and Marker API parameters.
+- Utilities in `docs_to_md/utils` and the caching layer in `docs_to_md/storage` are useful starting points for deeper customization.
+
 ## License
 
 This project is licensed under the MIT License - see the LICENSE file for details.

{pdf_to_markdown_cli-0.4.0 → pdf_to_markdown_cli-0.5.0}/src/pdf_to_markdown_cli.egg-info/SOURCES.txt

@@ -28,4 +28,8 @@ src/pdf_to_markdown_cli.egg-info/SOURCES.txt
 src/pdf_to_markdown_cli.egg-info/dependency_links.txt
 src/pdf_to_markdown_cli.egg-info/entry_points.txt
 src/pdf_to_markdown_cli.egg-info/requires.txt
-src/pdf_to_markdown_cli.egg-info/top_level.txt
+src/pdf_to_markdown_cli.egg-info/top_level.txt
+tests/test_cli.py
+tests/test_paths.py
+tests/test_settings.py
+tests/test_utils.py

{pdf_to_markdown_cli-0.4.0 → pdf_to_markdown_cli-0.5.0}/src/pdf_to_markdown_cli.egg-info/requires.txt

@@ -6,3 +6,5 @@ pydantic>=2.0
 ratelimit>=2.0
 requests>=2.0
 tqdm>=4.0
+
+[test]

pdf_to_markdown_cli-0.5.0/tests/test_cli.py

@@ -0,0 +1,37 @@
+import os
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+from unittest import mock
+
+from docs_to_md.config.cli import create_config_from_args
+
+
+class TestCLI(unittest.TestCase):
+    def test_create_config_from_args(self):
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            input_file = Path(tmp_dir) / "input.txt"
+            input_file.write_text("data")
+            env = {"MARKER_PDF_KEY": "abc"}
+            argv = [
+                "prog",
+                str(input_file),
+                "-cs",
+                "5",
+                "--max",
+                "-o",
+                tmp_dir,
+            ]
+            with mock.patch.dict(os.environ, env, clear=False):
+                with mock.patch.object(sys, "argv", argv):
+                    config = create_config_from_args()
+            self.assertEqual(config.input_path, str(input_file))
+            self.assertEqual(config.chunk_size, 5)
+            self.assertTrue(config.use_llm)
+            self.assertTrue(config.force_ocr)
+            self.assertEqual(config.output_dir, Path(tmp_dir))
+
+
+if __name__ == "__main__":
+    unittest.main()

pdf_to_markdown_cli-0.5.0/tests/test_paths.py

@@ -0,0 +1,25 @@
+import os
+import tempfile
+import unittest
+from pathlib import Path
+
+from docs_to_md.core.paths import determine_output_paths
+
+
+class TestPaths(unittest.TestCase):
+    def test_determine_output_paths_creates_base_dir(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            tmp_path = Path(tmp)
+            input_file = tmp_path / "input.txt"
+            input_file.write_text("data")
+            out_dir = tmp_path / "out"
+            result = determine_output_paths(input_file, out_dir, "markdown")
+            self.assertEqual(result.markdown_path.parent, out_dir)
+            self.assertEqual(result.images_dir.parent, out_dir)
+            self.assertTrue(result.unique_key)
+            self.assertEqual(result.markdown_path.suffix, ".md")
+            self.assertTrue(out_dir.exists())
+
+
+if __name__ == "__main__":
+    unittest.main()

pdf_to_markdown_cli-0.5.0/tests/test_settings.py

@@ -0,0 +1,65 @@
+import tempfile
+import unittest
+from pathlib import Path
+
+from docs_to_md.config.settings import Config
+from docs_to_md.utils.exceptions import ConfigurationError
+
+
+class TestSettings(unittest.TestCase):
+    def test_config_validation_success(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            tmp_path = Path(tmp)
+            test_file = tmp_path / "input.txt"
+            test_file.write_text("data")
+            cfg = Config(
+                api_key="key",
+                input_path=str(test_file),
+                output_dir=tmp_path,
+                output_format="markdown",
+            )
+            cfg.validate()
+
+    def test_config_missing_api_key(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            tmp_path = Path(tmp)
+            test_file = tmp_path / "in.txt"
+            test_file.write_text("x")
+            cfg = Config(
+                api_key="",
+                input_path=str(test_file),
+                output_dir=tmp_path,
+                output_format="markdown",
+            )
+            with self.assertRaises(ConfigurationError):
+                cfg.validate()
+
+    def test_config_nonexistent_input(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            tmp_path = Path(tmp)
+            cfg = Config(
+                api_key="key",
+                input_path=str(tmp_path / "no.txt"),
+                output_dir=tmp_path,
+                output_format="markdown",
+            )
+            with self.assertRaises(ConfigurationError):
+                cfg.validate()
+
+    def test_config_relative_output_dir(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            tmp_path = Path(tmp)
+            test_file = tmp_path / "file.txt"
+            test_file.write_text("x")
+            cfg = Config(
+                api_key="key",
+                input_path=str(test_file),
+                output_dir=Path("relative"),
+                output_format="markdown",
+            )
+            with self.assertRaises(ConfigurationError):
+                cfg.validate()
+
+
+if __name__ == "__main__":
+    unittest.main()

pdf_to_markdown_cli-0.5.0/tests/test_utils.py

@@ -0,0 +1,22 @@
+import tempfile
+import unittest
+from pathlib import Path
+
+from docs_to_md.utils.file_utils import get_unique_filename
+
+
+class TestFileUtils(unittest.TestCase):
+    def test_get_unique_filename(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            tmp_path = Path(tmp)
+            file_path = tmp_path / "file.txt"
+            file_path.write_text("data")
+            new_path = get_unique_filename(file_path)
+            self.assertNotEqual(new_path, file_path)
+            self.assertEqual(new_path.parent, file_path.parent)
+            self.assertTrue(new_path.stem.startswith("file_"))
+            self.assertEqual(new_path.suffix, ".txt")
+
+
+if __name__ == "__main__":
+    unittest.main()