grobid-client-python 0.2.1__tar.gz → 0.2.2__tar.gz

{grobid_client_python-0.2.1 → grobid_client_python-0.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: grobid-client-python
-Version: 0.2.1
+Version: 0.2.2
 Summary: Simple python client for GROBID REST services
 Author-email: Patrice Lopez <patrice.lopez@science-miner.com>
 Maintainer-email: Patrice Lopez <patrice.lopez@science-miner.com>, Luca Foppiano <lucanoro@duck.com>
@@ -411,6 +411,9 @@ grobid_client --input "~/data/**/*.pdf"   --output ~/results processFulltextDocu
 
 > [!NOTE]
 > `--input` accepts a directory, a single file, an **archive**, or a **glob pattern**:
+> - **Directories** are processed recursively and the output mirrors the input tree: `~/pdfs/2023/a.pdf` →
+>   `~/results/2023/a.grobid.tei.xml` (subdirectories of `--output` are created as needed, and the JSON/Markdown
+>   outputs and error files sit next to each TEI file). Archives keep their internal paths the same way.
 > - **Archives** (`.zip`, `.tar`, `.tar.gz`/`.tgz`, `.tar.bz2`/`.tbz2`) are streamed: eligible entries are read into
 >   memory in chunks of `queue_size` and sent to GROBID straight from there — the archive is never fully decompressed and
 >   nothing but the results in `--output` ever touches the disk. If `--output` is omitted, results go to a directory named

{grobid_client_python-0.2.1 → grobid_client_python-0.2.2}/Readme.md

@@ -189,6 +189,9 @@ grobid_client --input "~/data/**/*.pdf"   --output ~/results processFulltextDocu
 
 > [!NOTE]
 > `--input` accepts a directory, a single file, an **archive**, or a **glob pattern**:
+> - **Directories** are processed recursively and the output mirrors the input tree: `~/pdfs/2023/a.pdf` →
+>   `~/results/2023/a.grobid.tei.xml` (subdirectories of `--output` are created as needed, and the JSON/Markdown
+>   outputs and error files sit next to each TEI file). Archives keep their internal paths the same way.
 > - **Archives** (`.zip`, `.tar`, `.tar.gz`/`.tgz`, `.tar.bz2`/`.tbz2`) are streamed: eligible entries are read into
 >   memory in chunks of `queue_size` and sent to GROBID straight from there — the archive is never fully decompressed and
 >   nothing but the results in `--output` ever touches the disk. If `--output` is omitted, results go to a directory named

{grobid_client_python-0.2.1 → grobid_client_python-0.2.2}/grobid_client/grobid_client.py

@@ -34,7 +34,7 @@ import shutil
 import tarfile
 import tempfile
 import zipfile
-from typing import Any, BinaryIO, Callable, Optional, Tuple, Union
+from typing import Any, BinaryIO, Callable, List, Optional, Tuple, Union
 import copy
 
 from .format.TEI2LossyJSON import TEI2LossyJSONConverter
@@ -449,10 +449,15 @@ class GrobidClient(ApiClient):
         input_file_path = pathlib.Path(input_file)
 
         if output is not None:
-            # Calculate relative path from input_path, then join with output directory
+            # Mirror the input tree under the output directory: the path of the
+            # file relative to input_path is kept (only the extension changes),
+            # so sub/paper.pdf -> <output>/sub/paper.grobid.tei.xml. Keeping the
+            # subdirectories also prevents two inputs that share a basename
+            # from mapping to the same output file.
             input_path_abs = pathlib.Path(input_path).resolve()
             input_file_rel = input_file_path.resolve().relative_to(input_path_abs)
-            filename = pathlib.Path(output) / f"{input_file_rel.stem}.grobid.tei.xml"
+            filename = pathlib.Path(output) / input_file_rel.with_name(
+                f"{input_file_rel.stem}.grobid.tei.xml")
         else:
             # Use the same directory as the input file
             filename = input_file_path.parent / f"{input_file_path.stem}.grobid.tei.xml"
@@ -629,9 +634,14 @@ class GrobidClient(ApiClient):
 
         # Partition into archives (streamed), remote loose files (s3) and local
         # filesystem files (directories are expanded to their eligible files).
+        # Local files are grouped with the base directory their output paths
+        # are computed from: a directory given as input is the base of the
+        # files found inside it (so the output mirrors its tree), while loose
+        # files (single paths, glob matches) share a common ancestor.
         archive_paths = []
         remote_files = []
-        fs_files = []
+        fs_groups: List[Tuple[str, list]] = []
+        loose_files = []
         for path in matched_paths:
             if self._is_s3(path):
                 if self._looks_like_archive(path):
@@ -643,14 +653,20 @@ class GrobidClient(ApiClient):
             elif self._is_archive(path):
                 archive_paths.append(path)
             elif os.path.isdir(path):
-                fs_files.extend(self._collect_directory_files(path, service))
+                dir_files = self._collect_directory_files(path, service)
+                if dir_files:
+                    fs_groups.append((path, dir_files))
             elif os.path.isfile(path) and self._is_eligible_input(os.path.basename(path), service):
-                fs_files.append(path)
+                loose_files.append(path)
             elif not os.path.exists(path):
                 self.logger.warning(f"Input path does not exist: {path}")
             else:
                 self.logger.debug(f"Skipping input (not an eligible file/dir/archive): {path}")
 
+        if loose_files:
+            fs_groups.append((self._common_base(loose_files), loose_files))
+        fs_files = [f for _, files in fs_groups for f in files]
+
         if not fs_files and not archive_paths and not remote_files:
             self.logger.warning(f"No eligible files found in input(s): {inputs}")
             return
@@ -669,16 +685,17 @@ class GrobidClient(ApiClient):
         # Local files gathered from directories and/or loose glob matches
         if fs_files:
             print(f"Found {len(fs_files)} local file(s) to process")
-            bp, be, bs = self._run_file_batches(
-                service, fs_files, self._common_base(fs_files), output, n,
-                generate_ids, consolidate_header, consolidate_citations,
-                include_raw_citations, include_raw_affiliations, tei_coordinates,
-                segment_sentences, force, verbose, flavor, json_output, markdown_output,
-                skip_errors=skip_errors
-            )
-            processed_files_count += bp
-            errors_files_count += be
-            skipped_files_count += bs
+            for base_dir, group_files in fs_groups:
+                bp, be, bs = self._run_file_batches(
+                    service, group_files, base_dir, output, n,
+                    generate_ids, consolidate_header, consolidate_citations,
+                    include_raw_citations, include_raw_affiliations, tei_coordinates,
+                    segment_sentences, force, verbose, flavor, json_output, markdown_output,
+                    skip_errors=skip_errors
+                )
+                processed_files_count += bp
+                errors_files_count += be
+                skipped_files_count += bs
             total_files += len(fs_files)
 
         # Loose remote (s3) files: streamed into memory one chunk at a time
@@ -1096,8 +1113,9 @@ class GrobidClient(ApiClient):
         the chunk size and nothing but the results ever touches the disk. (The
         exception is ``processCitationList``, whose ``.txt`` inputs are read
         from a path and therefore still go through a temporary directory.)
-        Output files follow the same flat naming convention as directory
-        processing (one ``<stem>`` per result, in ``output``).
+        Output files follow the same naming convention as directory
+        processing: the entry's path inside the archive is mirrored under
+        ``output`` (``nested/y.pdf`` -> ``<output>/nested/y.grobid.tei.xml``).
         """
         start_time = time.time()
         self._warn_on_consolidation_timeout(consolidate_citations)

{grobid_client_python-0.2.1 → grobid_client_python-0.2.2}/grobid_client_python.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: grobid-client-python
-Version: 0.2.1
+Version: 0.2.2
 Summary: Simple python client for GROBID REST services
 Author-email: Patrice Lopez <patrice.lopez@science-miner.com>
 Maintainer-email: Patrice Lopez <patrice.lopez@science-miner.com>, Luca Foppiano <lucanoro@duck.com>
@@ -411,6 +411,9 @@ grobid_client --input "~/data/**/*.pdf"   --output ~/results processFulltextDocu
 
 > [!NOTE]
 > `--input` accepts a directory, a single file, an **archive**, or a **glob pattern**:
+> - **Directories** are processed recursively and the output mirrors the input tree: `~/pdfs/2023/a.pdf` →
+>   `~/results/2023/a.grobid.tei.xml` (subdirectories of `--output` are created as needed, and the JSON/Markdown
+>   outputs and error files sit next to each TEI file). Archives keep their internal paths the same way.
 > - **Archives** (`.zip`, `.tar`, `.tar.gz`/`.tgz`, `.tar.bz2`/`.tbz2`) are streamed: eligible entries are read into
 >   memory in chunks of `queue_size` and sent to GROBID straight from there — the archive is never fully decompressed and
 >   nothing but the results in `--output` ever touches the disk. If `--output` is omitted, results go to a directory named

grobid_client_python-0.2.2/grobid_client_python.egg-info/scm_version.json

@@ -0,0 +1,8 @@
+{
+  "tag": "0.2.2",
+  "distance": 0,
+  "node": "g53114ab6c37ab5421b22efbeceeddd171cbff8f3",
+  "dirty": false,
+  "branch": "HEAD",
+  "node_date": "2026-11-06"
+}

{grobid_client_python-0.2.1 → grobid_client_python-0.2.2}/pyproject.toml

@@ -3,7 +3,7 @@ requires = ["setuptools", "setuptools-scm"]
 build-backend = "setuptools.build_meta"
 
 [tool.bumpversion]
-current_version = "0.2.1"
+current_version = "0.2.2"
 commit = "true"
 tag = "true"
 tag_name = "v{new_version}"

{grobid_client_python-0.2.1 → grobid_client_python-0.2.2}/tests/test_grobid_client.py

@@ -226,6 +226,17 @@ class TestGrobidClient:
 
         assert result == expected
 
+    def test_output_file_name_preserves_subdirectories(self):
+        """The path relative to the input directory is mirrored under the output."""
+        with patch('grobid_client.grobid_client.GrobidClient._test_server_connection'):
+            with patch('grobid_client.grobid_client.GrobidClient._configure_logging'):
+                client = GrobidClient(check_server=False)
+
+        result = client._output_file_name('/input/path/sub/deeper/document.pdf', '/input/path', '/output/path')
+        expected = os.path.join('/output/path', 'sub', 'deeper', 'document.grobid.tei.xml')
+
+        assert result == expected
+
     def test_output_file_name_without_output_path(self):
         """Test _output_file_name method without output path."""
         with patch('grobid_client.grobid_client.GrobidClient._test_server_connection'):
@@ -753,11 +764,13 @@ class TestArchiveInput:
 
     @staticmethod
     def _tei_outputs(output_dir):
+        """TEI outputs under output_dir, as paths relative to it (posix separators)."""
         found = []
         for root, _, files in os.walk(output_dir):
             for f in files:
                 if f.endswith('.grobid.tei.xml'):
-                    found.append(f)
+                    rel = os.path.relpath(os.path.join(root, f), output_dir)
+                    found.append(rel.replace(os.sep, '/'))
         return sorted(found)
 
     def test_is_archive(self):
@@ -798,7 +811,7 @@ class TestArchiveInput:
             posted = self._run(client, zip_path, out)
 
             # all 3 PDFs processed, the .txt ignored
-            assert self._tei_outputs(out) == ['a.grobid.tei.xml', 'b.grobid.tei.xml', 'c.grobid.tei.xml']
+            assert self._tei_outputs(out) == ['a.grobid.tei.xml', 'c.grobid.tei.xml', 'sub/b.grobid.tei.xml']
             # each entry was posted straight from memory, under its archive name
             assert sorted(posted) == [
                 ('a.pdf', b'%PDF-a'),
@@ -813,7 +826,8 @@ class TestArchiveInput:
             self._make_targz(tar_path, {'x.pdf': b'%PDF-x', 'nested/y.pdf': b'%PDF-y'}, d)
             out = os.path.join(d, 'out')
             posted = self._run(client, tar_path, out)
-            assert self._tei_outputs(out) == ['x.grobid.tei.xml', 'y.grobid.tei.xml']
+            # the entry's path inside the archive is mirrored in the output
+            assert self._tei_outputs(out) == ['nested/y.grobid.tei.xml', 'x.grobid.tei.xml']
             assert sorted(posted) == [
                 (os.path.join('nested', 'y.pdf'), b'%PDF-y'),
                 ('x.pdf', b'%PDF-x'),
@@ -855,7 +869,7 @@ class TestArchiveInput:
                     client.process('processFulltextDocument', zip_path, output=out, force=True)
 
             mock_mkdtemp.assert_not_called()
-            assert self._tei_outputs(out) == ['a.grobid.tei.xml', 'b.grobid.tei.xml']
+            assert self._tei_outputs(out) == ['a.grobid.tei.xml', 'sub/b.grobid.tei.xml']
 
     def test_citation_lists_are_still_extracted_to_disk(self):
         """process_txt reads from a path, so citation lists keep the temp-dir route."""
@@ -1039,11 +1053,13 @@ class TestGlobInput:
 
     @staticmethod
     def _tei_outputs(output_dir):
+        """TEI outputs under output_dir, as paths relative to it (posix separators)."""
         found = []
         for root, _, files in os.walk(output_dir):
             for f in files:
                 if f.endswith('.grobid.tei.xml'):
-                    found.append(f)
+                    rel = os.path.relpath(os.path.join(root, f), output_dir)
+                    found.append(rel.replace(os.sep, '/'))
         return sorted(found)
 
     def _run(self, client, pattern, output):
@@ -1085,7 +1101,8 @@ class TestGlobInput:
             open(os.path.join(d, 'sub2', 'note.txt'), 'wb').close()
             out = os.path.join(d, 'out')
             self._run(client, os.path.join(d, '**', '*.pdf'), out)
-            assert self._tei_outputs(out) == ['a.grobid.tei.xml', 'b.grobid.tei.xml']
+            # loose glob matches are placed relative to their common ancestor
+            assert self._tei_outputs(out) == ['sub1/a.grobid.tei.xml', 'sub2/b.grobid.tei.xml']
 
     def test_glob_no_match_warns(self):
         client = self._client()
@@ -1107,6 +1124,107 @@ class TestGlobInput:
             assert f1.startswith(base) and f2.startswith(base)
 
 
+class TestDirectoryInput:
+    """A directory input is mirrored under --output (see issue about flat outputs).
+
+    The TEI of ``<in>/2023/p.pdf`` goes to ``<out>/2023/p.grobid.tei.xml``: the
+    subdirectories are kept, so two documents sharing a basename in different
+    subdirectories do not collide on the same output file.
+    """
+
+    def _client(self, queue_size=50):
+        with patch('grobid_client.grobid_client.GrobidClient._test_server_connection'):
+            with patch('grobid_client.grobid_client.GrobidClient._configure_logging'):
+                client = GrobidClient(check_server=False)
+        client.logger = Mock()
+        client.config['queue_size'] = queue_size
+        return client
+
+    @staticmethod
+    def _touch(*parts):
+        path = os.path.join(*parts)
+        os.makedirs(os.path.dirname(path), exist_ok=True)
+        open(path, 'wb').close()
+        return path
+
+    @staticmethod
+    def _tei_outputs(output_dir):
+        found = []
+        for root, _, files in os.walk(output_dir):
+            for f in files:
+                if f.endswith('.grobid.tei.xml'):
+                    rel = os.path.relpath(os.path.join(root, f), output_dir)
+                    found.append(rel.replace(os.sep, '/'))
+        return sorted(found)
+
+    def _run(self, client, input_dir, output, force=True):
+        posted = []
+
+        def fake_post(url, files=None, data=None, headers=None, timeout=None):
+            posted.append(files['input'][0])
+            resp = Mock()
+            resp.text = '<TEI>ok</TEI>'
+            return (resp, 200)
+
+        with patch.object(GrobidClient, 'post', side_effect=fake_post):
+            client.process('processFulltextDocument', input_dir, output=output, force=force)
+        return posted
+
+    def test_output_mirrors_input_tree(self):
+        client = self._client()
+        with tempfile.TemporaryDirectory() as d:
+            self._touch(d, 'in', 'top.pdf')
+            self._touch(d, 'in', '2023', 'p.pdf')
+            self._touch(d, 'in', '2024', 'p.pdf')       # same basename as 2023/p.pdf
+            self._touch(d, 'in', '2024', 'deep', 'q.pdf')
+            out = os.path.join(d, 'out')
+            posted = self._run(client, os.path.join(d, 'in'), out)
+            assert len(posted) == 4
+            assert self._tei_outputs(out) == [
+                '2023/p.grobid.tei.xml',
+                '2024/deep/q.grobid.tei.xml',
+                '2024/p.grobid.tei.xml',
+                'top.grobid.tei.xml',
+            ]
+
+    def test_same_basename_in_subdirectories_both_skipped_on_rerun(self):
+        """With flat outputs the second p.pdf used to be reported as already done."""
+        client = self._client()
+        with tempfile.TemporaryDirectory() as d:
+            self._touch(d, 'in', '2023', 'p.pdf')
+            self._touch(d, 'in', '2024', 'p.pdf')
+            out = os.path.join(d, 'out')
+            assert len(self._run(client, os.path.join(d, 'in'), out)) == 2
+            # second run without --force: both outputs exist, nothing is posted
+            assert self._run(client, os.path.join(d, 'in'), out, force=False) == []
+            assert self._tei_outputs(out) == ['2023/p.grobid.tei.xml', '2024/p.grobid.tei.xml']
+
+    def test_single_subdirectory_keeps_its_level(self):
+        """The base is the directory given as input, not the common path of the files."""
+        client = self._client()
+        with tempfile.TemporaryDirectory() as d:
+            self._touch(d, 'in', '2024', 'a.pdf')
+            self._touch(d, 'in', '2024', 'b.pdf')
+            out = os.path.join(d, 'out')
+            self._run(client, os.path.join(d, 'in'), out)
+            assert self._tei_outputs(out) == ['2024/a.grobid.tei.xml', '2024/b.grobid.tei.xml']
+
+    def test_trailing_slash_and_relative_input(self):
+        client = self._client()
+        with tempfile.TemporaryDirectory() as d:
+            self._touch(d, 'in', 'sub', 'a.pdf')
+            out = os.path.join(d, 'out')
+            self._run(client, os.path.join(d, 'in') + os.sep, out)
+            assert self._tei_outputs(out) == ['sub/a.grobid.tei.xml']
+
+    def test_no_output_writes_next_to_input(self):
+        client = self._client()
+        with tempfile.TemporaryDirectory() as d:
+            self._touch(d, 'in', 'sub', 'a.pdf')
+            self._run(client, os.path.join(d, 'in'), None)
+            assert os.path.isfile(os.path.join(d, 'in', 'sub', 'a.grobid.tei.xml'))
+
+
 class TestAtomicWrite:
     """A killed task must never leave a partial output that resume accepts.
 

grobid_client_python-0.2.1/grobid_client_python.egg-info/scm_version.json

@@ -1,8 +0,0 @@
-{
-  "tag": "0.2.1",
-  "distance": 0,
-  "node": "g39488b4dd319d720c68264c1cda19a1568bdcb76",
-  "dirty": false,
-  "branch": "HEAD",
-  "node_date": "2026-10-16"
-}