grobid-client-python 0.2.0__py3-none-any.whl → 0.2.2__py3-none-any.whl

grobid_client/grobid_client.py

@@ -34,7 +34,7 @@ import shutil
 import tarfile
 import tempfile
 import zipfile
-from typing import Any, BinaryIO, Callable, Optional, Tuple, Union
+from typing import Any, BinaryIO, Callable, List, Optional, Tuple, Union
 import copy
 
 from .format.TEI2LossyJSON import TEI2LossyJSONConverter
@@ -449,10 +449,15 @@ class GrobidClient(ApiClient):
         input_file_path = pathlib.Path(input_file)
 
         if output is not None:
-            # Calculate relative path from input_path, then join with output directory
+            # Mirror the input tree under the output directory: the path of the
+            # file relative to input_path is kept (only the extension changes),
+            # so sub/paper.pdf -> <output>/sub/paper.grobid.tei.xml. Keeping the
+            # subdirectories also prevents two inputs that share a basename
+            # from mapping to the same output file.
             input_path_abs = pathlib.Path(input_path).resolve()
             input_file_rel = input_file_path.resolve().relative_to(input_path_abs)
-            filename = pathlib.Path(output) / f"{input_file_rel.stem}.grobid.tei.xml"
+            filename = pathlib.Path(output) / input_file_rel.with_name(
+                f"{input_file_rel.stem}.grobid.tei.xml")
         else:
             # Use the same directory as the input file
             filename = input_file_path.parent / f"{input_file_path.stem}.grobid.tei.xml"
@@ -629,9 +634,14 @@ class GrobidClient(ApiClient):
 
         # Partition into archives (streamed), remote loose files (s3) and local
         # filesystem files (directories are expanded to their eligible files).
+        # Local files are grouped with the base directory their output paths
+        # are computed from: a directory given as input is the base of the
+        # files found inside it (so the output mirrors its tree), while loose
+        # files (single paths, glob matches) share a common ancestor.
         archive_paths = []
         remote_files = []
-        fs_files = []
+        fs_groups: List[Tuple[str, list]] = []
+        loose_files = []
         for path in matched_paths:
             if self._is_s3(path):
                 if self._looks_like_archive(path):
@@ -643,12 +653,20 @@ class GrobidClient(ApiClient):
             elif self._is_archive(path):
                 archive_paths.append(path)
             elif os.path.isdir(path):
-                fs_files.extend(self._collect_directory_files(path, service))
+                dir_files = self._collect_directory_files(path, service)
+                if dir_files:
+                    fs_groups.append((path, dir_files))
             elif os.path.isfile(path) and self._is_eligible_input(os.path.basename(path), service):
-                fs_files.append(path)
+                loose_files.append(path)
+            elif not os.path.exists(path):
+                self.logger.warning(f"Input path does not exist: {path}")
             else:
                 self.logger.debug(f"Skipping input (not an eligible file/dir/archive): {path}")
 
+        if loose_files:
+            fs_groups.append((self._common_base(loose_files), loose_files))
+        fs_files = [f for _, files in fs_groups for f in files]
+
         if not fs_files and not archive_paths and not remote_files:
             self.logger.warning(f"No eligible files found in input(s): {inputs}")
             return
@@ -667,16 +685,17 @@ class GrobidClient(ApiClient):
         # Local files gathered from directories and/or loose glob matches
         if fs_files:
             print(f"Found {len(fs_files)} local file(s) to process")
-            bp, be, bs = self._run_file_batches(
-                service, fs_files, self._common_base(fs_files), output, n,
-                generate_ids, consolidate_header, consolidate_citations,
-                include_raw_citations, include_raw_affiliations, tei_coordinates,
-                segment_sentences, force, verbose, flavor, json_output, markdown_output,
-                skip_errors=skip_errors
-            )
-            processed_files_count += bp
-            errors_files_count += be
-            skipped_files_count += bs
+            for base_dir, group_files in fs_groups:
+                bp, be, bs = self._run_file_batches(
+                    service, group_files, base_dir, output, n,
+                    generate_ids, consolidate_header, consolidate_citations,
+                    include_raw_citations, include_raw_affiliations, tei_coordinates,
+                    segment_sentences, force, verbose, flavor, json_output, markdown_output,
+                    skip_errors=skip_errors
+                )
+                processed_files_count += bp
+                errors_files_count += be
+                skipped_files_count += bs
             total_files += len(fs_files)
 
         # Loose remote (s3) files: streamed into memory one chunk at a time
@@ -927,6 +946,10 @@ class GrobidClient(ApiClient):
 
     def _is_eligible_input(self, filename: str, service: str) -> bool:
         """Return True if a file name is a valid input for the given service."""
+        # macOS AppleDouble metadata entries (._paper.pdf), which tar and zip
+        # add next to the real files, carry the extension but not the document
+        if filename.startswith("._"):
+            return False
         if filename.endswith(".pdf") or filename.endswith(".PDF"):
             return True
         if service == 'processCitationList' and (
@@ -1090,8 +1113,9 @@ class GrobidClient(ApiClient):
         the chunk size and nothing but the results ever touches the disk. (The
         exception is ``processCitationList``, whose ``.txt`` inputs are read
         from a path and therefore still go through a temporary directory.)
-        Output files follow the same flat naming convention as directory
-        processing (one ``<stem>`` per result, in ``output``).
+        Output files follow the same naming convention as directory
+        processing: the entry's path inside the archive is mirrored under
+        ``output`` (``nested/y.pdf`` -> ``<output>/nested/y.grobid.tei.xml``).
         """
         start_time = time.time()
         self._warn_on_consolidation_timeout(consolidate_citations)
@@ -1529,75 +1553,75 @@ class GrobidClient(ApiClient):
 
                 results.append(r)
 
-        for r in concurrent.futures.as_completed(results):
-            input_file, status, text = r.result()
-            filename = self._output_file_name(input_file, input_path, output)
+            for r in concurrent.futures.as_completed(results):
+                input_file, status, text = r.result()
+                filename = self._output_file_name(input_file, input_path, output)
 
-            if status != 200 or text is None:
-                self.logger.error(f"Processing of {input_file} failed with error {status}: {text}")
-                error_count += 1
-                # writing error file with suffixed error code
-                error_filename = self._error_file_name(filename, status)
-                try:
-                    self._write_atomic(error_filename, text if text is not None else "")
-                    self.logger.info(f"Error details written to {error_filename}")
-                    # A previous run may have failed with a different status code;
-                    # keep only the marker of this run so the error state of the
-                    # document stays unambiguous (and skippable).
-                    self._remove_error_files(filename, keep=error_filename)
-                except OSError as e:
-                    self.logger.error(f"Failed to write error file {filename}: {str(e)}")
-            else:
-                processed_count += 1
-                # writing TEI file
-                try:
-                    self._write_atomic(filename, text)
-                    self.logger.debug(f"Successfully wrote TEI file: {filename}")
+                if status != 200 or text is None:
+                    self.logger.error(f"Processing of {input_file} failed with error {status}: {text}")
+                    error_count += 1
+                    # writing error file with suffixed error code
+                    error_filename = self._error_file_name(filename, status)
+                    try:
+                        self._write_atomic(error_filename, text if text is not None else "")
+                        self.logger.info(f"Error details written to {error_filename}")
+                        # A previous run may have failed with a different status code;
+                        # keep only the marker of this run so the error state of the
+                        # document stays unambiguous (and skippable).
+                        self._remove_error_files(filename, keep=error_filename)
+                    except OSError as e:
+                        self.logger.error(f"Failed to write error file {filename}: {str(e)}")
+                else:
+                    processed_count += 1
+                    # writing TEI file
+                    try:
+                        self._write_atomic(filename, text)
+                        self.logger.debug(f"Successfully wrote TEI file: {filename}")
 
-                    # The document no longer is in error: drop the marker left by
-                    # a previous failed run so it is not skipped by --skip-errors.
-                    self._remove_error_files(filename)
+                        # The document no longer is in error: drop the marker left by
+                        # a previous failed run so it is not skipped by --skip-errors.
+                        self._remove_error_files(filename)
 
 
-                    # Convert to JSON if requested
-                    if json_output:
-                        try:
-                            converter = TEI2LossyJSONConverter()
-                            json_data = converter.convert_tei_file(filename, stream=False)
-                            
-                            if json_data:
-                                json_filename = filename.replace('.grobid.tei.xml', '.json')
-                                # Always write JSON file when TEI is written (respects --force behavior)
-                                json_filename_expanded = os.path.expanduser(json_filename)
-                                self._write_atomic(
-                                    json_filename_expanded,
-                                    json.dumps(json_data, indent=2, ensure_ascii=False))
-                                self.logger.debug(f"Successfully wrote JSON file: {json_filename_expanded}")
-                            else:
-                                self.logger.warning(f"Failed to convert TEI to JSON for {filename}")
-                        except Exception as e:
-                            self.logger.error(f"Failed to convert TEI to JSON for {filename}: {str(e)}")
-                    
-                    # Convert to Markdown if requested
-                    if markdown_output:
-                        try:
-                            from .format.TEI2Markdown import TEI2MarkdownConverter
-                            converter = TEI2MarkdownConverter()
-                            markdown_data = converter.convert_tei_file(filename)
-
-                            if markdown_data is not None:
-                                markdown_filename = filename.replace('.grobid.tei.xml', '.md')
-                                # Always write Markdown file when TEI is written (respects --force behavior)
-                                markdown_filename_expanded = os.path.expanduser(markdown_filename)
-                                self._write_atomic(markdown_filename_expanded, markdown_data)
-                                self.logger.debug(f"Successfully wrote Markdown file: {markdown_filename_expanded}")
-                            else:
-                                self.logger.warning(f"Failed to convert TEI to Markdown for {filename}")
-                        except Exception as e:
-                            self.logger.error(f"Failed to convert TEI to Markdown for {filename}: {str(e)}")
-                            
-                except OSError as e:
-                    self.logger.error(f"Failed to write TEI XML file {filename}: {str(e)}")
+                        # Convert to JSON if requested
+                        if json_output:
+                            try:
+                                converter = TEI2LossyJSONConverter()
+                                json_data = converter.convert_tei_file(filename, stream=False)
+                                
+                                if json_data:
+                                    json_filename = filename.replace('.grobid.tei.xml', '.json')
+                                    # Always write JSON file when TEI is written (respects --force behavior)
+                                    json_filename_expanded = os.path.expanduser(json_filename)
+                                    self._write_atomic(
+                                        json_filename_expanded,
+                                        json.dumps(json_data, indent=2, ensure_ascii=False))
+                                    self.logger.debug(f"Successfully wrote JSON file: {json_filename_expanded}")
+                                else:
+                                    self.logger.warning(f"Failed to convert TEI to JSON for {filename}")
+                            except Exception as e:
+                                self.logger.error(f"Failed to convert TEI to JSON for {filename}: {str(e)}")
+                        
+                        # Convert to Markdown if requested
+                        if markdown_output:
+                            try:
+                                from .format.TEI2Markdown import TEI2MarkdownConverter
+                                converter = TEI2MarkdownConverter()
+                                markdown_data = converter.convert_tei_file(filename)
+
+                                if markdown_data is not None:
+                                    markdown_filename = filename.replace('.grobid.tei.xml', '.md')
+                                    # Always write Markdown file when TEI is written (respects --force behavior)
+                                    markdown_filename_expanded = os.path.expanduser(markdown_filename)
+                                    self._write_atomic(markdown_filename_expanded, markdown_data)
+                                    self.logger.debug(f"Successfully wrote Markdown file: {markdown_filename_expanded}")
+                                else:
+                                    self.logger.warning(f"Failed to convert TEI to Markdown for {filename}")
+                            except Exception as e:
+                                self.logger.error(f"Failed to convert TEI to Markdown for {filename}: {str(e)}")
+                                
+                    except OSError as e:
+                        self.logger.error(f"Failed to write TEI XML file {filename}: {str(e)}")
 
         # Calculate batch statistics
         batch_runtime = time.time() - batch_start_time

{grobid_client_python-0.2.0.dist-info → grobid_client_python-0.2.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: grobid-client-python
-Version: 0.2.0
+Version: 0.2.2
 Summary: Simple python client for GROBID REST services
 Author-email: Patrice Lopez <patrice.lopez@science-miner.com>
 Maintainer-email: Patrice Lopez <patrice.lopez@science-miner.com>, Luca Foppiano <lucanoro@duck.com>
@@ -411,6 +411,9 @@ grobid_client --input "~/data/**/*.pdf"   --output ~/results processFulltextDocu
 
 > [!NOTE]
 > `--input` accepts a directory, a single file, an **archive**, or a **glob pattern**:
+> - **Directories** are processed recursively and the output mirrors the input tree: `~/pdfs/2023/a.pdf` →
+>   `~/results/2023/a.grobid.tei.xml` (subdirectories of `--output` are created as needed, and the JSON/Markdown
+>   outputs and error files sit next to each TEI file). Archives keep their internal paths the same way.
 > - **Archives** (`.zip`, `.tar`, `.tar.gz`/`.tgz`, `.tar.bz2`/`.tbz2`) are streamed: eligible entries are read into
 >   memory in chunks of `queue_size` and sent to GROBID straight from there — the archive is never fully decompressed and
 >   nothing but the results in `--output` ever touches the disk. If `--output` is omitted, results go to a directory named
@@ -636,18 +639,19 @@ settings.
 
 ### Choosing a queue size
 
-`queue_size` controls how many files are grouped into one processing chunk. It is a memory/durability knob, not a
+`queue_size` controls how many files are grouped into one processing chunk. It is a memory knob, not a
 concurrency one: parallelism toward the GROBID server is set by `-n`, and each chunk is processed `n` files at a time.
 When `queue_size` is not set, the client picks a sensible default per input type (see the table above); set it
 explicitly only if you need to override that. A few guidelines:
 
 - **Never set it below `n`.** Effective parallelism is `min(n, queue_size)`: a queue smaller than the thread pool
   leaves workers idle. A bit of headroom above `n` (the default streaming value is 1.2 × `n`) keeps the pool busy.
-- **Results are written per chunk.** Output files land on disk only once a whole chunk has been processed, so a
-  larger queue means results are written less frequently and an interrupted run loses at most one chunk of work
+- **Results are written as they complete.** Each output file lands on disk as soon as its document has been
+  processed, whatever the queue size, so an interrupted run only loses the documents that were in flight
   (already-written results are skipped on re-run unless `--force` is used).
-- **Local PDF directories:** the queue holds only file paths, so large values are essentially free — the default is
-  1000. Lower it if you want results flushed to disk more often on long runs.
+- **Local PDF directories:** the queue holds only file paths (each file is opened when it is sent), so large values
+  are essentially free — the default is 1000. At the end of each chunk the pool waits for its slowest document, so a
+  large queue keeps the workers busy.
 - **Archives (zip/tar) and S3:** each chunk is read or downloaded *into memory* before processing starts, so peak RAM
   grows with `queue_size × average file size`. Keep it moderate — the 1.2 × `n` default is a safe floor; going up to
   a few multiples of `n` (e.g. 2–5 ×) trades memory for slightly better throughput around chunk boundaries.

{grobid_client_python-0.2.0.dist-info → grobid_client_python-0.2.2.dist-info}/RECORD

@@ -1,6 +1,6 @@
 grobid_client/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 grobid_client/client.py,sha256=kAmRDQECMt6ck7AytnDAvMz8-KnM4QCIEp6rfFHqdzI,6925
-grobid_client/grobid_client.py,sha256=qwHMiOFQYcGcHonYM9yZgbJE9kY8SUTB2925kW4d0Io,93286
+grobid_client/grobid_client.py,sha256=slqvpBW-3m-p1JQMQtfDJiZTrEEeLQ4-TyC3Ns_CRQc,94970
 grobid_client/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 grobid_client/format/TEI2LossyJSON.py,sha256=5KtHaQbP2leMEiFLi3yV70SPSDqKvCq-Zf3jmHTSEj8,50337
 grobid_client/format/TEI2LossyJSON_cli.py,sha256=TPsrE913MJBlTsVrd4Oc16AF3tXqwi_IzTt8UYcpZnU,3848
@@ -9,8 +9,8 @@ grobid_client/format/TEI2Markdown_cli.py,sha256=SZ5zMT0Lo7QG0DXypPyuA5TuJrgieVJs
 grobid_client/format/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 grobid_client/format/__main__.py,sha256=jklj0sd9_2QbZm7MWwnZsdH1XtepKTJX2hbZBCMvK78,1848
 grobid_client/format/validate_json_refs.py,sha256=iJimV0d9iPYqpw9b54C-c39AmtAKuyfMEWLS2F-QwRQ,14893
-grobid_client_python-0.2.0.dist-info/METADATA,sha256=oKiamjdg8kseXjlvVt5CknHxKxYPd9Fhj87Rj_S6ryg,43610
-grobid_client_python-0.2.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
-grobid_client_python-0.2.0.dist-info/entry_points.txt,sha256=-WwPi8D8eXEbDB6QvitvHjxSGBeyLCyFvAoZcaDGswQ,67
-grobid_client_python-0.2.0.dist-info/top_level.txt,sha256=HnjgWxdJNKTLIN86XkgRAfVeEK4C71fc9X4fe7qdMvA,14
-grobid_client_python-0.2.0.dist-info/RECORD,,
+grobid_client_python-0.2.2.dist-info/METADATA,sha256=X1FrPOqH42bSaFRAjPXQ1Qgg529YjpbR-YtQQes__Ss,43992
+grobid_client_python-0.2.2.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
+grobid_client_python-0.2.2.dist-info/entry_points.txt,sha256=-WwPi8D8eXEbDB6QvitvHjxSGBeyLCyFvAoZcaDGswQ,67
+grobid_client_python-0.2.2.dist-info/top_level.txt,sha256=HnjgWxdJNKTLIN86XkgRAfVeEK4C71fc9X4fe7qdMvA,14
+grobid_client_python-0.2.2.dist-info/RECORD,,

{grobid_client_python-0.2.0.dist-info → grobid_client_python-0.2.2.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (84.0.0)
+Generator: setuptools (84.2.1)
 Root-Is-Purelib: true
 Tag: py3-none-any