grobid-client-python 0.2.0__tar.gz → 0.2.1__tar.gz

{grobid_client_python-0.2.0 → grobid_client_python-0.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: grobid-client-python
-Version: 0.2.0
+Version: 0.2.1
 Summary: Simple python client for GROBID REST services
 Author-email: Patrice Lopez <patrice.lopez@science-miner.com>
 Maintainer-email: Patrice Lopez <patrice.lopez@science-miner.com>, Luca Foppiano <lucanoro@duck.com>
@@ -636,18 +636,19 @@ settings.
 
 ### Choosing a queue size
 
-`queue_size` controls how many files are grouped into one processing chunk. It is a memory/durability knob, not a
+`queue_size` controls how many files are grouped into one processing chunk. It is a memory knob, not a
 concurrency one: parallelism toward the GROBID server is set by `-n`, and each chunk is processed `n` files at a time.
 When `queue_size` is not set, the client picks a sensible default per input type (see the table above); set it
 explicitly only if you need to override that. A few guidelines:
 
 - **Never set it below `n`.** Effective parallelism is `min(n, queue_size)`: a queue smaller than the thread pool
   leaves workers idle. A bit of headroom above `n` (the default streaming value is 1.2 × `n`) keeps the pool busy.
-- **Results are written per chunk.** Output files land on disk only once a whole chunk has been processed, so a
-  larger queue means results are written less frequently and an interrupted run loses at most one chunk of work
+- **Results are written as they complete.** Each output file lands on disk as soon as its document has been
+  processed, whatever the queue size, so an interrupted run only loses the documents that were in flight
   (already-written results are skipped on re-run unless `--force` is used).
-- **Local PDF directories:** the queue holds only file paths, so large values are essentially free — the default is
-  1000. Lower it if you want results flushed to disk more often on long runs.
+- **Local PDF directories:** the queue holds only file paths (each file is opened when it is sent), so large values
+  are essentially free — the default is 1000. At the end of each chunk the pool waits for its slowest document, so a
+  large queue keeps the workers busy.
 - **Archives (zip/tar) and S3:** each chunk is read or downloaded *into memory* before processing starts, so peak RAM
   grows with `queue_size × average file size`. Keep it moderate — the 1.2 × `n` default is a safe floor; going up to
   a few multiples of `n` (e.g. 2–5 ×) trades memory for slightly better throughput around chunk boundaries.

{grobid_client_python-0.2.0 → grobid_client_python-0.2.1}/Readme.md

@@ -414,18 +414,19 @@ settings.
 
 ### Choosing a queue size
 
-`queue_size` controls how many files are grouped into one processing chunk. It is a memory/durability knob, not a
+`queue_size` controls how many files are grouped into one processing chunk. It is a memory knob, not a
 concurrency one: parallelism toward the GROBID server is set by `-n`, and each chunk is processed `n` files at a time.
 When `queue_size` is not set, the client picks a sensible default per input type (see the table above); set it
 explicitly only if you need to override that. A few guidelines:
 
 - **Never set it below `n`.** Effective parallelism is `min(n, queue_size)`: a queue smaller than the thread pool
   leaves workers idle. A bit of headroom above `n` (the default streaming value is 1.2 × `n`) keeps the pool busy.
-- **Results are written per chunk.** Output files land on disk only once a whole chunk has been processed, so a
-  larger queue means results are written less frequently and an interrupted run loses at most one chunk of work
+- **Results are written as they complete.** Each output file lands on disk as soon as its document has been
+  processed, whatever the queue size, so an interrupted run only loses the documents that were in flight
   (already-written results are skipped on re-run unless `--force` is used).
-- **Local PDF directories:** the queue holds only file paths, so large values are essentially free — the default is
-  1000. Lower it if you want results flushed to disk more often on long runs.
+- **Local PDF directories:** the queue holds only file paths (each file is opened when it is sent), so large values
+  are essentially free — the default is 1000. At the end of each chunk the pool waits for its slowest document, so a
+  large queue keeps the workers busy.
 - **Archives (zip/tar) and S3:** each chunk is read or downloaded *into memory* before processing starts, so peak RAM
   grows with `queue_size × average file size`. Keep it moderate — the 1.2 × `n` default is a safe floor; going up to
   a few multiples of `n` (e.g. 2–5 ×) trades memory for slightly better throughput around chunk boundaries.

{grobid_client_python-0.2.0 → grobid_client_python-0.2.1}/grobid_client/grobid_client.py

@@ -646,6 +646,8 @@ class GrobidClient(ApiClient):
                 fs_files.extend(self._collect_directory_files(path, service))
             elif os.path.isfile(path) and self._is_eligible_input(os.path.basename(path), service):
                 fs_files.append(path)
+            elif not os.path.exists(path):
+                self.logger.warning(f"Input path does not exist: {path}")
             else:
                 self.logger.debug(f"Skipping input (not an eligible file/dir/archive): {path}")
 
@@ -927,6 +929,10 @@ class GrobidClient(ApiClient):
 
     def _is_eligible_input(self, filename: str, service: str) -> bool:
         """Return True if a file name is a valid input for the given service."""
+        # macOS AppleDouble metadata entries (._paper.pdf), which tar and zip
+        # add next to the real files, carry the extension but not the document
+        if filename.startswith("._"):
+            return False
         if filename.endswith(".pdf") or filename.endswith(".PDF"):
             return True
         if service == 'processCitationList' and (
@@ -1529,75 +1535,75 @@ class GrobidClient(ApiClient):
 
                 results.append(r)
 
-        for r in concurrent.futures.as_completed(results):
-            input_file, status, text = r.result()
-            filename = self._output_file_name(input_file, input_path, output)
+            for r in concurrent.futures.as_completed(results):
+                input_file, status, text = r.result()
+                filename = self._output_file_name(input_file, input_path, output)
 
-            if status != 200 or text is None:
-                self.logger.error(f"Processing of {input_file} failed with error {status}: {text}")
-                error_count += 1
-                # writing error file with suffixed error code
-                error_filename = self._error_file_name(filename, status)
-                try:
-                    self._write_atomic(error_filename, text if text is not None else "")
-                    self.logger.info(f"Error details written to {error_filename}")
-                    # A previous run may have failed with a different status code;
-                    # keep only the marker of this run so the error state of the
-                    # document stays unambiguous (and skippable).
-                    self._remove_error_files(filename, keep=error_filename)
-                except OSError as e:
-                    self.logger.error(f"Failed to write error file {filename}: {str(e)}")
-            else:
-                processed_count += 1
-                # writing TEI file
-                try:
-                    self._write_atomic(filename, text)
-                    self.logger.debug(f"Successfully wrote TEI file: {filename}")
+                if status != 200 or text is None:
+                    self.logger.error(f"Processing of {input_file} failed with error {status}: {text}")
+                    error_count += 1
+                    # writing error file with suffixed error code
+                    error_filename = self._error_file_name(filename, status)
+                    try:
+                        self._write_atomic(error_filename, text if text is not None else "")
+                        self.logger.info(f"Error details written to {error_filename}")
+                        # A previous run may have failed with a different status code;
+                        # keep only the marker of this run so the error state of the
+                        # document stays unambiguous (and skippable).
+                        self._remove_error_files(filename, keep=error_filename)
+                    except OSError as e:
+                        self.logger.error(f"Failed to write error file {filename}: {str(e)}")
+                else:
+                    processed_count += 1
+                    # writing TEI file
+                    try:
+                        self._write_atomic(filename, text)
+                        self.logger.debug(f"Successfully wrote TEI file: {filename}")
 
-                    # The document no longer is in error: drop the marker left by
-                    # a previous failed run so it is not skipped by --skip-errors.
-                    self._remove_error_files(filename)
+                        # The document no longer is in error: drop the marker left by
+                        # a previous failed run so it is not skipped by --skip-errors.
+                        self._remove_error_files(filename)
 
 
-                    # Convert to JSON if requested
-                    if json_output:
-                        try:
-                            converter = TEI2LossyJSONConverter()
-                            json_data = converter.convert_tei_file(filename, stream=False)
-                            
-                            if json_data:
-                                json_filename = filename.replace('.grobid.tei.xml', '.json')
-                                # Always write JSON file when TEI is written (respects --force behavior)
-                                json_filename_expanded = os.path.expanduser(json_filename)
-                                self._write_atomic(
-                                    json_filename_expanded,
-                                    json.dumps(json_data, indent=2, ensure_ascii=False))
-                                self.logger.debug(f"Successfully wrote JSON file: {json_filename_expanded}")
-                            else:
-                                self.logger.warning(f"Failed to convert TEI to JSON for {filename}")
-                        except Exception as e:
-                            self.logger.error(f"Failed to convert TEI to JSON for {filename}: {str(e)}")
-                    
-                    # Convert to Markdown if requested
-                    if markdown_output:
-                        try:
-                            from .format.TEI2Markdown import TEI2MarkdownConverter
-                            converter = TEI2MarkdownConverter()
-                            markdown_data = converter.convert_tei_file(filename)
-
-                            if markdown_data is not None:
-                                markdown_filename = filename.replace('.grobid.tei.xml', '.md')
-                                # Always write Markdown file when TEI is written (respects --force behavior)
-                                markdown_filename_expanded = os.path.expanduser(markdown_filename)
-                                self._write_atomic(markdown_filename_expanded, markdown_data)
-                                self.logger.debug(f"Successfully wrote Markdown file: {markdown_filename_expanded}")
-                            else:
-                                self.logger.warning(f"Failed to convert TEI to Markdown for {filename}")
-                        except Exception as e:
-                            self.logger.error(f"Failed to convert TEI to Markdown for {filename}: {str(e)}")
-                            
-                except OSError as e:
-                    self.logger.error(f"Failed to write TEI XML file {filename}: {str(e)}")
+                        # Convert to JSON if requested
+                        if json_output:
+                            try:
+                                converter = TEI2LossyJSONConverter()
+                                json_data = converter.convert_tei_file(filename, stream=False)
+                                
+                                if json_data:
+                                    json_filename = filename.replace('.grobid.tei.xml', '.json')
+                                    # Always write JSON file when TEI is written (respects --force behavior)
+                                    json_filename_expanded = os.path.expanduser(json_filename)
+                                    self._write_atomic(
+                                        json_filename_expanded,
+                                        json.dumps(json_data, indent=2, ensure_ascii=False))
+                                    self.logger.debug(f"Successfully wrote JSON file: {json_filename_expanded}")
+                                else:
+                                    self.logger.warning(f"Failed to convert TEI to JSON for {filename}")
+                            except Exception as e:
+                                self.logger.error(f"Failed to convert TEI to JSON for {filename}: {str(e)}")
+                        
+                        # Convert to Markdown if requested
+                        if markdown_output:
+                            try:
+                                from .format.TEI2Markdown import TEI2MarkdownConverter
+                                converter = TEI2MarkdownConverter()
+                                markdown_data = converter.convert_tei_file(filename)
+
+                                if markdown_data is not None:
+                                    markdown_filename = filename.replace('.grobid.tei.xml', '.md')
+                                    # Always write Markdown file when TEI is written (respects --force behavior)
+                                    markdown_filename_expanded = os.path.expanduser(markdown_filename)
+                                    self._write_atomic(markdown_filename_expanded, markdown_data)
+                                    self.logger.debug(f"Successfully wrote Markdown file: {markdown_filename_expanded}")
+                                else:
+                                    self.logger.warning(f"Failed to convert TEI to Markdown for {filename}")
+                            except Exception as e:
+                                self.logger.error(f"Failed to convert TEI to Markdown for {filename}: {str(e)}")
+                                
+                    except OSError as e:
+                        self.logger.error(f"Failed to write TEI XML file {filename}: {str(e)}")
 
         # Calculate batch statistics
         batch_runtime = time.time() - batch_start_time

{grobid_client_python-0.2.0 → grobid_client_python-0.2.1}/grobid_client_python.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: grobid-client-python
-Version: 0.2.0
+Version: 0.2.1
 Summary: Simple python client for GROBID REST services
 Author-email: Patrice Lopez <patrice.lopez@science-miner.com>
 Maintainer-email: Patrice Lopez <patrice.lopez@science-miner.com>, Luca Foppiano <lucanoro@duck.com>
@@ -636,18 +636,19 @@ settings.
 
 ### Choosing a queue size
 
-`queue_size` controls how many files are grouped into one processing chunk. It is a memory/durability knob, not a
+`queue_size` controls how many files are grouped into one processing chunk. It is a memory knob, not a
 concurrency one: parallelism toward the GROBID server is set by `-n`, and each chunk is processed `n` files at a time.
 When `queue_size` is not set, the client picks a sensible default per input type (see the table above); set it
 explicitly only if you need to override that. A few guidelines:
 
 - **Never set it below `n`.** Effective parallelism is `min(n, queue_size)`: a queue smaller than the thread pool
   leaves workers idle. A bit of headroom above `n` (the default streaming value is 1.2 × `n`) keeps the pool busy.
-- **Results are written per chunk.** Output files land on disk only once a whole chunk has been processed, so a
-  larger queue means results are written less frequently and an interrupted run loses at most one chunk of work
+- **Results are written as they complete.** Each output file lands on disk as soon as its document has been
+  processed, whatever the queue size, so an interrupted run only loses the documents that were in flight
   (already-written results are skipped on re-run unless `--force` is used).
-- **Local PDF directories:** the queue holds only file paths, so large values are essentially free — the default is
-  1000. Lower it if you want results flushed to disk more often on long runs.
+- **Local PDF directories:** the queue holds only file paths (each file is opened when it is sent), so large values
+  are essentially free — the default is 1000. At the end of each chunk the pool waits for its slowest document, so a
+  large queue keeps the workers busy.
 - **Archives (zip/tar) and S3:** each chunk is read or downloaded *into memory* before processing starts, so peak RAM
   grows with `queue_size × average file size`. Keep it moderate — the 1.2 × `n` default is a safe floor; going up to
   a few multiples of `n` (e.g. 2–5 ×) trades memory for slightly better throughput around chunk boundaries.

grobid_client_python-0.2.1/grobid_client_python.egg-info/scm_version.json

@@ -0,0 +1,8 @@
+{
+  "tag": "0.2.1",
+  "distance": 0,
+  "node": "g39488b4dd319d720c68264c1cda19a1568bdcb76",
+  "dirty": false,
+  "branch": "HEAD",
+  "node_date": "2026-10-16"
+}

{grobid_client_python-0.2.0 → grobid_client_python-0.2.1}/pyproject.toml

@@ -3,7 +3,7 @@ requires = ["setuptools", "setuptools-scm"]
 build-backend = "setuptools.build_meta"
 
 [tool.bumpversion]
-current_version = "0.2.0"
+current_version = "0.2.1"
 commit = "true"
 tag = "true"
 tag_name = "v{new_version}"

{grobid_client_python-0.2.0 → grobid_client_python-0.2.1}/tests/test_grobid_client.py

@@ -1378,6 +1378,43 @@ class TestSkipErrors:
             assert (processed, errors, skipped) == (0, 0, 1)
 
 
+class TestIncrementalWrites:
+    """Results are written as each document finishes, not once the whole queue is done."""
+
+    def test_finished_document_is_written_while_others_are_in_flight(self):
+        with patch('grobid_client.grobid_client.GrobidClient._test_server_connection'):
+            with patch('grobid_client.grobid_client.GrobidClient._configure_logging'):
+                client = GrobidClient(check_server=False)
+        client.logger = Mock()
+
+        with tempfile.TemporaryDirectory() as d:
+            inp = os.path.join(d, 'in')
+            out = os.path.join(d, 'out')
+            os.makedirs(inp)
+            for name in ('a.pdf', 'b.pdf'):
+                with open(os.path.join(inp, name), 'wb') as f:
+                    f.write(b'%PDF')
+            a_output = os.path.join(out, 'a.grobid.tei.xml')
+            seen_while_in_flight = []
+
+            def fake_post(url, files=None, data=None, headers=None, timeout=None):
+                if os.path.basename(files['input'][0]) == 'b.pdf':
+                    # Hold b until a's result reaches the disk (or give up)
+                    deadline = time.time() + 5
+                    while not os.path.isfile(a_output) and time.time() < deadline:
+                        time.sleep(0.01)
+                    seen_while_in_flight.append(os.path.isfile(a_output))
+                resp = Mock()
+                resp.text = '<TEI>ok</TEI>'
+                return (resp, 200)
+
+            with patch.object(GrobidClient, 'post', side_effect=fake_post):
+                client.process('processFulltextDocument', inp, output=out, n=2)
+
+            assert seen_while_in_flight == [True]
+            assert os.path.isfile(os.path.join(out, 'b.grobid.tei.xml'))
+
+
 class TestInMemoryDocuments:
     """Processing a PDF that is already in memory, without writing it to disk.
 

grobid_client_python-0.2.0/grobid_client_python.egg-info/scm_version.json

@@ -1,8 +0,0 @@
-{
-  "tag": "0.2.0",
-  "distance": 0,
-  "node": "gfd1858e863d62f7a4ee0d1a2d8dc4f96d1e407c9",
-  "dirty": false,
-  "branch": "HEAD",
-  "node_date": "2026-08-22"
-}