grobid-client-python 0.1.4__py3-none-any.whl → 0.1.5__py3-none-any.whl

grobid_client/grobid_client.py

@@ -20,6 +20,7 @@ import argparse
 import time
 import concurrent.futures
 import ntpath
+import re
 import requests
 import pathlib
 import logging
@@ -39,6 +40,11 @@ class ServerUnavailableException(Exception):
 
 
 class GrobidClient(ApiClient):
+    # Below this client-side timeout (in seconds) citation consolidation is
+    # likely to trigger HTTP 408 (Request Timeout) errors, so we warn the user.
+    # See https://github.com/grobidOrg/grobid-client-python/issues/54
+    CONSOLIDATE_CITATIONS_MIN_TIMEOUT = 120
+
     # Default configuration values
     DEFAULT_CONFIG = {
         'grobid_server': 'http://localhost:8070',
@@ -112,6 +118,27 @@ class GrobidClient(ApiClient):
             if value is not None:
                 self.config[key] = value
 
+    def _warn_on_consolidation_timeout(self, consolidate_citations):
+        """Warn when citation consolidation is enabled with a low client timeout.
+
+        Consolidating citations makes GROBID query external services and can be
+        much slower than a plain extraction. A short client-side timeout often
+        leads to HTTP 408 errors, so we recommend at least a couple of minutes.
+        See https://github.com/grobidOrg/grobid-client-python/issues/54
+        """
+        if not consolidate_citations:
+            return
+
+        timeout = self.config.get("timeout", self.DEFAULT_CONFIG["timeout"])
+        if timeout < self.CONSOLIDATE_CITATIONS_MIN_TIMEOUT:
+            self.logger.warning(
+                f"Citation consolidation is enabled but the timeout is only {timeout}s. "
+                f"Consolidation queries external services and can be slow; a low timeout "
+                f"frequently causes HTTP 408 (Request Timeout) errors. Consider increasing "
+                f"the 'timeout' setting to at least {self.CONSOLIDATE_CITATIONS_MIN_TIMEOUT}s "
+                f"(2-3 minutes is recommended)."
+            )
+
     def _handle_server_busy_retry(self, file_path, retry_func, *args, **kwargs):
         """Handle server busy (503) retry logic."""
         self.logger.warning(f"Server busy (503), retrying {file_path} after {self.config['sleep_time']} seconds")
@@ -209,7 +236,6 @@ class GrobidClient(ApiClient):
         size_str = str(size_str).upper().strip()
 
         # Extract number and unit
-        import re
         match = re.match(r'(\d+(?:\.\d+)?)\s*([KMGT]?B?)', size_str)
         if not match:
             return 10 * 1024 * 1024  # Default 10MB
@@ -329,7 +355,7 @@ class GrobidClient(ApiClient):
             input_path,
             output=None,
             n=10,
-            generateIDs=False,
+            generate_ids=False,
             consolidate_header=True,
             consolidate_citations=False,
             include_raw_citations=False,
@@ -345,6 +371,13 @@ class GrobidClient(ApiClient):
         start_time = time.time()
         batch_size_pdf = self.config["batch_size"]
 
+        # Warn if citation consolidation is requested with a short timeout: the
+        # consolidation step queries external services (e.g. CrossRef) and can
+        # be significantly slower, frequently resulting in HTTP 408 errors when
+        # the client-side timeout is too low.
+        # See https://github.com/grobidOrg/grobid-client-python/issues/54
+        self._warn_on_consolidation_timeout(consolidate_citations)
+
         # First pass: count all eligible files
         all_input_files = []
         for (dirpath, dirnames, filenames) in os.walk(input_path):
@@ -391,7 +424,7 @@ class GrobidClient(ApiClient):
                     input_path,
                     output,
                     n,
-                    generateIDs,
+                    generate_ids,
                     consolidate_header,
                     consolidate_citations,
                     include_raw_citations,
@@ -417,7 +450,7 @@ class GrobidClient(ApiClient):
                 input_path,
                 output,
                 n,
-                generateIDs,
+                generate_ids,
                 consolidate_header,
                 consolidate_citations,
                 include_raw_citations,
@@ -455,7 +488,7 @@ class GrobidClient(ApiClient):
             input_path,
             output,
             n,
-            generateIDs,
+            generate_ids,
             consolidate_header,
             consolidate_citations,
             include_raw_citations,
@@ -542,7 +575,7 @@ class GrobidClient(ApiClient):
                     selected_process,
                     service,
                     input_file,
-                    generateIDs,
+                    generate_ids,
                     consolidate_header,
                     consolidate_citations,
                     include_raw_citations,
@@ -639,7 +672,7 @@ class GrobidClient(ApiClient):
             self,
             service,
             pdf_file,
-            generateIDs,
+            generate_ids,
             consolidate_header,
             consolidate_citations,
             include_raw_citations,
@@ -667,7 +700,7 @@ class GrobidClient(ApiClient):
 
             # set the GROBID parameters
             the_data = {}
-            if generateIDs:
+            if generate_ids:
                 the_data["generateIDs"] = "1"
             if consolidate_header:
                 the_data["consolidateHeader"] = "1"
@@ -699,7 +732,7 @@ class GrobidClient(ApiClient):
                     self.process_pdf,
                     service,
                     pdf_file,
-                    generateIDs,
+                    generate_ids,
                     consolidate_header,
                     consolidate_citations,
                     include_raw_citations,
@@ -734,7 +767,7 @@ class GrobidClient(ApiClient):
             self,
             service,
             txt_file,
-            generateIDs,
+            generate_ids,
             consolidate_header,
             consolidate_citations,
             include_raw_citations,
@@ -777,7 +810,7 @@ class GrobidClient(ApiClient):
                     self.process_txt,
                     service,
                     txt_file,
-                    generateIDs,
+                    generate_ids,
                     consolidate_header,
                     consolidate_citations,
                     include_raw_citations,
@@ -840,7 +873,8 @@ def main():
         help="concurrency for service usage"
     )
     parser.add_argument(
-        "--generateIDs",
+        "--generate_ids", "--generateIDs",
+        dest="generate_ids",
         action="store_true",
         help="generate random xml:id to textual XML elements of the result files",
     )
@@ -870,12 +904,14 @@ def main():
         help="force re-processing pdf input files when tei output files already exist",
     )
     parser.add_argument(
-        "--teiCoordinates",
+        "--tei_coordinates", "--teiCoordinates",
+        dest="tei_coordinates",
         action="store_true",
         help="add the original PDF coordinates (bounding boxes) to the extracted elements",
     )
     parser.add_argument(
-        "--segmentSentences",
+        "--segment_sentences", "--segmentSentences",
+        dest="segment_sentences",
         action="store_true",
         help="segment sentences in the text content of the document with additional <s> elements",
     )
@@ -951,14 +987,14 @@ def main():
             exit(1)
 
     service = args.service
-    generateIDs = args.generateIDs
+    generate_ids = args.generate_ids
     consolidate_header = args.consolidate_header
     consolidate_citations = args.consolidate_citations
     include_raw_citations = args.include_raw_citations
     include_raw_affiliations = args.include_raw_affiliations
     force = args.force
-    tei_coordinates = args.teiCoordinates
-    segment_sentences = args.segmentSentences
+    tei_coordinates = args.tei_coordinates
+    segment_sentences = args.segment_sentences
     verbose = args.verbose
 
     if service is None or service not in valid_services:
@@ -973,7 +1009,7 @@ def main():
             input_path,
             output=output_path,
             n=n,
-            generateIDs=generateIDs,
+            generate_ids=generate_ids,
             consolidate_header=consolidate_header,
             consolidate_citations=consolidate_citations,
             include_raw_citations=include_raw_citations,

{grobid_client_python-0.1.4.dist-info → grobid_client_python-0.1.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: grobid-client-python
-Version: 0.1.4
+Version: 0.1.5
 Summary: Simple python client for GROBID REST services
 Author-email: Patrice Lopez <patrice.lopez@science-miner.com>
 Maintainer-email: Patrice Lopez <patrice.lopez@science-miner.com>, Luca Foppiano <lucanoro@duck.com>
@@ -353,13 +353,13 @@ grobid_client [OPTIONS] SERVICE
 
 | Option                       | Description                               |
 |------------------------------|-------------------------------------------|
-| `--generateIDs`              | Generate random XML IDs                   |
+| `--generate_ids`             | Generate random XML IDs                   |
 | `--consolidate_header`       | Consolidate header metadata               |
 | `--consolidate_citations`    | Consolidate bibliographic references      |
 | `--include_raw_citations`    | Include raw citation text                 |
 | `--include_raw_affiliations` | Include raw affiliation text              |
-| `--teiCoordinates`           | Add PDF coordinates to XML                |
-| `--segmentSentences`         | Segment sentences with coordinates        |
+| `--tei_coordinates`          | Add PDF coordinates to XML                |
+| `--segment_sentences`        | Segment sentences with coordinates        |
 | `--flavor`                   | Processing flavor for fulltext extraction |
 | `--json`                     | Convert TEI output to JSON format         |
 | `--markdown`                 | Convert TEI output to Markdown format     |
@@ -372,7 +372,7 @@ grobid_client [OPTIONS] SERVICE
 grobid_client --input ~/documents --output ~/results processFulltextDocument
 
 # High concurrency with coordinates
-grobid_client --input ~/pdfs --output ~/tei --n 20 --teiCoordinates processFulltextDocument
+grobid_client --input ~/pdfs --output ~/tei --n 20 --tei_coordinates processFulltextDocument
 
 # Process with JSON output
 grobid_client --input ~/pdfs --output ~/results --json processFulltextDocument
@@ -384,7 +384,7 @@ grobid_client --input ~/pdfs --output ~/results --markdown processFulltextDocume
 grobid_client --server https://grobid.example.com --input ~/citations.txt processCitationList
 
 # Force reprocessing with sentence segmentation and JSON output
-grobid_client --input ~/docs --force --segmentSentences --json processFulltextDocument
+grobid_client --input ~/docs --force --segment_sentences --json processFulltextDocument
 ```
 
 ### Python Library
@@ -421,10 +421,10 @@ client.process(
     input_path="/path/to/pdfs",
     output_path="/path/to/output",
     n=10,
-    generateIDs=True,
+    generate_ids=True,
     consolidate_header=True,
-    teiCoordinates=True,
-    segmentSentences=True
+    tei_coordinates=True,
+    segment_sentences=True
 )
 
 # Process with JSON output
@@ -517,6 +517,14 @@ settings.
 > Since version 0.0.12, the config file is optional. The client will use default localhost settings if no configuration
 > is provided.
 
+> [!WARNING]
+> **Citation consolidation and the `timeout` setting.** When `--consolidate_citations` (or `consolidate_citations=True`)
+> is enabled, GROBID queries external services (e.g. CrossRef) to enrich the extracted references. This is considerably
+> slower than a plain extraction, and a low `timeout` frequently causes `HTTP 408 (Request Timeout)` errors.
+> Set the `timeout` to at least **120 seconds (2-3 minutes recommended)** when consolidating citations. The client emits
+> a warning when consolidation is requested with a timeout below 120 seconds.
+> See [issue #54](https://github.com/grobidOrg/grobid-client-python/issues/54).
+
 ### Logging Configuration
 
 The client provides configurable logging with different verbosity levels. By default, only essential statistics and warnings are shown.
@@ -673,7 +681,7 @@ When using the `--json` flag, the client converts TEI XML output to a structured
 grobid_client --input pdfs/ --output results/ --json processFulltextDocument
 
 # JSON output with coordinates and sentence segmentation
-grobid_client --input pdfs/ --output results/ --json --teiCoordinates --segmentSentences processFulltextDocument
+grobid_client --input pdfs/ --output results/ --json --tei_coordinates --segment_sentences processFulltextDocument
 ```
 
 ```python
@@ -754,7 +762,7 @@ Competing interests statement...
 grobid_client --input pdfs/ --output results/ --markdown processFulltextDocument
 
 # Markdown output with coordinates and sentence segmentation
-grobid_client --input pdfs/ --output results/ --markdown --teiCoordinates --segmentSentences processFulltextDocument
+grobid_client --input pdfs/ --output results/ --markdown --tei_coordinates --segment_sentences processFulltextDocument
 ```
 
 ```python

{grobid_client_python-0.1.4.dist-info → grobid_client_python-0.1.5.dist-info}/RECORD

@@ -1,6 +1,6 @@
 grobid_client/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 grobid_client/client.py,sha256=crvUiVnnkn6tFwO4lKQg2cTxAHZxCBAmeNSVtuL4ZA0,6009
-grobid_client/grobid_client.py,sha256=uXmDwbgEQosKwJXPcNu1aIIdqCxeanmr8RqtSeghIgk,40547
+grobid_client/grobid_client.py,sha256=sEBvP-lpL079SWqHLV7wxGt9N4lmdgWr-RzNcz8sDqc,42548
 grobid_client/format/TEI2LossyJSON.py,sha256=1Ew2Y3zbLtWfz2LEyqAVRPu66850Nney9JHjwoTyI3Y,49162
 grobid_client/format/TEI2LossyJSON_cli.py,sha256=6l5nNVc7m03TGI19eIqhN6cxYMH5mZT4K9aK1P0g3So,3796
 grobid_client/format/TEI2Markdown.py,sha256=7HTLuuy8OJFhA58BZ1cxKKF4GGIXIWuxTMRd1tg9PdQ,29505
@@ -8,8 +8,8 @@ grobid_client/format/TEI2Markdown_cli.py,sha256=wVaMIXfDQUW8Fqu9b5Gj0Q-MbZ2Gv1jC
 grobid_client/format/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 grobid_client/format/__main__.py,sha256=jBiiNtyhn7Ys6RC6ynUD4cv1YWXHmpZw4rHmnYfVybI,1804
 grobid_client/format/validate_json_refs.py,sha256=_ieADmrHD77EFwblTNH1_i0x3P7SCmsaTi-PZ6SI79M,14842
-grobid_client_python-0.1.4.dist-info/METADATA,sha256=GRs12gLkqakZJ1YXO0Ils_DLLf9k5wSSkSX-ZkCbwtk,34414
-grobid_client_python-0.1.4.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-grobid_client_python-0.1.4.dist-info/entry_points.txt,sha256=-WwPi8D8eXEbDB6QvitvHjxSGBeyLCyFvAoZcaDGswQ,67
-grobid_client_python-0.1.4.dist-info/top_level.txt,sha256=HnjgWxdJNKTLIN86XkgRAfVeEK4C71fc9X4fe7qdMvA,14
-grobid_client_python-0.1.4.dist-info/RECORD,,
+grobid_client_python-0.1.5.dist-info/METADATA,sha256=Rq2Ac6aIlIW7NACByuIYQW-Ohuz9l555XTBk3eMFxZ0,35066
+grobid_client_python-0.1.5.dist-info/WHEEL,sha256=K260EYznzXsJYBQGqmI8VTxEdiZYNvDZwW9cBh9-_MA,91
+grobid_client_python-0.1.5.dist-info/entry_points.txt,sha256=-WwPi8D8eXEbDB6QvitvHjxSGBeyLCyFvAoZcaDGswQ,67
+grobid_client_python-0.1.5.dist-info/top_level.txt,sha256=HnjgWxdJNKTLIN86XkgRAfVeEK4C71fc9X4fe7qdMvA,14
+grobid_client_python-0.1.5.dist-info/RECORD,,

{grobid_client_python-0.1.4.dist-info → grobid_client_python-0.1.5.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.9.0)
+Generator: setuptools (83.0.0)
 Root-Is-Purelib: true
 Tag: py3-none-any