grobid-client-python 0.1.4__tar.gz → 0.1.5__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. grobid_client_python-0.1.5/.github/dependabot.yml +9 -0
  2. {grobid_client_python-0.1.4 → grobid_client_python-0.1.5}/.github/workflows/ci-build.yml +3 -3
  3. {grobid_client_python-0.1.4 → grobid_client_python-0.1.5}/.github/workflows/ci-release.yml +2 -2
  4. {grobid_client_python-0.1.4 → grobid_client_python-0.1.5}/PKG-INFO +19 -11
  5. {grobid_client_python-0.1.4 → grobid_client_python-0.1.5}/Readme.md +18 -10
  6. {grobid_client_python-0.1.4 → grobid_client_python-0.1.5}/grobid_client/grobid_client.py +54 -18
  7. {grobid_client_python-0.1.4 → grobid_client_python-0.1.5}/grobid_client_python.egg-info/PKG-INFO +19 -11
  8. {grobid_client_python-0.1.4 → grobid_client_python-0.1.5}/grobid_client_python.egg-info/SOURCES.txt +3 -0
  9. grobid_client_python-0.1.5/grobid_client_python.egg-info/scm_file_list.json +63 -0
  10. grobid_client_python-0.1.5/grobid_client_python.egg-info/scm_version.json +8 -0
  11. {grobid_client_python-0.1.4 → grobid_client_python-0.1.5}/pyproject.toml +1 -1
  12. {grobid_client_python-0.1.4 → grobid_client_python-0.1.5}/tests/test_grobid_client.py +9 -9
  13. {grobid_client_python-0.1.4 → grobid_client_python-0.1.5}/tests/test_integration.py +2 -2
  14. {grobid_client_python-0.1.4 → grobid_client_python-0.1.5}/.gitignore +0 -0
  15. {grobid_client_python-0.1.4 → grobid_client_python-0.1.5}/LICENSE +0 -0
  16. {grobid_client_python-0.1.4 → grobid_client_python-0.1.5}/MANIFEST.in +0 -0
  17. {grobid_client_python-0.1.4 → grobid_client_python-0.1.5}/config.json +0 -0
  18. {grobid_client_python-0.1.4 → grobid_client_python-0.1.5}/example.py +0 -0
  19. {grobid_client_python-0.1.4 → grobid_client_python-0.1.5}/grobid_client/__init__.py +0 -0
  20. {grobid_client_python-0.1.4 → grobid_client_python-0.1.5}/grobid_client/client.py +0 -0
  21. {grobid_client_python-0.1.4 → grobid_client_python-0.1.5}/grobid_client/format/TEI2LossyJSON.py +0 -0
  22. {grobid_client_python-0.1.4 → grobid_client_python-0.1.5}/grobid_client/format/TEI2LossyJSON_cli.py +0 -0
  23. {grobid_client_python-0.1.4 → grobid_client_python-0.1.5}/grobid_client/format/TEI2Markdown.py +0 -0
  24. {grobid_client_python-0.1.4 → grobid_client_python-0.1.5}/grobid_client/format/TEI2Markdown_cli.py +0 -0
  25. {grobid_client_python-0.1.4 → grobid_client_python-0.1.5}/grobid_client/format/__init__.py +0 -0
  26. {grobid_client_python-0.1.4 → grobid_client_python-0.1.5}/grobid_client/format/__main__.py +0 -0
  27. {grobid_client_python-0.1.4 → grobid_client_python-0.1.5}/grobid_client/format/validate_json_refs.py +0 -0
  28. {grobid_client_python-0.1.4 → grobid_client_python-0.1.5}/grobid_client_python.egg-info/dependency_links.txt +0 -0
  29. {grobid_client_python-0.1.4 → grobid_client_python-0.1.5}/grobid_client_python.egg-info/entry_points.txt +0 -0
  30. {grobid_client_python-0.1.4 → grobid_client_python-0.1.5}/grobid_client_python.egg-info/requires.txt +0 -0
  31. {grobid_client_python-0.1.4 → grobid_client_python-0.1.5}/grobid_client_python.egg-info/top_level.txt +0 -0
  32. {grobid_client_python-0.1.4 → grobid_client_python-0.1.5}/pytest.ini +0 -0
  33. {grobid_client_python-0.1.4 → grobid_client_python-0.1.5}/requirements.txt +0 -0
  34. {grobid_client_python-0.1.4 → grobid_client_python-0.1.5}/resources/20180928112135.png +0 -0
  35. {grobid_client_python-0.1.4 → grobid_client_python-0.1.5}/resources/20180928112138.csv +0 -0
  36. {grobid_client_python-0.1.4 → grobid_client_python-0.1.5}/resources/test_pdf/0046d83a-edd6-4631-b57c-755cdcce8b7f.pdf +0 -0
  37. {grobid_client_python-0.1.4 → grobid_client_python-0.1.5}/resources/test_pdf/0046d83a-edd6-4631-b57c-755cdcce8b7f.tei.xml +0 -0
  38. {grobid_client_python-0.1.4 → grobid_client_python-0.1.5}/resources/test_pdf/mjb3wlzxcb2mc-migowebupload-1766042162782.grobid.tei.xml +0 -0
  39. {grobid_client_python-0.1.4 → grobid_client_python-0.1.5}/resources/test_txt/reference-strings-1.txt +0 -0
  40. {grobid_client_python-0.1.4 → grobid_client_python-0.1.5}/resources/test_txt/reference-strings-2.txt +0 -0
  41. {grobid_client_python-0.1.4 → grobid_client_python-0.1.5}/resources/test_txt/reference-strings-3.txt +0 -0
  42. {grobid_client_python-0.1.4 → grobid_client_python-0.1.5}/resources/test_txt/reference-strings-4.txt +0 -0
  43. {grobid_client_python-0.1.4 → grobid_client_python-0.1.5}/resources/test_txt/reference-strings-5.txt +0 -0
  44. {grobid_client_python-0.1.4 → grobid_client_python-0.1.5}/resources/test_txt/reference-strings-6.txt +0 -0
  45. {grobid_client_python-0.1.4 → grobid_client_python-0.1.5}/resources/test_txt/reference-strings-7.txt +0 -0
  46. {grobid_client_python-0.1.4 → grobid_client_python-0.1.5}/setup.cfg +0 -0
  47. {grobid_client_python-0.1.4 → grobid_client_python-0.1.5}/tests/__init__.py +0 -0
  48. {grobid_client_python-0.1.4 → grobid_client_python-0.1.5}/tests/conftest.py +0 -0
  49. {grobid_client_python-0.1.4 → grobid_client_python-0.1.5}/tests/resources/0046d83a-edd6-4631-b57c-755cdcce8b7f.tei.xml +0 -0
  50. {grobid_client_python-0.1.4 → grobid_client_python-0.1.5}/tests/resources/__init__.py +0 -0
  51. {grobid_client_python-0.1.4 → grobid_client_python-0.1.5}/tests/resources/article_withdrawn.grobid.tei.xml +0 -0
  52. {grobid_client_python-0.1.4 → grobid_client_python-0.1.5}/tests/resources/refs_offsets/10.1038_s41477-023-01501-1.grobid.tei.xml +0 -0
  53. {grobid_client_python-0.1.4 → grobid_client_python-0.1.5}/tests/resources/refs_offsets/10.1038_s41477-023-01501-1.json +0 -0
  54. {grobid_client_python-0.1.4 → grobid_client_python-0.1.5}/tests/resources/refs_offsets/10.1038_s41586-023-05895-y.grobid.tei.xml +0 -0
  55. {grobid_client_python-0.1.4 → grobid_client_python-0.1.5}/tests/resources/refs_offsets/10.1038_s41586-023-05895-y.json +0 -0
  56. {grobid_client_python-0.1.4 → grobid_client_python-0.1.5}/tests/resources/refs_offsets/10.1038_s41598-023-32039-z.grobid.tei.xml +0 -0
  57. {grobid_client_python-0.1.4 → grobid_client_python-0.1.5}/tests/resources/refs_offsets/10.1038_s41598-023-32039-z.json +0 -0
  58. {grobid_client_python-0.1.4 → grobid_client_python-0.1.5}/tests/resources/refs_offsets/10.1371_journal.pone.0218311.grobid.tei.xml +0 -0
  59. {grobid_client_python-0.1.4 → grobid_client_python-0.1.5}/tests/resources/refs_offsets/10.1371_journal.pone.0218311.json +0 -0
  60. {grobid_client_python-0.1.4 → grobid_client_python-0.1.5}/tests/resources/refs_offsets/10.7554_elife.78558.grobid.tei.xml +0 -0
  61. {grobid_client_python-0.1.4 → grobid_client_python-0.1.5}/tests/resources/refs_offsets/10.7554_elife.78558.json +0 -0
  62. {grobid_client_python-0.1.4 → grobid_client_python-0.1.5}/tests/resources/refs_offsets/2021.naacl-main.224.grobid.tei.xml +0 -0
  63. {grobid_client_python-0.1.4 → grobid_client_python-0.1.5}/tests/resources/repeated_text/ijms-24-05988.grobid.tei.xml +0 -0
  64. {grobid_client_python-0.1.4 → grobid_client_python-0.1.5}/tests/resources/repeated_text/ijms-24-05988.json +0 -0
  65. {grobid_client_python-0.1.4 → grobid_client_python-0.1.5}/tests/resources/repeated_text/ijms-24-05988.md +0 -0
  66. {grobid_client_python-0.1.4 → grobid_client_python-0.1.5}/tests/run_tests.py +0 -0
  67. {grobid_client_python-0.1.4 → grobid_client_python-0.1.5}/tests/test_client.py +0 -0
  68. {grobid_client_python-0.1.4 → grobid_client_python-0.1.5}/tests/test_config.json +0 -0
  69. {grobid_client_python-0.1.4 → grobid_client_python-0.1.5}/tests/test_conversions.py +0 -0
@@ -0,0 +1,9 @@
1
+ version: 2
2
+ updates:
3
+ # Keep GitHub Actions used in the workflows up to date
4
+ - package-ecosystem: "github-actions"
5
+ directory: "/"
6
+ schedule:
7
+ interval: "weekly"
8
+ commit-message:
9
+ prefix: "ci"
@@ -15,9 +15,9 @@ jobs:
15
15
  fail-fast: true
16
16
 
17
17
  steps:
18
- - uses: actions/checkout@v4
18
+ - uses: actions/checkout@v7
19
19
  - name: Set up Python ${{ matrix.python-version }}
20
- uses: actions/setup-python@v5
20
+ uses: actions/setup-python@v7
21
21
  with:
22
22
  python-version: ${{ matrix.python-version }}
23
23
  cache: 'pip'
@@ -43,7 +43,7 @@ jobs:
43
43
  pytest tests/ -v --cov=grobid_client --cov-report=xml --cov-report=term-missing
44
44
  - name: Upload coverage to Codecov
45
45
  if: matrix.python-version == '3.11'
46
- uses: codecov/codecov-action@v4
46
+ uses: codecov/codecov-action@v7
47
47
  with:
48
48
  file: ./coverage.xml
49
49
  flags: unittests
@@ -14,9 +14,9 @@ jobs:
14
14
  build:
15
15
  runs-on: ubuntu-latest
16
16
  steps:
17
- - uses: actions/checkout@v4
17
+ - uses: actions/checkout@v7
18
18
  - name: Set up Python 3.9
19
- uses: actions/setup-python@v5
19
+ uses: actions/setup-python@v7
20
20
  with:
21
21
  python-version: "3.9"
22
22
  cache: 'pip'
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: grobid-client-python
3
- Version: 0.1.4
3
+ Version: 0.1.5
4
4
  Summary: Simple python client for GROBID REST services
5
5
  Author-email: Patrice Lopez <patrice.lopez@science-miner.com>
6
6
  Maintainer-email: Patrice Lopez <patrice.lopez@science-miner.com>, Luca Foppiano <lucanoro@duck.com>
@@ -353,13 +353,13 @@ grobid_client [OPTIONS] SERVICE
353
353
 
354
354
  | Option | Description |
355
355
  |------------------------------|-------------------------------------------|
356
- | `--generateIDs` | Generate random XML IDs |
356
+ | `--generate_ids` | Generate random XML IDs |
357
357
  | `--consolidate_header` | Consolidate header metadata |
358
358
  | `--consolidate_citations` | Consolidate bibliographic references |
359
359
  | `--include_raw_citations` | Include raw citation text |
360
360
  | `--include_raw_affiliations` | Include raw affiliation text |
361
- | `--teiCoordinates` | Add PDF coordinates to XML |
362
- | `--segmentSentences` | Segment sentences with coordinates |
361
+ | `--tei_coordinates` | Add PDF coordinates to XML |
362
+ | `--segment_sentences` | Segment sentences with coordinates |
363
363
  | `--flavor` | Processing flavor for fulltext extraction |
364
364
  | `--json` | Convert TEI output to JSON format |
365
365
  | `--markdown` | Convert TEI output to Markdown format |
@@ -372,7 +372,7 @@ grobid_client [OPTIONS] SERVICE
372
372
  grobid_client --input ~/documents --output ~/results processFulltextDocument
373
373
 
374
374
  # High concurrency with coordinates
375
- grobid_client --input ~/pdfs --output ~/tei --n 20 --teiCoordinates processFulltextDocument
375
+ grobid_client --input ~/pdfs --output ~/tei --n 20 --tei_coordinates processFulltextDocument
376
376
 
377
377
  # Process with JSON output
378
378
  grobid_client --input ~/pdfs --output ~/results --json processFulltextDocument
@@ -384,7 +384,7 @@ grobid_client --input ~/pdfs --output ~/results --markdown processFulltextDocume
384
384
  grobid_client --server https://grobid.example.com --input ~/citations.txt processCitationList
385
385
 
386
386
  # Force reprocessing with sentence segmentation and JSON output
387
- grobid_client --input ~/docs --force --segmentSentences --json processFulltextDocument
387
+ grobid_client --input ~/docs --force --segment_sentences --json processFulltextDocument
388
388
  ```
389
389
 
390
390
  ### Python Library
@@ -421,10 +421,10 @@ client.process(
421
421
  input_path="/path/to/pdfs",
422
422
  output_path="/path/to/output",
423
423
  n=10,
424
- generateIDs=True,
424
+ generate_ids=True,
425
425
  consolidate_header=True,
426
- teiCoordinates=True,
427
- segmentSentences=True
426
+ tei_coordinates=True,
427
+ segment_sentences=True
428
428
  )
429
429
 
430
430
  # Process with JSON output
@@ -517,6 +517,14 @@ settings.
517
517
  > Since version 0.0.12, the config file is optional. The client will use default localhost settings if no configuration
518
518
  > is provided.
519
519
 
520
+ > [!WARNING]
521
+ > **Citation consolidation and the `timeout` setting.** When `--consolidate_citations` (or `consolidate_citations=True`)
522
+ > is enabled, GROBID queries external services (e.g. CrossRef) to enrich the extracted references. This is considerably
523
+ > slower than a plain extraction, and a low `timeout` frequently causes `HTTP 408 (Request Timeout)` errors.
524
+ > Set the `timeout` to at least **120 seconds (2-3 minutes recommended)** when consolidating citations. The client emits
525
+ > a warning when consolidation is requested with a timeout below 120 seconds.
526
+ > See [issue #54](https://github.com/grobidOrg/grobid-client-python/issues/54).
527
+
520
528
  ### Logging Configuration
521
529
 
522
530
  The client provides configurable logging with different verbosity levels. By default, only essential statistics and warnings are shown.
@@ -673,7 +681,7 @@ When using the `--json` flag, the client converts TEI XML output to a structured
673
681
  grobid_client --input pdfs/ --output results/ --json processFulltextDocument
674
682
 
675
683
  # JSON output with coordinates and sentence segmentation
676
- grobid_client --input pdfs/ --output results/ --json --teiCoordinates --segmentSentences processFulltextDocument
684
+ grobid_client --input pdfs/ --output results/ --json --tei_coordinates --segment_sentences processFulltextDocument
677
685
  ```
678
686
 
679
687
  ```python
@@ -754,7 +762,7 @@ Competing interests statement...
754
762
  grobid_client --input pdfs/ --output results/ --markdown processFulltextDocument
755
763
 
756
764
  # Markdown output with coordinates and sentence segmentation
757
- grobid_client --input pdfs/ --output results/ --markdown --teiCoordinates --segmentSentences processFulltextDocument
765
+ grobid_client --input pdfs/ --output results/ --markdown --tei_coordinates --segment_sentences processFulltextDocument
758
766
  ```
759
767
 
760
768
  ```python
@@ -134,13 +134,13 @@ grobid_client [OPTIONS] SERVICE
134
134
 
135
135
  | Option | Description |
136
136
  |------------------------------|-------------------------------------------|
137
- | `--generateIDs` | Generate random XML IDs |
137
+ | `--generate_ids` | Generate random XML IDs |
138
138
  | `--consolidate_header` | Consolidate header metadata |
139
139
  | `--consolidate_citations` | Consolidate bibliographic references |
140
140
  | `--include_raw_citations` | Include raw citation text |
141
141
  | `--include_raw_affiliations` | Include raw affiliation text |
142
- | `--teiCoordinates` | Add PDF coordinates to XML |
143
- | `--segmentSentences` | Segment sentences with coordinates |
142
+ | `--tei_coordinates` | Add PDF coordinates to XML |
143
+ | `--segment_sentences` | Segment sentences with coordinates |
144
144
  | `--flavor` | Processing flavor for fulltext extraction |
145
145
  | `--json` | Convert TEI output to JSON format |
146
146
  | `--markdown` | Convert TEI output to Markdown format |
@@ -153,7 +153,7 @@ grobid_client [OPTIONS] SERVICE
153
153
  grobid_client --input ~/documents --output ~/results processFulltextDocument
154
154
 
155
155
  # High concurrency with coordinates
156
- grobid_client --input ~/pdfs --output ~/tei --n 20 --teiCoordinates processFulltextDocument
156
+ grobid_client --input ~/pdfs --output ~/tei --n 20 --tei_coordinates processFulltextDocument
157
157
 
158
158
  # Process with JSON output
159
159
  grobid_client --input ~/pdfs --output ~/results --json processFulltextDocument
@@ -165,7 +165,7 @@ grobid_client --input ~/pdfs --output ~/results --markdown processFulltextDocume
165
165
  grobid_client --server https://grobid.example.com --input ~/citations.txt processCitationList
166
166
 
167
167
  # Force reprocessing with sentence segmentation and JSON output
168
- grobid_client --input ~/docs --force --segmentSentences --json processFulltextDocument
168
+ grobid_client --input ~/docs --force --segment_sentences --json processFulltextDocument
169
169
  ```
170
170
 
171
171
  ### Python Library
@@ -202,10 +202,10 @@ client.process(
202
202
  input_path="/path/to/pdfs",
203
203
  output_path="/path/to/output",
204
204
  n=10,
205
- generateIDs=True,
205
+ generate_ids=True,
206
206
  consolidate_header=True,
207
- teiCoordinates=True,
208
- segmentSentences=True
207
+ tei_coordinates=True,
208
+ segment_sentences=True
209
209
  )
210
210
 
211
211
  # Process with JSON output
@@ -298,6 +298,14 @@ settings.
298
298
  > Since version 0.0.12, the config file is optional. The client will use default localhost settings if no configuration
299
299
  > is provided.
300
300
 
301
+ > [!WARNING]
302
+ > **Citation consolidation and the `timeout` setting.** When `--consolidate_citations` (or `consolidate_citations=True`)
303
+ > is enabled, GROBID queries external services (e.g. CrossRef) to enrich the extracted references. This is considerably
304
+ > slower than a plain extraction, and a low `timeout` frequently causes `HTTP 408 (Request Timeout)` errors.
305
+ > Set the `timeout` to at least **120 seconds (2-3 minutes recommended)** when consolidating citations. The client emits
306
+ > a warning when consolidation is requested with a timeout below 120 seconds.
307
+ > See [issue #54](https://github.com/grobidOrg/grobid-client-python/issues/54).
308
+
301
309
  ### Logging Configuration
302
310
 
303
311
  The client provides configurable logging with different verbosity levels. By default, only essential statistics and warnings are shown.
@@ -454,7 +462,7 @@ When using the `--json` flag, the client converts TEI XML output to a structured
454
462
  grobid_client --input pdfs/ --output results/ --json processFulltextDocument
455
463
 
456
464
  # JSON output with coordinates and sentence segmentation
457
- grobid_client --input pdfs/ --output results/ --json --teiCoordinates --segmentSentences processFulltextDocument
465
+ grobid_client --input pdfs/ --output results/ --json --tei_coordinates --segment_sentences processFulltextDocument
458
466
  ```
459
467
 
460
468
  ```python
@@ -535,7 +543,7 @@ Competing interests statement...
535
543
  grobid_client --input pdfs/ --output results/ --markdown processFulltextDocument
536
544
 
537
545
  # Markdown output with coordinates and sentence segmentation
538
- grobid_client --input pdfs/ --output results/ --markdown --teiCoordinates --segmentSentences processFulltextDocument
546
+ grobid_client --input pdfs/ --output results/ --markdown --tei_coordinates --segment_sentences processFulltextDocument
539
547
  ```
540
548
 
541
549
  ```python
@@ -20,6 +20,7 @@ import argparse
20
20
  import time
21
21
  import concurrent.futures
22
22
  import ntpath
23
+ import re
23
24
  import requests
24
25
  import pathlib
25
26
  import logging
@@ -39,6 +40,11 @@ class ServerUnavailableException(Exception):
39
40
 
40
41
 
41
42
  class GrobidClient(ApiClient):
43
+ # Below this client-side timeout (in seconds) citation consolidation is
44
+ # likely to trigger HTTP 408 (Request Timeout) errors, so we warn the user.
45
+ # See https://github.com/grobidOrg/grobid-client-python/issues/54
46
+ CONSOLIDATE_CITATIONS_MIN_TIMEOUT = 120
47
+
42
48
  # Default configuration values
43
49
  DEFAULT_CONFIG = {
44
50
  'grobid_server': 'http://localhost:8070',
@@ -112,6 +118,27 @@ class GrobidClient(ApiClient):
112
118
  if value is not None:
113
119
  self.config[key] = value
114
120
 
121
+ def _warn_on_consolidation_timeout(self, consolidate_citations):
122
+ """Warn when citation consolidation is enabled with a low client timeout.
123
+
124
+ Consolidating citations makes GROBID query external services and can be
125
+ much slower than a plain extraction. A short client-side timeout often
126
+ leads to HTTP 408 errors, so we recommend at least a couple of minutes.
127
+ See https://github.com/grobidOrg/grobid-client-python/issues/54
128
+ """
129
+ if not consolidate_citations:
130
+ return
131
+
132
+ timeout = self.config.get("timeout", self.DEFAULT_CONFIG["timeout"])
133
+ if timeout < self.CONSOLIDATE_CITATIONS_MIN_TIMEOUT:
134
+ self.logger.warning(
135
+ f"Citation consolidation is enabled but the timeout is only {timeout}s. "
136
+ f"Consolidation queries external services and can be slow; a low timeout "
137
+ f"frequently causes HTTP 408 (Request Timeout) errors. Consider increasing "
138
+ f"the 'timeout' setting to at least {self.CONSOLIDATE_CITATIONS_MIN_TIMEOUT}s "
139
+ f"(2-3 minutes is recommended)."
140
+ )
141
+
115
142
  def _handle_server_busy_retry(self, file_path, retry_func, *args, **kwargs):
116
143
  """Handle server busy (503) retry logic."""
117
144
  self.logger.warning(f"Server busy (503), retrying {file_path} after {self.config['sleep_time']} seconds")
@@ -209,7 +236,6 @@ class GrobidClient(ApiClient):
209
236
  size_str = str(size_str).upper().strip()
210
237
 
211
238
  # Extract number and unit
212
- import re
213
239
  match = re.match(r'(\d+(?:\.\d+)?)\s*([KMGT]?B?)', size_str)
214
240
  if not match:
215
241
  return 10 * 1024 * 1024 # Default 10MB
@@ -329,7 +355,7 @@ class GrobidClient(ApiClient):
329
355
  input_path,
330
356
  output=None,
331
357
  n=10,
332
- generateIDs=False,
358
+ generate_ids=False,
333
359
  consolidate_header=True,
334
360
  consolidate_citations=False,
335
361
  include_raw_citations=False,
@@ -345,6 +371,13 @@ class GrobidClient(ApiClient):
345
371
  start_time = time.time()
346
372
  batch_size_pdf = self.config["batch_size"]
347
373
 
374
+ # Warn if citation consolidation is requested with a short timeout: the
375
+ # consolidation step queries external services (e.g. CrossRef) and can
376
+ # be significantly slower, frequently resulting in HTTP 408 errors when
377
+ # the client-side timeout is too low.
378
+ # See https://github.com/grobidOrg/grobid-client-python/issues/54
379
+ self._warn_on_consolidation_timeout(consolidate_citations)
380
+
348
381
  # First pass: count all eligible files
349
382
  all_input_files = []
350
383
  for (dirpath, dirnames, filenames) in os.walk(input_path):
@@ -391,7 +424,7 @@ class GrobidClient(ApiClient):
391
424
  input_path,
392
425
  output,
393
426
  n,
394
- generateIDs,
427
+ generate_ids,
395
428
  consolidate_header,
396
429
  consolidate_citations,
397
430
  include_raw_citations,
@@ -417,7 +450,7 @@ class GrobidClient(ApiClient):
417
450
  input_path,
418
451
  output,
419
452
  n,
420
- generateIDs,
453
+ generate_ids,
421
454
  consolidate_header,
422
455
  consolidate_citations,
423
456
  include_raw_citations,
@@ -455,7 +488,7 @@ class GrobidClient(ApiClient):
455
488
  input_path,
456
489
  output,
457
490
  n,
458
- generateIDs,
491
+ generate_ids,
459
492
  consolidate_header,
460
493
  consolidate_citations,
461
494
  include_raw_citations,
@@ -542,7 +575,7 @@ class GrobidClient(ApiClient):
542
575
  selected_process,
543
576
  service,
544
577
  input_file,
545
- generateIDs,
578
+ generate_ids,
546
579
  consolidate_header,
547
580
  consolidate_citations,
548
581
  include_raw_citations,
@@ -639,7 +672,7 @@ class GrobidClient(ApiClient):
639
672
  self,
640
673
  service,
641
674
  pdf_file,
642
- generateIDs,
675
+ generate_ids,
643
676
  consolidate_header,
644
677
  consolidate_citations,
645
678
  include_raw_citations,
@@ -667,7 +700,7 @@ class GrobidClient(ApiClient):
667
700
 
668
701
  # set the GROBID parameters
669
702
  the_data = {}
670
- if generateIDs:
703
+ if generate_ids:
671
704
  the_data["generateIDs"] = "1"
672
705
  if consolidate_header:
673
706
  the_data["consolidateHeader"] = "1"
@@ -699,7 +732,7 @@ class GrobidClient(ApiClient):
699
732
  self.process_pdf,
700
733
  service,
701
734
  pdf_file,
702
- generateIDs,
735
+ generate_ids,
703
736
  consolidate_header,
704
737
  consolidate_citations,
705
738
  include_raw_citations,
@@ -734,7 +767,7 @@ class GrobidClient(ApiClient):
734
767
  self,
735
768
  service,
736
769
  txt_file,
737
- generateIDs,
770
+ generate_ids,
738
771
  consolidate_header,
739
772
  consolidate_citations,
740
773
  include_raw_citations,
@@ -777,7 +810,7 @@ class GrobidClient(ApiClient):
777
810
  self.process_txt,
778
811
  service,
779
812
  txt_file,
780
- generateIDs,
813
+ generate_ids,
781
814
  consolidate_header,
782
815
  consolidate_citations,
783
816
  include_raw_citations,
@@ -840,7 +873,8 @@ def main():
840
873
  help="concurrency for service usage"
841
874
  )
842
875
  parser.add_argument(
843
- "--generateIDs",
876
+ "--generate_ids", "--generateIDs",
877
+ dest="generate_ids",
844
878
  action="store_true",
845
879
  help="generate random xml:id to textual XML elements of the result files",
846
880
  )
@@ -870,12 +904,14 @@ def main():
870
904
  help="force re-processing pdf input files when tei output files already exist",
871
905
  )
872
906
  parser.add_argument(
873
- "--teiCoordinates",
907
+ "--tei_coordinates", "--teiCoordinates",
908
+ dest="tei_coordinates",
874
909
  action="store_true",
875
910
  help="add the original PDF coordinates (bounding boxes) to the extracted elements",
876
911
  )
877
912
  parser.add_argument(
878
- "--segmentSentences",
913
+ "--segment_sentences", "--segmentSentences",
914
+ dest="segment_sentences",
879
915
  action="store_true",
880
916
  help="segment sentences in the text content of the document with additional <s> elements",
881
917
  )
@@ -951,14 +987,14 @@ def main():
951
987
  exit(1)
952
988
 
953
989
  service = args.service
954
- generateIDs = args.generateIDs
990
+ generate_ids = args.generate_ids
955
991
  consolidate_header = args.consolidate_header
956
992
  consolidate_citations = args.consolidate_citations
957
993
  include_raw_citations = args.include_raw_citations
958
994
  include_raw_affiliations = args.include_raw_affiliations
959
995
  force = args.force
960
- tei_coordinates = args.teiCoordinates
961
- segment_sentences = args.segmentSentences
996
+ tei_coordinates = args.tei_coordinates
997
+ segment_sentences = args.segment_sentences
962
998
  verbose = args.verbose
963
999
 
964
1000
  if service is None or service not in valid_services:
@@ -973,7 +1009,7 @@ def main():
973
1009
  input_path,
974
1010
  output=output_path,
975
1011
  n=n,
976
- generateIDs=generateIDs,
1012
+ generate_ids=generate_ids,
977
1013
  consolidate_header=consolidate_header,
978
1014
  consolidate_citations=consolidate_citations,
979
1015
  include_raw_citations=include_raw_citations,
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: grobid-client-python
3
- Version: 0.1.4
3
+ Version: 0.1.5
4
4
  Summary: Simple python client for GROBID REST services
5
5
  Author-email: Patrice Lopez <patrice.lopez@science-miner.com>
6
6
  Maintainer-email: Patrice Lopez <patrice.lopez@science-miner.com>, Luca Foppiano <lucanoro@duck.com>
@@ -353,13 +353,13 @@ grobid_client [OPTIONS] SERVICE
353
353
 
354
354
  | Option | Description |
355
355
  |------------------------------|-------------------------------------------|
356
- | `--generateIDs` | Generate random XML IDs |
356
+ | `--generate_ids` | Generate random XML IDs |
357
357
  | `--consolidate_header` | Consolidate header metadata |
358
358
  | `--consolidate_citations` | Consolidate bibliographic references |
359
359
  | `--include_raw_citations` | Include raw citation text |
360
360
  | `--include_raw_affiliations` | Include raw affiliation text |
361
- | `--teiCoordinates` | Add PDF coordinates to XML |
362
- | `--segmentSentences` | Segment sentences with coordinates |
361
+ | `--tei_coordinates` | Add PDF coordinates to XML |
362
+ | `--segment_sentences` | Segment sentences with coordinates |
363
363
  | `--flavor` | Processing flavor for fulltext extraction |
364
364
  | `--json` | Convert TEI output to JSON format |
365
365
  | `--markdown` | Convert TEI output to Markdown format |
@@ -372,7 +372,7 @@ grobid_client [OPTIONS] SERVICE
372
372
  grobid_client --input ~/documents --output ~/results processFulltextDocument
373
373
 
374
374
  # High concurrency with coordinates
375
- grobid_client --input ~/pdfs --output ~/tei --n 20 --teiCoordinates processFulltextDocument
375
+ grobid_client --input ~/pdfs --output ~/tei --n 20 --tei_coordinates processFulltextDocument
376
376
 
377
377
  # Process with JSON output
378
378
  grobid_client --input ~/pdfs --output ~/results --json processFulltextDocument
@@ -384,7 +384,7 @@ grobid_client --input ~/pdfs --output ~/results --markdown processFulltextDocume
384
384
  grobid_client --server https://grobid.example.com --input ~/citations.txt processCitationList
385
385
 
386
386
  # Force reprocessing with sentence segmentation and JSON output
387
- grobid_client --input ~/docs --force --segmentSentences --json processFulltextDocument
387
+ grobid_client --input ~/docs --force --segment_sentences --json processFulltextDocument
388
388
  ```
389
389
 
390
390
  ### Python Library
@@ -421,10 +421,10 @@ client.process(
421
421
  input_path="/path/to/pdfs",
422
422
  output_path="/path/to/output",
423
423
  n=10,
424
- generateIDs=True,
424
+ generate_ids=True,
425
425
  consolidate_header=True,
426
- teiCoordinates=True,
427
- segmentSentences=True
426
+ tei_coordinates=True,
427
+ segment_sentences=True
428
428
  )
429
429
 
430
430
  # Process with JSON output
@@ -517,6 +517,14 @@ settings.
517
517
  > Since version 0.0.12, the config file is optional. The client will use default localhost settings if no configuration
518
518
  > is provided.
519
519
 
520
+ > [!WARNING]
521
+ > **Citation consolidation and the `timeout` setting.** When `--consolidate_citations` (or `consolidate_citations=True`)
522
+ > is enabled, GROBID queries external services (e.g. CrossRef) to enrich the extracted references. This is considerably
523
+ > slower than a plain extraction, and a low `timeout` frequently causes `HTTP 408 (Request Timeout)` errors.
524
+ > Set the `timeout` to at least **120 seconds (2-3 minutes recommended)** when consolidating citations. The client emits
525
+ > a warning when consolidation is requested with a timeout below 120 seconds.
526
+ > See [issue #54](https://github.com/grobidOrg/grobid-client-python/issues/54).
527
+
520
528
  ### Logging Configuration
521
529
 
522
530
  The client provides configurable logging with different verbosity levels. By default, only essential statistics and warnings are shown.
@@ -673,7 +681,7 @@ When using the `--json` flag, the client converts TEI XML output to a structured
673
681
  grobid_client --input pdfs/ --output results/ --json processFulltextDocument
674
682
 
675
683
  # JSON output with coordinates and sentence segmentation
676
- grobid_client --input pdfs/ --output results/ --json --teiCoordinates --segmentSentences processFulltextDocument
684
+ grobid_client --input pdfs/ --output results/ --json --tei_coordinates --segment_sentences processFulltextDocument
677
685
  ```
678
686
 
679
687
  ```python
@@ -754,7 +762,7 @@ Competing interests statement...
754
762
  grobid_client --input pdfs/ --output results/ --markdown processFulltextDocument
755
763
 
756
764
  # Markdown output with coordinates and sentence segmentation
757
- grobid_client --input pdfs/ --output results/ --markdown --teiCoordinates --segmentSentences processFulltextDocument
765
+ grobid_client --input pdfs/ --output results/ --markdown --tei_coordinates --segment_sentences processFulltextDocument
758
766
  ```
759
767
 
760
768
  ```python
@@ -7,6 +7,7 @@ example.py
7
7
  pyproject.toml
8
8
  pytest.ini
9
9
  requirements.txt
10
+ .github/dependabot.yml
10
11
  .github/workflows/ci-build.yml
11
12
  .github/workflows/ci-release.yml
12
13
  grobid_client/__init__.py
@@ -24,6 +25,8 @@ grobid_client_python.egg-info/SOURCES.txt
24
25
  grobid_client_python.egg-info/dependency_links.txt
25
26
  grobid_client_python.egg-info/entry_points.txt
26
27
  grobid_client_python.egg-info/requires.txt
28
+ grobid_client_python.egg-info/scm_file_list.json
29
+ grobid_client_python.egg-info/scm_version.json
27
30
  grobid_client_python.egg-info/top_level.txt
28
31
  resources/20180928112135.png
29
32
  resources/20180928112138.csv
@@ -0,0 +1,63 @@
1
+ {
2
+ "files": [
3
+ ".gitignore",
4
+ "example.py",
5
+ "pytest.ini",
6
+ "Readme.md",
7
+ "requirements.txt",
8
+ "pyproject.toml",
9
+ "MANIFEST.in",
10
+ "config.json",
11
+ "LICENSE",
12
+ ".github/dependabot.yml",
13
+ ".github/workflows/ci-build.yml",
14
+ ".github/workflows/ci-release.yml",
15
+ "resources/20180928112135.png",
16
+ "resources/20180928112138.csv",
17
+ "resources/test_txt/reference-strings-6.txt",
18
+ "resources/test_txt/reference-strings-1.txt",
19
+ "resources/test_txt/reference-strings-5.txt",
20
+ "resources/test_txt/reference-strings-7.txt",
21
+ "resources/test_txt/reference-strings-3.txt",
22
+ "resources/test_txt/reference-strings-2.txt",
23
+ "resources/test_txt/reference-strings-4.txt",
24
+ "resources/test_pdf/mjb3wlzxcb2mc-migowebupload-1766042162782.grobid.tei.xml",
25
+ "resources/test_pdf/0046d83a-edd6-4631-b57c-755cdcce8b7f.pdf",
26
+ "resources/test_pdf/0046d83a-edd6-4631-b57c-755cdcce8b7f.tei.xml",
27
+ "grobid_client/client.py",
28
+ "grobid_client/grobid_client.py",
29
+ "grobid_client/__init__.py",
30
+ "grobid_client/format/validate_json_refs.py",
31
+ "grobid_client/format/TEI2LossyJSON_cli.py",
32
+ "grobid_client/format/TEI2LossyJSON.py",
33
+ "grobid_client/format/__main__.py",
34
+ "grobid_client/format/TEI2Markdown_cli.py",
35
+ "grobid_client/format/TEI2Markdown.py",
36
+ "grobid_client/format/__init__.py",
37
+ "tests/test_integration.py",
38
+ "tests/conftest.py",
39
+ "tests/run_tests.py",
40
+ "tests/test_config.json",
41
+ "tests/test_client.py",
42
+ "tests/test_conversions.py",
43
+ "tests/test_grobid_client.py",
44
+ "tests/__init__.py",
45
+ "tests/resources/article_withdrawn.grobid.tei.xml",
46
+ "tests/resources/0046d83a-edd6-4631-b57c-755cdcce8b7f.tei.xml",
47
+ "tests/resources/__init__.py",
48
+ "tests/resources/refs_offsets/10.1038_s41477-023-01501-1.json",
49
+ "tests/resources/refs_offsets/10.1371_journal.pone.0218311.json",
50
+ "tests/resources/refs_offsets/10.1038_s41477-023-01501-1.grobid.tei.xml",
51
+ "tests/resources/refs_offsets/10.1038_s41586-023-05895-y.grobid.tei.xml",
52
+ "tests/resources/refs_offsets/10.1038_s41598-023-32039-z.json",
53
+ "tests/resources/refs_offsets/10.7554_elife.78558.json",
54
+ "tests/resources/refs_offsets/10.1038_s41586-023-05895-y.json",
55
+ "tests/resources/refs_offsets/10.1038_s41598-023-32039-z.grobid.tei.xml",
56
+ "tests/resources/refs_offsets/2021.naacl-main.224.grobid.tei.xml",
57
+ "tests/resources/refs_offsets/10.1371_journal.pone.0218311.grobid.tei.xml",
58
+ "tests/resources/refs_offsets/10.7554_elife.78558.grobid.tei.xml",
59
+ "tests/resources/repeated_text/ijms-24-05988.grobid.tei.xml",
60
+ "tests/resources/repeated_text/ijms-24-05988.md",
61
+ "tests/resources/repeated_text/ijms-24-05988.json"
62
+ ]
63
+ }
@@ -0,0 +1,8 @@
1
+ {
2
+ "tag": "0.1.5",
3
+ "distance": 0,
4
+ "node": "g161e0f45189c8592b2e2c58e9638cc6218bc75fb",
5
+ "dirty": false,
6
+ "branch": "HEAD",
7
+ "node_date": "2026-07-23"
8
+ }
@@ -3,7 +3,7 @@ requires = ["setuptools", "setuptools-scm"]
3
3
  build-backend = "setuptools.build_meta"
4
4
 
5
5
  [tool.bumpversion]
6
- current_version = "0.1.4"
6
+ current_version = "0.1.5"
7
7
  commit = "true"
8
8
  tag = "true"
9
9
  tag_name = "v{new_version}"
@@ -284,7 +284,7 @@ class TestGrobidClient:
284
284
  result = client.process_pdf(
285
285
  'processFulltextDocument',
286
286
  '/test/document.pdf',
287
- generateIDs=True,
287
+ generate_ids=True,
288
288
  consolidate_header=True,
289
289
  consolidate_citations=False,
290
290
  include_raw_citations=False,
@@ -308,7 +308,7 @@ class TestGrobidClient:
308
308
  result = client.process_pdf(
309
309
  'processFulltextDocument',
310
310
  '/nonexistent/document.pdf',
311
- generateIDs=False,
311
+ generate_ids=False,
312
312
  consolidate_header=False,
313
313
  consolidate_citations=False,
314
314
  include_raw_citations=False,
@@ -335,7 +335,7 @@ class TestGrobidClient:
335
335
  result = client.process_txt(
336
336
  'processCitationList',
337
337
  '/test/references.txt',
338
- generateIDs=False,
338
+ generate_ids=False,
339
339
  consolidate_header=False,
340
340
  consolidate_citations=True,
341
341
  include_raw_citations=True,
@@ -371,7 +371,7 @@ class TestGrobidClient:
371
371
  result = client.process_pdf(
372
372
  'processFulltextDocument',
373
373
  '/test/document.pdf',
374
- generateIDs=False,
374
+ generate_ids=False,
375
375
  consolidate_header=False,
376
376
  consolidate_citations=False,
377
377
  include_raw_citations=False,
@@ -412,7 +412,7 @@ class TestGrobidClient:
412
412
  '/test',
413
413
  '/output',
414
414
  n=2,
415
- generateIDs=False,
415
+ generate_ids=False,
416
416
  consolidate_header=False,
417
417
  consolidate_citations=False,
418
418
  include_raw_citations=False,
@@ -493,7 +493,7 @@ class TestEdgeCases:
493
493
  input_path='/test',
494
494
  output='/output',
495
495
  n=1,
496
- generateIDs=False,
496
+ generate_ids=False,
497
497
  consolidate_header=False,
498
498
  consolidate_citations=False,
499
499
  include_raw_citations=False,
@@ -535,7 +535,7 @@ class TestEdgeCases:
535
535
  result = client.process_txt(
536
536
  'processCitationList',
537
537
  '/test/references.txt',
538
- generateIDs=False,
538
+ generate_ids=False,
539
539
  consolidate_header=False,
540
540
  consolidate_citations=False,
541
541
  include_raw_citations=False,
@@ -585,7 +585,7 @@ class TestEdgeCases:
585
585
  result = client.process_pdf(
586
586
  'processFulltextDocument',
587
587
  '/test/document.pdf',
588
- generateIDs=False,
588
+ generate_ids=False,
589
589
  consolidate_header=False,
590
590
  consolidate_citations=False,
591
591
  include_raw_citations=False,
@@ -600,7 +600,7 @@ class TestEdgeCases:
600
600
  result = client.process_pdf(
601
601
  'processFulltextDocument',
602
602
  '/test/document.pdf',
603
- generateIDs=False,
603
+ generate_ids=False,
604
604
  consolidate_header=False,
605
605
  consolidate_citations=False,
606
606
  include_raw_citations=False,
@@ -171,7 +171,7 @@ class TestGrobidClientIntegration:
171
171
  '/test',
172
172
  '/output',
173
173
  n=2,
174
- generateIDs=False,
174
+ generate_ids=False,
175
175
  consolidate_header=False,
176
176
  consolidate_citations=False,
177
177
  include_raw_citations=False,
@@ -332,7 +332,7 @@ class TestGrobidClientIntegration:
332
332
  '/test',
333
333
  '/output',
334
334
  n=5, # 5 concurrent threads
335
- generateIDs=False,
335
+ generate_ids=False,
336
336
  consolidate_header=False,
337
337
  consolidate_citations=False,
338
338
  include_raw_citations=False,