grobid-client-python 0.0.14__py3-none-any.whl → 0.0.16__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- grobid_client/grobid_client.py +132 -94
- {grobid_client_python-0.0.14.dist-info → grobid_client_python-0.0.16.dist-info}/METADATA +300 -141
- grobid_client_python-0.0.16.dist-info/RECORD +8 -0
- grobid_client_python-0.0.14.dist-info/RECORD +0 -8
- {grobid_client_python-0.0.14.dist-info → grobid_client_python-0.0.16.dist-info}/WHEEL +0 -0
- {grobid_client_python-0.0.14.dist-info → grobid_client_python-0.0.16.dist-info}/entry_points.txt +0 -0
- {grobid_client_python-0.0.14.dist-info → grobid_client_python-0.0.16.dist-info}/top_level.txt +0 -0
grobid_client/grobid_client.py
CHANGED
|
@@ -24,6 +24,7 @@ import requests
|
|
|
24
24
|
import pathlib
|
|
25
25
|
import logging
|
|
26
26
|
from typing import Tuple
|
|
27
|
+
import copy
|
|
27
28
|
|
|
28
29
|
from .client import ApiClient
|
|
29
30
|
|
|
@@ -37,53 +38,62 @@ class ServerUnavailableException(Exception):
|
|
|
37
38
|
|
|
38
39
|
|
|
39
40
|
class GrobidClient(ApiClient):
|
|
41
|
+
# Default configuration values
|
|
42
|
+
DEFAULT_CONFIG = {
|
|
43
|
+
'grobid_server': 'http://localhost:8070',
|
|
44
|
+
'batch_size': 1000,
|
|
45
|
+
'sleep_time': 5,
|
|
46
|
+
'timeout': 180,
|
|
47
|
+
'coordinates': [
|
|
48
|
+
"title",
|
|
49
|
+
"persName",
|
|
50
|
+
"affiliation",
|
|
51
|
+
"orgName",
|
|
52
|
+
"formula",
|
|
53
|
+
"figure",
|
|
54
|
+
"ref",
|
|
55
|
+
"biblStruct",
|
|
56
|
+
"head",
|
|
57
|
+
"p",
|
|
58
|
+
"s",
|
|
59
|
+
"note"
|
|
60
|
+
],
|
|
61
|
+
'logging': {
|
|
62
|
+
'level': 'INFO',
|
|
63
|
+
'format': '%(asctime)s - %(name)s - %(levelname)s - %(message)s',
|
|
64
|
+
'console': True,
|
|
65
|
+
'file': None, # Disabled by default
|
|
66
|
+
'max_file_size': '10MB',
|
|
67
|
+
'backup_count': 3
|
|
68
|
+
}
|
|
69
|
+
}
|
|
40
70
|
|
|
41
71
|
def __init__(
|
|
42
72
|
self,
|
|
43
|
-
grobid_server=
|
|
44
|
-
batch_size=
|
|
73
|
+
grobid_server=None,
|
|
74
|
+
batch_size=None,
|
|
45
75
|
coordinates=None,
|
|
46
|
-
sleep_time=
|
|
47
|
-
timeout=
|
|
76
|
+
sleep_time=None,
|
|
77
|
+
timeout=None,
|
|
48
78
|
config_path=None,
|
|
49
79
|
check_server=True
|
|
50
80
|
):
|
|
51
|
-
#
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
"biblStruct",
|
|
62
|
-
"head",
|
|
63
|
-
"p",
|
|
64
|
-
"s",
|
|
65
|
-
"note"
|
|
66
|
-
]
|
|
67
|
-
|
|
68
|
-
self.config = {
|
|
81
|
+
# Initialize config with defaults
|
|
82
|
+
self.config = copy.deepcopy(self.DEFAULT_CONFIG)
|
|
83
|
+
|
|
84
|
+
# Load config file (which may override current values)
|
|
85
|
+
if config_path:
|
|
86
|
+
self._load_config(config_path)
|
|
87
|
+
|
|
88
|
+
# Constructor parameters take precedence over config file values
|
|
89
|
+
# This ensures CLI arguments override config file values
|
|
90
|
+
self._set_config_params({
|
|
69
91
|
'grobid_server': grobid_server,
|
|
70
92
|
'batch_size': batch_size,
|
|
71
93
|
'coordinates': coordinates,
|
|
72
94
|
'sleep_time': sleep_time,
|
|
73
|
-
'timeout': timeout
|
|
74
|
-
|
|
75
|
-
'level': 'INFO',
|
|
76
|
-
'format': '%(asctime)s - %(name)s - %(levelname)s - %(message)s',
|
|
77
|
-
'console': True,
|
|
78
|
-
'file': None, # Disabled by default
|
|
79
|
-
'max_file_size': '10MB',
|
|
80
|
-
'backup_count': 3
|
|
81
|
-
}
|
|
82
|
-
}
|
|
83
|
-
|
|
84
|
-
# Load config first (which may override logging settings)
|
|
85
|
-
if config_path:
|
|
86
|
-
self._load_config(config_path)
|
|
95
|
+
'timeout': timeout
|
|
96
|
+
})
|
|
87
97
|
|
|
88
98
|
# Configure logging based on config
|
|
89
99
|
self._configure_logging()
|
|
@@ -91,6 +101,28 @@ class GrobidClient(ApiClient):
|
|
|
91
101
|
if check_server:
|
|
92
102
|
self._test_server_connection()
|
|
93
103
|
|
|
104
|
+
def _set_config_params(self, params):
|
|
105
|
+
"""Set configuration parameters, only if they are not None."""
|
|
106
|
+
for key, value in params.items():
|
|
107
|
+
if value is not None:
|
|
108
|
+
self.config[key] = value
|
|
109
|
+
|
|
110
|
+
def _handle_server_busy_retry(self, file_path, retry_func, *args, **kwargs):
|
|
111
|
+
"""Handle server busy (503) retry logic."""
|
|
112
|
+
self.logger.warning(f"Server busy (503), retrying {file_path} after {self.config['sleep_time']} seconds")
|
|
113
|
+
time.sleep(self.config["sleep_time"])
|
|
114
|
+
return retry_func(*args, **kwargs)
|
|
115
|
+
|
|
116
|
+
def _handle_request_error(self, file_path, error, error_type="Request"):
|
|
117
|
+
"""Handle request errors with consistent logging and return format."""
|
|
118
|
+
self.logger.error(f"{error_type} failed for {file_path}: {str(error)}")
|
|
119
|
+
return (file_path, 500, f"{error_type} failed: {str(error)}")
|
|
120
|
+
|
|
121
|
+
def _handle_unexpected_error(self, file_path, error):
|
|
122
|
+
"""Handle unexpected errors with consistent logging and return format."""
|
|
123
|
+
self.logger.error(f"Unexpected error processing {file_path}: {str(error)}")
|
|
124
|
+
return (file_path, 500, f"Unexpected error: {str(error)}")
|
|
125
|
+
|
|
94
126
|
def _configure_logging(self):
|
|
95
127
|
"""Configure logging based on the configuration settings."""
|
|
96
128
|
# Get logging config with defaults
|
|
@@ -479,56 +511,53 @@ class GrobidClient(ApiClient):
|
|
|
479
511
|
start=-1,
|
|
480
512
|
end=-1
|
|
481
513
|
):
|
|
514
|
+
pdf_handle = None
|
|
482
515
|
try:
|
|
483
516
|
pdf_handle = open(pdf_file, "rb")
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
{"Expires": "0"},
|
|
494
|
-
)
|
|
495
|
-
}
|
|
496
|
-
|
|
497
|
-
the_url = self.get_server_url(service)
|
|
517
|
+
|
|
518
|
+
files = {
|
|
519
|
+
"input": (
|
|
520
|
+
pdf_file,
|
|
521
|
+
pdf_handle,
|
|
522
|
+
"application/pdf",
|
|
523
|
+
{"Expires": "0"},
|
|
524
|
+
)
|
|
525
|
+
}
|
|
498
526
|
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
the_data
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
527
|
+
the_url = self.get_server_url(service)
|
|
528
|
+
|
|
529
|
+
# set the GROBID parameters
|
|
530
|
+
the_data = {}
|
|
531
|
+
if generateIDs:
|
|
532
|
+
the_data["generateIDs"] = "1"
|
|
533
|
+
if consolidate_header:
|
|
534
|
+
the_data["consolidateHeader"] = "1"
|
|
535
|
+
if consolidate_citations:
|
|
536
|
+
the_data["consolidateCitations"] = "1"
|
|
537
|
+
if include_raw_citations:
|
|
538
|
+
the_data["includeRawCitations"] = "1"
|
|
539
|
+
if include_raw_affiliations:
|
|
540
|
+
the_data["includeRawAffiliations"] = "1"
|
|
541
|
+
if tei_coordinates:
|
|
542
|
+
the_data["teiCoordinates"] = self.config["coordinates"]
|
|
543
|
+
if segment_sentences:
|
|
544
|
+
the_data["segmentSentences"] = "1"
|
|
545
|
+
if flavor:
|
|
546
|
+
the_data["flavor"] = flavor
|
|
547
|
+
if start and start > 0:
|
|
548
|
+
the_data["start"] = str(start)
|
|
549
|
+
if end and end > 0:
|
|
550
|
+
the_data["end"] = str(end)
|
|
521
551
|
|
|
522
|
-
try:
|
|
523
552
|
res, status = self.post(
|
|
524
553
|
url=the_url, files=files, data=the_data, headers={"Accept": "text/plain"},
|
|
525
554
|
timeout=self.config['timeout']
|
|
526
555
|
)
|
|
527
556
|
|
|
528
557
|
if status == 503:
|
|
529
|
-
self.
|
|
530
|
-
|
|
531
|
-
|
|
558
|
+
return self._handle_server_busy_retry(
|
|
559
|
+
pdf_file,
|
|
560
|
+
self.process_pdf,
|
|
532
561
|
service,
|
|
533
562
|
pdf_file,
|
|
534
563
|
generateIDs,
|
|
@@ -542,21 +571,22 @@ class GrobidClient(ApiClient):
|
|
|
542
571
|
start,
|
|
543
572
|
end
|
|
544
573
|
)
|
|
574
|
+
|
|
575
|
+
return (pdf_file, status, res.text)
|
|
576
|
+
|
|
577
|
+
except IOError as e:
|
|
578
|
+
self.logger.error(f"Failed to open PDF file {pdf_file}: {str(e)}")
|
|
579
|
+
return (pdf_file, 400, f"Failed to open file: {str(e)}")
|
|
545
580
|
except requests.exceptions.ReadTimeout as e:
|
|
546
581
|
self.logger.error(f"Request timeout for {pdf_file}: {str(e)}")
|
|
547
|
-
pdf_handle.close()
|
|
548
582
|
return (pdf_file, 408, f"Request timeout: {str(e)}")
|
|
549
583
|
except requests.exceptions.RequestException as e:
|
|
550
|
-
self.
|
|
551
|
-
pdf_handle.close()
|
|
552
|
-
return (pdf_file, 500, f"Request failed: {str(e)}")
|
|
584
|
+
return self._handle_request_error(pdf_file, e)
|
|
553
585
|
except Exception as e:
|
|
554
|
-
self.
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
pdf_handle.close()
|
|
559
|
-
return (pdf_file, status, res.text)
|
|
586
|
+
return self._handle_unexpected_error(pdf_file, e)
|
|
587
|
+
finally:
|
|
588
|
+
if pdf_handle:
|
|
589
|
+
pdf_handle.close()
|
|
560
590
|
|
|
561
591
|
def get_server_url(self, service):
|
|
562
592
|
return self.config['grobid_server'] + "/api/" + service
|
|
@@ -600,9 +630,9 @@ class GrobidClient(ApiClient):
|
|
|
600
630
|
)
|
|
601
631
|
|
|
602
632
|
if status == 503:
|
|
603
|
-
self.
|
|
604
|
-
|
|
605
|
-
|
|
633
|
+
return self._handle_server_busy_retry(
|
|
634
|
+
txt_file,
|
|
635
|
+
self.process_txt,
|
|
606
636
|
service,
|
|
607
637
|
txt_file,
|
|
608
638
|
generateIDs,
|
|
@@ -614,11 +644,9 @@ class GrobidClient(ApiClient):
|
|
|
614
644
|
segment_sentences
|
|
615
645
|
)
|
|
616
646
|
except requests.exceptions.RequestException as e:
|
|
617
|
-
self.
|
|
618
|
-
return (txt_file, 500, f"Request failed: {str(e)}")
|
|
647
|
+
return self._handle_request_error(txt_file, e)
|
|
619
648
|
except Exception as e:
|
|
620
|
-
self.
|
|
621
|
-
return (txt_file, 500, f"Unexpected error: {str(e)}")
|
|
649
|
+
return self._handle_unexpected_error(txt_file, e)
|
|
622
650
|
|
|
623
651
|
return (txt_file, status, res.text)
|
|
624
652
|
|
|
@@ -718,6 +746,11 @@ def main():
|
|
|
718
746
|
default=None,
|
|
719
747
|
help="Define the flavor to be used for the fulltext extraction",
|
|
720
748
|
)
|
|
749
|
+
parser.add_argument(
|
|
750
|
+
"--server",
|
|
751
|
+
default=None,
|
|
752
|
+
help="GROBID server URL override of the config file. If config not provided, default is http://localhost:8070",
|
|
753
|
+
)
|
|
721
754
|
|
|
722
755
|
args = parser.parse_args()
|
|
723
756
|
|
|
@@ -736,7 +769,12 @@ def main():
|
|
|
736
769
|
|
|
737
770
|
# Initialize GrobidClient which will configure logging based on config.json
|
|
738
771
|
try:
|
|
739
|
-
|
|
772
|
+
# Only pass grobid_server if it was explicitly provided (not the default)
|
|
773
|
+
client_kwargs = {'config_path': config_path}
|
|
774
|
+
if args.server is not None: # Only override if user specified a different server
|
|
775
|
+
client_kwargs['grobid_server'] = args.server
|
|
776
|
+
|
|
777
|
+
client = GrobidClient(**client_kwargs)
|
|
740
778
|
# Now use the client's logger for all subsequent logging
|
|
741
779
|
logger = client.logger
|
|
742
780
|
except ServerUnavailableException as e:
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: grobid-client-python
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.16
|
|
4
4
|
Summary: Simple python client for GROBID REST services
|
|
5
5
|
Author-email: Patrice Lopez <patrice.lopez@science-miner.com>
|
|
6
6
|
Maintainer-email: Patrice Lopez <patrice.lopez@science-miner.com>, Luca Foppiano <lucanoro@duck.com>
|
|
@@ -214,222 +214,381 @@ Project-URL: Changelog, https://github.com/kermitt2/grobid_client_python
|
|
|
214
214
|
Description-Content-Type: text/markdown
|
|
215
215
|
Requires-Dist: requests
|
|
216
216
|
|
|
217
|
+
# GROBID Client Python
|
|
218
|
+
|
|
217
219
|
[](https://badge.fury.io/py/grobid_client_python)
|
|
218
220
|
[](https://archive.softwareheritage.org/browse/origin/https://github.com/kermitt2/grobid_client_python/)
|
|
219
221
|
[](http://www.apache.org/licenses/LICENSE-2.0.html)
|
|
220
222
|
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
223
|
+
A simple, efficient Python client for [GROBID](https://github.com/kermitt2/grobid) REST services that provides concurrent processing capabilities for PDF documents, reference strings, and patents.
|
|
224
|
+
|
|
225
|
+
## 📋 Table of Contents
|
|
226
|
+
|
|
227
|
+
- [Features](#-features)
|
|
228
|
+
- [Prerequisites](#-prerequisites)
|
|
229
|
+
- [Installation](#-installation)
|
|
230
|
+
- [Quick Start](#-quick-start)
|
|
231
|
+
- [Usage](#-usage)
|
|
232
|
+
- [Command Line Interface](#command-line-interface)
|
|
233
|
+
- [Python Library](#python-library)
|
|
234
|
+
- [Configuration](#-configuration)
|
|
235
|
+
- [Services](#-services)
|
|
236
|
+
- [Testing](#-testing)
|
|
237
|
+
- [Performance](#-performance)
|
|
238
|
+
- [Development](#-development)
|
|
239
|
+
- [License](#-license)
|
|
240
|
+
|
|
241
|
+
## ✨ Features
|
|
242
|
+
|
|
243
|
+
- **Concurrent Processing**: Efficiently process multiple documents in parallel
|
|
244
|
+
- **Flexible Input**: Process PDF files, text files with references, and XML patents
|
|
245
|
+
- **Configurable**: Customizable server settings, timeouts, and processing options
|
|
246
|
+
- **Command Line & Library**: Use as a standalone CLI tool or import into your Python projects
|
|
247
|
+
- **Coordinate Extraction**: Optional PDF coordinate extraction for precise element positioning
|
|
248
|
+
- **Sentence Segmentation**: Layout-aware sentence segmentation capabilities
|
|
249
|
+
|
|
250
|
+
## 📋 Prerequisites
|
|
251
|
+
|
|
252
|
+
- **Python**: 3.8 - 3.13 (tested versions)
|
|
253
|
+
- **GROBID Server**: A running GROBID service instance
|
|
254
|
+
- Local installation: [GROBID Documentation](http://grobid.readthedocs.io/)
|
|
255
|
+
- Docker: `docker run -t --rm -p 8070:8070 lfoppiano/grobid:0.8.2`
|
|
256
|
+
- Default server: `http://localhost:8070`
|
|
257
|
+
- Online demo: https://lfoppiano-grobid.hf.space (usage limits apply)
|
|
224
258
|
|
|
225
259
|
> [!IMPORTANT]
|
|
226
|
-
>
|
|
260
|
+
> GROBID supports Windows only through Docker containers. See the [Docker documentation](https://grobid.readthedocs.io/en/latest/Grobid-docker/) for details.
|
|
227
261
|
|
|
228
|
-
##
|
|
262
|
+
## 🚀 Installation
|
|
229
263
|
|
|
230
|
-
|
|
231
|
-
By default, it is assumed that the server will run on the address `http://localhost:8070`.
|
|
232
|
-
You can change the server address by editing the file `config.json`, see below.
|
|
264
|
+
Choose one of the following installation methods:
|
|
233
265
|
|
|
234
|
-
|
|
266
|
+
### PyPI (Recommended)
|
|
267
|
+
```bash
|
|
268
|
+
pip install grobid-client-python
|
|
269
|
+
```
|
|
235
270
|
|
|
236
|
-
|
|
237
|
-
|
|
271
|
+
### Development Version
|
|
272
|
+
```bash
|
|
273
|
+
pip install git+https://github.com/kermitt2/grobid_client_python.git
|
|
274
|
+
```
|
|
238
275
|
|
|
239
|
-
|
|
276
|
+
### Local Development
|
|
277
|
+
```bash
|
|
278
|
+
git clone https://github.com/kermitt2/grobid_client_python
|
|
279
|
+
cd grobid_client_python
|
|
280
|
+
pip install -e .
|
|
281
|
+
```
|
|
240
282
|
|
|
241
|
-
|
|
283
|
+
## ⚡ Quick Start
|
|
242
284
|
|
|
243
|
-
|
|
285
|
+
### Command Line
|
|
286
|
+
```bash
|
|
287
|
+
# Process PDFs in a directory
|
|
288
|
+
grobid_client --input ./pdfs --output ./output processFulltextDocument
|
|
244
289
|
|
|
245
|
-
|
|
246
|
-
|
|
290
|
+
# Process with custom server
|
|
291
|
+
grobid_client --server https://your-grobid-server.com --input ./pdfs processFulltextDocument
|
|
247
292
|
```
|
|
248
293
|
|
|
249
|
-
|
|
294
|
+
### Python Library
|
|
295
|
+
```python
|
|
296
|
+
from grobid_client.grobid_client import GrobidClient
|
|
297
|
+
|
|
298
|
+
# Create client instance
|
|
299
|
+
client = GrobidClient(config_path="./config.json")
|
|
250
300
|
|
|
251
|
-
|
|
252
|
-
|
|
301
|
+
# Process documents
|
|
302
|
+
client.process("processFulltextDocument", "/path/to/pdfs", n=10)
|
|
253
303
|
```
|
|
254
304
|
|
|
255
|
-
|
|
305
|
+
## 📖 Usage
|
|
306
|
+
|
|
307
|
+
### Command Line Interface
|
|
256
308
|
|
|
309
|
+
The client provides a comprehensive CLI with the following syntax:
|
|
310
|
+
|
|
311
|
+
```bash
|
|
312
|
+
grobid_client [OPTIONS] SERVICE
|
|
257
313
|
```
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
314
|
+
|
|
315
|
+
#### Available Services
|
|
316
|
+
|
|
317
|
+
| Service | Description | Input Format |
|
|
318
|
+
|---------|-------------|--------------|
|
|
319
|
+
| `processFulltextDocument` | Extract full document structure | PDF files |
|
|
320
|
+
| `processHeaderDocument` | Extract document metadata | PDF files |
|
|
321
|
+
| `processReferences` | Extract bibliographic references | PDF files |
|
|
322
|
+
| `processCitationList` | Parse citation strings | Text files (one citation per line) |
|
|
323
|
+
| `processCitationPatentST36` | Process patent citations | XML ST36 format |
|
|
324
|
+
| `processCitationPatentPDF` | Process patent PDFs | PDF files |
|
|
325
|
+
|
|
326
|
+
#### Common Options
|
|
327
|
+
|
|
328
|
+
| Option | Description | Default |
|
|
329
|
+
|--------|-------------|---------|
|
|
330
|
+
| `--input` | Input directory path | Required |
|
|
331
|
+
| `--output` | Output directory path | Same as input |
|
|
332
|
+
| `--server` | GROBID server URL | `http://localhost:8070` |
|
|
333
|
+
| `--n` | Concurrency level | 10 |
|
|
334
|
+
| `--config` | Config file path | Optional |
|
|
335
|
+
| `--force` | Overwrite existing files | False |
|
|
336
|
+
| `--verbose` | Enable verbose logging | False |
|
|
337
|
+
|
|
338
|
+
#### Processing Options
|
|
339
|
+
|
|
340
|
+
| Option | Description |
|
|
341
|
+
|--------|-------------|
|
|
342
|
+
| `--generateIDs` | Generate random XML IDs |
|
|
343
|
+
| `--consolidate_header` | Consolidate header metadata |
|
|
344
|
+
| `--consolidate_citations` | Consolidate bibliographic references |
|
|
345
|
+
| `--include_raw_citations` | Include raw citation text |
|
|
346
|
+
| `--include_raw_affiliations` | Include raw affiliation text |
|
|
347
|
+
| `--teiCoordinates` | Add PDF coordinates to XML |
|
|
348
|
+
| `--segmentSentences` | Segment sentences with coordinates |
|
|
349
|
+
| `--flavor` | Processing flavor for fulltext extraction |
|
|
350
|
+
|
|
351
|
+
#### Examples
|
|
352
|
+
|
|
353
|
+
```bash
|
|
354
|
+
# Basic fulltext processing
|
|
355
|
+
grobid_client --input ~/documents --output ~/results processFulltextDocument
|
|
356
|
+
|
|
357
|
+
# High concurrency with coordinates
|
|
358
|
+
grobid_client --input ~/pdfs --output ~/tei --n 20 --teiCoordinates processFulltextDocument
|
|
359
|
+
|
|
360
|
+
# Process citations with custom server
|
|
361
|
+
grobid_client --server https://grobid.example.com --input ~/citations.txt processCitationList
|
|
362
|
+
|
|
363
|
+
# Force reprocessing with sentence segmentation
|
|
364
|
+
grobid_client --input ~/docs --force --segmentSentences processFulltextDocument
|
|
261
365
|
```
|
|
262
366
|
|
|
263
|
-
|
|
367
|
+
### Python Library
|
|
264
368
|
|
|
265
|
-
|
|
369
|
+
#### Basic Usage
|
|
266
370
|
|
|
267
|
-
|
|
371
|
+
```python
|
|
372
|
+
from grobid_client.grobid_client import GrobidClient
|
|
268
373
|
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
[--n N] [--generateIDs] [--consolidate_header]
|
|
272
|
-
[--consolidate_citations] [--include_raw_citations]
|
|
273
|
-
[--include_raw_affiliations] [--force] [--teiCoordinates]
|
|
274
|
-
[--verbose] [--flavor FLAVOR]
|
|
275
|
-
service
|
|
276
|
-
|
|
277
|
-
Client for GROBID services
|
|
278
|
-
|
|
279
|
-
positional arguments:
|
|
280
|
-
service one of ['processFulltextDocument',
|
|
281
|
-
'processHeaderDocument', 'processReferences',
|
|
282
|
-
'processCitationList','processCitationPatentST36',
|
|
283
|
-
'processCitationPatentPDF']
|
|
284
|
-
|
|
285
|
-
optional arguments:
|
|
286
|
-
-h, --help show this help message and exit
|
|
287
|
-
--input INPUT path to the directory containing PDF files or .txt
|
|
288
|
-
(for processCitationList only, one reference per line)
|
|
289
|
-
to process
|
|
290
|
-
--output OUTPUT path to the directory where to put the results
|
|
291
|
-
(optional)
|
|
292
|
-
--config CONFIG path to the config file, default is ./config.json
|
|
293
|
-
--n N concurrency for service usage
|
|
294
|
-
--generateIDs generate random xml:id to textual XML elements of the
|
|
295
|
-
result files
|
|
296
|
-
--consolidate_header call GROBID with consolidation of the metadata
|
|
297
|
-
extracted from the header
|
|
298
|
-
--consolidate_citations
|
|
299
|
-
call GROBID with consolidation of the extracted
|
|
300
|
-
bibliographical references
|
|
301
|
-
--include_raw_citations
|
|
302
|
-
call GROBID requesting the extraction of raw citations
|
|
303
|
-
--include_raw_affiliations
|
|
304
|
-
call GROBID requestiong the extraciton of raw
|
|
305
|
-
affiliations
|
|
306
|
-
--force force re-processing pdf input files when tei output
|
|
307
|
-
files already exist
|
|
308
|
-
--teiCoordinates add the original PDF coordinates (bounding boxes) to
|
|
309
|
-
the extracted elements
|
|
310
|
-
--segmentSentences segment sentences in the text content of the document
|
|
311
|
-
with additional <s> elements
|
|
312
|
-
--verbose print information about processed files in the console
|
|
313
|
-
--flavor specifcy the flavor to be used. Current accepted
|
|
314
|
-
values: `article/light`, `article/light-ref`, `sdo/ietf`.
|
|
315
|
-
```
|
|
374
|
+
# Initialize with default localhost server
|
|
375
|
+
client = GrobidClient()
|
|
316
376
|
|
|
317
|
-
|
|
318
|
-
|
|
377
|
+
# Initialize with custom server
|
|
378
|
+
client = GrobidClient(grobid_server="https://your-server.com")
|
|
379
|
+
|
|
380
|
+
# Initialize with config file
|
|
381
|
+
client = GrobidClient(config_path="./config.json")
|
|
319
382
|
|
|
383
|
+
# Process documents
|
|
384
|
+
client.process(
|
|
385
|
+
service="processFulltextDocument",
|
|
386
|
+
input_path="/path/to/pdfs",
|
|
387
|
+
output_path="/path/to/output",
|
|
388
|
+
n=20
|
|
389
|
+
)
|
|
390
|
+
```
|
|
320
391
|
|
|
321
|
-
|
|
392
|
+
#### Advanced Usage
|
|
322
393
|
|
|
323
|
-
```
|
|
324
|
-
|
|
394
|
+
```python
|
|
395
|
+
# Process with specific options
|
|
396
|
+
client.process(
|
|
397
|
+
service="processFulltextDocument",
|
|
398
|
+
input_path="/path/to/pdfs",
|
|
399
|
+
output_path="/path/to/output",
|
|
400
|
+
n=10,
|
|
401
|
+
generateIDs=True,
|
|
402
|
+
consolidate_header=True,
|
|
403
|
+
teiCoordinates=True,
|
|
404
|
+
segmentSentences=True
|
|
405
|
+
)
|
|
406
|
+
|
|
407
|
+
# Process citation lists
|
|
408
|
+
client.process(
|
|
409
|
+
service="processCitationList",
|
|
410
|
+
input_path="/path/to/citations.txt",
|
|
411
|
+
output_path="/path/to/output"
|
|
412
|
+
)
|
|
325
413
|
```
|
|
326
414
|
|
|
327
|
-
|
|
415
|
+
## ⚙️ Configuration
|
|
416
|
+
|
|
417
|
+
Configuration can be provided via a JSON file. When using the CLI, the `--server` argument overrides the config file settings.
|
|
328
418
|
|
|
329
|
-
|
|
419
|
+
### Default Configuration
|
|
330
420
|
|
|
331
|
-
```
|
|
332
|
-
|
|
421
|
+
```json
|
|
422
|
+
{
|
|
423
|
+
"grobid_server": "http://localhost:8070",
|
|
424
|
+
"batch_size": 1000,
|
|
425
|
+
"sleep_time": 5,
|
|
426
|
+
"timeout": 60,
|
|
427
|
+
"coordinates": ["persName", "figure", "ref", "biblStruct", "formula", "s"]
|
|
428
|
+
}
|
|
333
429
|
```
|
|
334
430
|
|
|
335
|
-
|
|
431
|
+
### Configuration Parameters
|
|
336
432
|
|
|
337
|
-
|
|
433
|
+
| Parameter | Description | Default |
|
|
434
|
+
|-----------|-------------|---------|
|
|
435
|
+
| `grobid_server` | GROBID server URL | `http://localhost:8070` |
|
|
436
|
+
| `batch_size` | Thread pool size | 1000 |
|
|
437
|
+
| `sleep_time` | Wait time when server is busy (seconds) | 5 |
|
|
438
|
+
| `timeout` | Client-side timeout (seconds) | 60 |
|
|
439
|
+
| `coordinates` | XML elements for coordinate extraction | See above |
|
|
338
440
|
|
|
339
|
-
|
|
441
|
+
> [!TIP]
|
|
442
|
+
> Since version 0.0.12, the config file is optional. The client will use default localhost settings if no configuration is provided.
|
|
340
443
|
|
|
341
|
-
|
|
342
|
-
> grobid_client --input resources/test_txt/ --output resources/test_out/ --n 20 processCitationList
|
|
343
|
-
```
|
|
444
|
+
## 🔬 Services
|
|
344
445
|
|
|
345
|
-
|
|
446
|
+
### Fulltext Document Processing
|
|
447
|
+
Extracts complete document structure including headers, body text, figures, tables, and references.
|
|
346
448
|
|
|
347
|
-
```
|
|
348
|
-
|
|
449
|
+
```bash
|
|
450
|
+
grobid_client --input pdfs/ --output results/ processFulltextDocument
|
|
349
451
|
```
|
|
350
452
|
|
|
351
|
-
|
|
453
|
+
### Header Document Processing
|
|
454
|
+
Extracts only document metadata (title, authors, abstract, etc.).
|
|
352
455
|
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
456
|
+
```bash
|
|
457
|
+
grobid_client --input pdfs/ --output headers/ processHeaderDocument
|
|
458
|
+
```
|
|
356
459
|
|
|
357
|
-
|
|
358
|
-
from
|
|
460
|
+
### Reference Processing
|
|
461
|
+
Extracts and structures bibliographic references from documents.
|
|
359
462
|
|
|
360
|
-
|
|
361
|
-
|
|
463
|
+
```bash
|
|
464
|
+
grobid_client --input pdfs/ --output refs/ processReferences
|
|
362
465
|
```
|
|
363
466
|
|
|
364
|
-
|
|
467
|
+
### Citation List Processing
|
|
468
|
+
Parses raw citation strings from text files.
|
|
365
469
|
|
|
366
|
-
|
|
470
|
+
```bash
|
|
471
|
+
grobid_client --input citations.txt --output parsed/ processCitationList
|
|
472
|
+
```
|
|
367
473
|
|
|
368
474
|
> [!TIP]
|
|
369
|
-
>
|
|
475
|
+
> For citation lists, input should be text files with one citation string per line.
|
|
370
476
|
|
|
371
|
-
|
|
477
|
+
## 🧪 Testing
|
|
372
478
|
|
|
373
|
-
|
|
479
|
+
The project includes comprehensive unit and integration tests using pytest.
|
|
374
480
|
|
|
375
|
-
|
|
481
|
+
### Running Tests
|
|
376
482
|
|
|
377
|
-
|
|
483
|
+
```bash
|
|
484
|
+
# Install development dependencies
|
|
485
|
+
pip install -e .[dev]
|
|
378
486
|
|
|
379
|
-
|
|
487
|
+
# Run all tests
|
|
488
|
+
pytest
|
|
380
489
|
|
|
381
|
-
|
|
490
|
+
# Run with coverage
|
|
491
|
+
pytest --cov=grobid_client
|
|
382
492
|
|
|
383
|
-
|
|
493
|
+
# Run specific test file
|
|
494
|
+
pytest tests/test_client.py
|
|
384
495
|
|
|
496
|
+
# Run with verbose output
|
|
497
|
+
pytest -v
|
|
385
498
|
```
|
|
386
|
-
{
|
|
387
|
-
"grobid_server": "http://localhost:8070",
|
|
388
|
-
"batch_size": 1000,
|
|
389
|
-
"sleep_time": 5,
|
|
390
|
-
"timeout": 60,
|
|
391
|
-
"coordinates": [ "persName", "figure", "ref", "biblStruct", "formula", "s" ]
|
|
392
|
-
}
|
|
393
|
-
```
|
|
394
499
|
|
|
395
|
-
|
|
500
|
+
### Test Structure
|
|
501
|
+
|
|
502
|
+
- `tests/test_client.py` - Unit tests for the base API client
|
|
503
|
+
- `tests/test_grobid_client.py` - Unit tests for the GROBID client
|
|
504
|
+
- `tests/test_integration.py` - Integration tests with real GROBID server
|
|
505
|
+
- `tests/conftest.py` - Test configuration and fixtures
|
|
396
506
|
|
|
397
|
-
|
|
507
|
+
### Continuous Integration
|
|
398
508
|
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
509
|
+
Tests are automatically run via GitHub Actions on:
|
|
510
|
+
- Push to main branch
|
|
511
|
+
- Pull requests
|
|
512
|
+
- Multiple Python versions (3.8-3.13)
|
|
513
|
+
|
|
514
|
+
## 📊 Performance
|
|
515
|
+
|
|
516
|
+
Benchmark results for processing **136 PDFs** (3,443 pages total, ~25 pages per PDF) on Intel Core i7-4790K CPU 4.00GHz:
|
|
517
|
+
|
|
518
|
+
| Concurrency | Runtime (s) | s/PDF | PDF/s |
|
|
519
|
+
|-------------|-------------|-------|-------|
|
|
520
|
+
| 1 | 209.0 | 1.54 | 0.65 |
|
|
521
|
+
| 2 | 112.0 | 0.82 | 1.21 |
|
|
522
|
+
| 3 | 80.4 | 0.59 | 1.69 |
|
|
523
|
+
| 5 | 62.9 | 0.46 | 2.16 |
|
|
524
|
+
| 8 | 55.7 | 0.41 | 2.44 |
|
|
525
|
+
| 10 | 55.3 | 0.40 | 2.45 |
|
|
407
526
|
|
|
408
527
|

|
|
409
528
|
|
|
410
|
-
|
|
529
|
+
### Additional Benchmarks
|
|
530
|
+
|
|
531
|
+
- **Header processing**: 3.74s for 136 PDFs (36 PDF/s) with n=10
|
|
532
|
+
- **Reference extraction**: 26.9s for 136 PDFs (5.1 PDF/s) with n=10
|
|
533
|
+
- **Citation parsing**: 4.3s for 3,500 citations (814 citations/s) with n=10
|
|
411
534
|
|
|
412
|
-
|
|
535
|
+
## 🛠️ Development
|
|
413
536
|
|
|
414
|
-
|
|
537
|
+
### Setting Up Development Environment
|
|
415
538
|
|
|
539
|
+
```bash
|
|
540
|
+
# Clone the repository
|
|
541
|
+
git clone https://github.com/kermitt2/grobid_client_python
|
|
542
|
+
cd grobid_client_python
|
|
416
543
|
|
|
417
|
-
|
|
544
|
+
# Create virtual environment
|
|
545
|
+
python -m venv venv
|
|
546
|
+
source venv/bin/activate # On Windows: venv\Scripts\activate
|
|
418
547
|
|
|
419
|
-
|
|
548
|
+
# Install in development mode with test dependencies
|
|
549
|
+
pip install -e .[dev]
|
|
550
|
+
|
|
551
|
+
# Install pre-commit hooks (optional)
|
|
552
|
+
pre-commit install
|
|
553
|
+
```
|
|
420
554
|
|
|
421
|
-
|
|
555
|
+
### Creating a New Release
|
|
422
556
|
|
|
423
|
-
|
|
557
|
+
The project uses `bump-my-version` for version management:
|
|
558
|
+
|
|
559
|
+
```bash
|
|
560
|
+
# Install bump-my-version
|
|
424
561
|
pip install bump-my-version
|
|
425
|
-
|
|
562
|
+
|
|
563
|
+
# Bump version (patch, minor, or major)
|
|
564
|
+
bump-my-version bump patch
|
|
565
|
+
|
|
566
|
+
# The release will be automatically published to PyPI
|
|
426
567
|
```
|
|
427
568
|
|
|
428
|
-
|
|
429
|
-
|
|
569
|
+
### Contributing
|
|
570
|
+
|
|
571
|
+
1. Fork the repository
|
|
572
|
+
2. Create a feature branch (`git checkout -b feature/amazing-feature`)
|
|
573
|
+
3. Make your changes
|
|
574
|
+
4. Add tests for new functionality
|
|
575
|
+
5. Run the test suite (`pytest`)
|
|
576
|
+
6. Commit your changes (`git commit -m 'Add amazing feature'`)
|
|
577
|
+
7. Push to the branch (`git push origin feature/amazing-feature`)
|
|
578
|
+
8. Open a Pull Request
|
|
579
|
+
|
|
580
|
+
## 📄 License
|
|
581
|
+
|
|
582
|
+
Distributed under the [Apache 2.0 License](http://www.apache.org/licenses/LICENSE-2.0). See `LICENSE` for more information.
|
|
583
|
+
|
|
584
|
+
## 👥 Authors & Contact
|
|
430
585
|
|
|
431
|
-
|
|
586
|
+
**Main Author**: Patrice Lopez (patrice.lopez@science-miner.com)
|
|
587
|
+
**Maintainer**: Luca Foppiano (luca@sciencialab.com)
|
|
432
588
|
|
|
433
|
-
|
|
589
|
+
## 🔗 Links
|
|
434
590
|
|
|
435
|
-
|
|
591
|
+
- [GROBID Documentation](https://grobid.readthedocs.io/)
|
|
592
|
+
- [PyPI Package](https://pypi.org/project/grobid-client-python/)
|
|
593
|
+
- [GitHub Repository](https://github.com/kermitt2/grobid_client_python)
|
|
594
|
+
- [Issue Tracker](https://github.com/kermitt2/grobid_client_python/issues)
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
grobid_client/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
2
|
+
grobid_client/client.py,sha256=crvUiVnnkn6tFwO4lKQg2cTxAHZxCBAmeNSVtuL4ZA0,6009
|
|
3
|
+
grobid_client/grobid_client.py,sha256=3XlEXrg8v0TWeXWE1jhcYicrLOu5BOWE7l5FBO7YvjE,31063
|
|
4
|
+
grobid_client_python-0.0.16.dist-info/METADATA,sha256=H2mnjLSnP8xjj5NN_6iONKghxMmPTfGZ-EKoDKWjNcY,24633
|
|
5
|
+
grobid_client_python-0.0.16.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
6
|
+
grobid_client_python-0.0.16.dist-info/entry_points.txt,sha256=-WwPi8D8eXEbDB6QvitvHjxSGBeyLCyFvAoZcaDGswQ,67
|
|
7
|
+
grobid_client_python-0.0.16.dist-info/top_level.txt,sha256=HnjgWxdJNKTLIN86XkgRAfVeEK4C71fc9X4fe7qdMvA,14
|
|
8
|
+
grobid_client_python-0.0.16.dist-info/RECORD,,
|
|
@@ -1,8 +0,0 @@
|
|
|
1
|
-
grobid_client/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
2
|
-
grobid_client/client.py,sha256=crvUiVnnkn6tFwO4lKQg2cTxAHZxCBAmeNSVtuL4ZA0,6009
|
|
3
|
-
grobid_client/grobid_client.py,sha256=1ZHOhlPEgPDExK8RCYpZTq4M0kfMRVgrKEEpCCqUlZg,29533
|
|
4
|
-
grobid_client_python-0.0.14.dist-info/METADATA,sha256=3YobdYYLxOSqu_Rjb_W350xzSff8N-5Pk0Y2eoPNb4M,24213
|
|
5
|
-
grobid_client_python-0.0.14.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
6
|
-
grobid_client_python-0.0.14.dist-info/entry_points.txt,sha256=-WwPi8D8eXEbDB6QvitvHjxSGBeyLCyFvAoZcaDGswQ,67
|
|
7
|
-
grobid_client_python-0.0.14.dist-info/top_level.txt,sha256=HnjgWxdJNKTLIN86XkgRAfVeEK4C71fc9X4fe7qdMvA,14
|
|
8
|
-
grobid_client_python-0.0.14.dist-info/RECORD,,
|
|
File without changes
|
{grobid_client_python-0.0.14.dist-info → grobid_client_python-0.0.16.dist-info}/entry_points.txt
RENAMED
|
File without changes
|
{grobid_client_python-0.0.14.dist-info → grobid_client_python-0.0.16.dist-info}/top_level.txt
RENAMED
|
File without changes
|