grobid-client-python 0.0.16__tar.gz → 0.0.18__tar.gz

{grobid_client_python-0.0.16 → grobid_client_python-0.0.18}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: grobid-client-python
-Version: 0.0.16
+Version: 0.0.18
 Summary: Simple python client for GROBID REST services
 Author-email: Patrice Lopez <patrice.lopez@science-miner.com>
 Maintainer-email: Patrice Lopez <patrice.lopez@science-miner.com>, Luca Foppiano <lucanoro@duck.com>
@@ -254,7 +254,7 @@ A simple, efficient Python client for [GROBID](https://github.com/kermitt2/grobi
   - Local installation: [GROBID Documentation](http://grobid.readthedocs.io/)
   - Docker: `docker run -t --rm -p 8070:8070 lfoppiano/grobid:0.8.2`
   - Default server: `http://localhost:8070`
-  - Online demo: https://lfoppiano-grobid.hf.space (usage limits apply)
+  - Online demo: https://lfoppiano-grobid.hf.space (usage limits apply), more details [here](https://grobid.readthedocs.io/en/latest/getting_started/#using-grobid-from-the-cloud).
 
 > [!IMPORTANT]
 > GROBID supports Windows only through Docker containers. See the [Docker documentation](https://grobid.readthedocs.io/en/latest/Grobid-docker/) for details.
@@ -314,39 +314,39 @@ grobid_client [OPTIONS] SERVICE
 
 #### Available Services
 
-| Service | Description | Input Format |
-|---------|-------------|--------------|
-| `processFulltextDocument` | Extract full document structure | PDF files |
-| `processHeaderDocument` | Extract document metadata | PDF files |
-| `processReferences` | Extract bibliographic references | PDF files |
-| `processCitationList` | Parse citation strings | Text files (one citation per line) |
-| `processCitationPatentST36` | Process patent citations | XML ST36 format |
-| `processCitationPatentPDF` | Process patent PDFs | PDF files |
+| Service                     | Description                       | Input Format                       |
+|-----------------------------|-----------------------------------|------------------------------------|
+| `processFulltextDocument`   | Extract full document structure   | PDF files                          |
+| `processHeaderDocument`     | Extract document metadata         | PDF files                          |
+| `processReferences`         | Extract bibliographic references  | PDF files                          |
+| `processCitationList`       | Parse citation strings            | Text files (one citation per line) |
+| `processCitationPatentST36` | Process patent citations          | XML ST36 format                    |
+| `processCitationPatentPDF`  | Process patent PDFs               | PDF files                          |
 
 #### Common Options
 
-| Option | Description | Default |
-|--------|-------------|---------|
-| `--input` | Input directory path | Required |
-| `--output` | Output directory path | Same as input |
-| `--server` | GROBID server URL | `http://localhost:8070` |
-| `--n` | Concurrency level | 10 |
-| `--config` | Config file path | Optional |
-| `--force` | Overwrite existing files | False |
-| `--verbose` | Enable verbose logging | False |
+| Option      | Description              | Default                 |
+|-------------|--------------------------|-------------------------|
+| `--input`   | Input directory path     | Required                |
+| `--output`  | Output directory path    | Same as input           |
+| `--server`  | GROBID server URL        | `http://localhost:8070` |
+| `--n`       | Concurrency level        | 10                      |
+| `--config`  | Config file path         | Optional                |
+| `--force`   | Overwrite existing files | False                   |
+| `--verbose` | Enable verbose logging   | False                   |
 
 #### Processing Options
 
-| Option | Description |
-|--------|-------------|
-| `--generateIDs` | Generate random XML IDs |
-| `--consolidate_header` | Consolidate header metadata |
-| `--consolidate_citations` | Consolidate bibliographic references |
-| `--include_raw_citations` | Include raw citation text |
-| `--include_raw_affiliations` | Include raw affiliation text |
-| `--teiCoordinates` | Add PDF coordinates to XML |
-| `--segmentSentences` | Segment sentences with coordinates |
-| `--flavor` | Processing flavor for fulltext extraction |
+| Option                       | Description                               |
+|------------------------------|-------------------------------------------|
+| `--generateIDs`              | Generate random XML IDs                   |
+| `--consolidate_header`       | Consolidate header metadata               |
+| `--consolidate_citations`    | Consolidate bibliographic references      |
+| `--include_raw_citations`    | Include raw citation text                 |
+| `--include_raw_affiliations` | Include raw affiliation text              |
+| `--teiCoordinates`           | Add PDF coordinates to XML                |
+| `--segmentSentences`         | Segment sentences with coordinates        |
+| `--flavor`                   | Processing flavor for fulltext extraction |
 
 #### Examples
 
@@ -430,13 +430,13 @@ Configuration can be provided via a JSON file. When using the CLI, the `--server
 
 ### Configuration Parameters
 
-| Parameter | Description | Default |
-|-----------|-------------|---------|
-| `grobid_server` | GROBID server URL | `http://localhost:8070` |
-| `batch_size` | Thread pool size | 1000 |
-| `sleep_time` | Wait time when server is busy (seconds) | 5 |
-| `timeout` | Client-side timeout (seconds) | 60 |
-| `coordinates` | XML elements for coordinate extraction | See above |
+| Parameter       | Description                                                                                                      | Default                 |
+|-----------------|------------------------------------------------------------------------------------------------------------------|-------------------------|
+| `grobid_server` | GROBID server URL                                                                                                | `http://localhost:8070` |
+| `batch_size`    | Thread pool size. **Tune carefully: a large batch size will result in the data being written less frequently**   | 1000                    |
+| `sleep_time`    | Wait time when server is busy (seconds)                                                                          | 5                       |
+| `timeout`       | Client-side timeout (seconds)                                                                                    | 180                     |
+| `coordinates`   | XML elements for coordinate extraction                                                                           | See above               |
 
 > [!TIP]
 > Since version 0.0.12, the config file is optional. The client will use default localhost settings if no configuration is provided.

{grobid_client_python-0.0.16 → grobid_client_python-0.0.18}/Readme.md

@@ -38,7 +38,7 @@ A simple, efficient Python client for [GROBID](https://github.com/kermitt2/grobi
   - Local installation: [GROBID Documentation](http://grobid.readthedocs.io/)
   - Docker: `docker run -t --rm -p 8070:8070 lfoppiano/grobid:0.8.2`
   - Default server: `http://localhost:8070`
-  - Online demo: https://lfoppiano-grobid.hf.space (usage limits apply)
+  - Online demo: https://lfoppiano-grobid.hf.space (usage limits apply), more details [here](https://grobid.readthedocs.io/en/latest/getting_started/#using-grobid-from-the-cloud).
 
 > [!IMPORTANT]
 > GROBID supports Windows only through Docker containers. See the [Docker documentation](https://grobid.readthedocs.io/en/latest/Grobid-docker/) for details.
@@ -98,39 +98,39 @@ grobid_client [OPTIONS] SERVICE
 
 #### Available Services
 
-| Service | Description | Input Format |
-|---------|-------------|--------------|
-| `processFulltextDocument` | Extract full document structure | PDF files |
-| `processHeaderDocument` | Extract document metadata | PDF files |
-| `processReferences` | Extract bibliographic references | PDF files |
-| `processCitationList` | Parse citation strings | Text files (one citation per line) |
-| `processCitationPatentST36` | Process patent citations | XML ST36 format |
-| `processCitationPatentPDF` | Process patent PDFs | PDF files |
+| Service                     | Description                       | Input Format                       |
+|-----------------------------|-----------------------------------|------------------------------------|
+| `processFulltextDocument`   | Extract full document structure   | PDF files                          |
+| `processHeaderDocument`     | Extract document metadata         | PDF files                          |
+| `processReferences`         | Extract bibliographic references  | PDF files                          |
+| `processCitationList`       | Parse citation strings            | Text files (one citation per line) |
+| `processCitationPatentST36` | Process patent citations          | XML ST36 format                    |
+| `processCitationPatentPDF`  | Process patent PDFs               | PDF files                          |
 
 #### Common Options
 
-| Option | Description | Default |
-|--------|-------------|---------|
-| `--input` | Input directory path | Required |
-| `--output` | Output directory path | Same as input |
-| `--server` | GROBID server URL | `http://localhost:8070` |
-| `--n` | Concurrency level | 10 |
-| `--config` | Config file path | Optional |
-| `--force` | Overwrite existing files | False |
-| `--verbose` | Enable verbose logging | False |
+| Option      | Description              | Default                 |
+|-------------|--------------------------|-------------------------|
+| `--input`   | Input directory path     | Required                |
+| `--output`  | Output directory path    | Same as input           |
+| `--server`  | GROBID server URL        | `http://localhost:8070` |
+| `--n`       | Concurrency level        | 10                      |
+| `--config`  | Config file path         | Optional                |
+| `--force`   | Overwrite existing files | False                   |
+| `--verbose` | Enable verbose logging   | False                   |
 
 #### Processing Options
 
-| Option | Description |
-|--------|-------------|
-| `--generateIDs` | Generate random XML IDs |
-| `--consolidate_header` | Consolidate header metadata |
-| `--consolidate_citations` | Consolidate bibliographic references |
-| `--include_raw_citations` | Include raw citation text |
-| `--include_raw_affiliations` | Include raw affiliation text |
-| `--teiCoordinates` | Add PDF coordinates to XML |
-| `--segmentSentences` | Segment sentences with coordinates |
-| `--flavor` | Processing flavor for fulltext extraction |
+| Option                       | Description                               |
+|------------------------------|-------------------------------------------|
+| `--generateIDs`              | Generate random XML IDs                   |
+| `--consolidate_header`       | Consolidate header metadata               |
+| `--consolidate_citations`    | Consolidate bibliographic references      |
+| `--include_raw_citations`    | Include raw citation text                 |
+| `--include_raw_affiliations` | Include raw affiliation text              |
+| `--teiCoordinates`           | Add PDF coordinates to XML                |
+| `--segmentSentences`         | Segment sentences with coordinates        |
+| `--flavor`                   | Processing flavor for fulltext extraction |
 
 #### Examples
 
@@ -214,13 +214,13 @@ Configuration can be provided via a JSON file. When using the CLI, the `--server
 
 ### Configuration Parameters
 
-| Parameter | Description | Default |
-|-----------|-------------|---------|
-| `grobid_server` | GROBID server URL | `http://localhost:8070` |
-| `batch_size` | Thread pool size | 1000 |
-| `sleep_time` | Wait time when server is busy (seconds) | 5 |
-| `timeout` | Client-side timeout (seconds) | 60 |
-| `coordinates` | XML elements for coordinate extraction | See above |
+| Parameter       | Description                                                                                                      | Default                 |
+|-----------------|------------------------------------------------------------------------------------------------------------------|-------------------------|
+| `grobid_server` | GROBID server URL                                                                                                | `http://localhost:8070` |
+| `batch_size`    | Thread pool size. **Tune carefully: a large batch size will result in the data being written less frequently**   | 1000                    |
+| `sleep_time`    | Wait time when server is busy (seconds)                                                                          | 5                       |
+| `timeout`       | Client-side timeout (seconds)                                                                                    | 180                     |
+| `coordinates`   | XML elements for coordinate extraction                                                                           | See above               |
 
 > [!TIP]
 > Since version 0.0.12, the config file is optional. The client will use default localhost settings if no configuration is provided.

{grobid_client_python-0.0.16 → grobid_client_python-0.0.18}/config.json

@@ -19,7 +19,7 @@
   ],
   "logging": {
     "level": "INFO",
-    "format": "%(asctime)s - %(name)s - %(levelname)s - %(message)s",
+    "format": "%(asctime)s - %(levelname)s - %(message)s",
     "console": true,
     "file": null,
     "max_file_size": "10MB",

{grobid_client_python-0.0.16 → grobid_client_python-0.0.18}/grobid_client/grobid_client.py

@@ -41,7 +41,7 @@ class GrobidClient(ApiClient):
     # Default configuration values
     DEFAULT_CONFIG = {
         'grobid_server': 'http://localhost:8070',
-        'batch_size': 1000,
+        'batch_size': 10,
         'sleep_time': 5,
         'timeout': 180,
         'coordinates': [
@@ -60,7 +60,7 @@ class GrobidClient(ApiClient):
         ],
         'logging': {
             'level': 'INFO',
-            'format': '%(asctime)s - %(name)s - %(levelname)s - %(message)s',
+            'format': '%(asctime)s - %(levelname)s - %(message)s',
             'console': True,
             'file': None,  # Disabled by default
             'max_file_size': '10MB',
@@ -133,7 +133,7 @@ class GrobidClient(ApiClient):
         log_level = getattr(logging, log_level_str, logging.INFO)
 
         # Parse log format
-        log_format = log_config.get('format', '%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+        log_format = log_config.get('format', '%(asctime)s - %(levelname)s - %(message)s')
 
         # Create formatter
         formatter = logging.Formatter(log_format)
@@ -347,6 +347,7 @@ class GrobidClient(ApiClient):
 
         # Counter for actually processed files
         processed_files_count = 0
+        errors_files_count = 0
         input_files = []
 
         for input_file in all_input_files:
@@ -363,7 +364,7 @@ class GrobidClient(ApiClient):
             input_files.append(input_file)
 
             if len(input_files) == batch_size_pdf:
-                batch_processed = self.process_batch(
+                batch_processed, batch_errors = self.process_batch(
                     service,
                     input_files,
                     input_path,
@@ -381,11 +382,12 @@ class GrobidClient(ApiClient):
                     flavor
                 )
                 processed_files_count += batch_processed
+                errors_files_count += batch_errors
                 input_files = []
 
         # last batch
         if len(input_files) > 0:
-            batch_processed = self.process_batch(
+            batch_processed, batch_errors = self.process_batch(
                 service,
                 input_files,
                 input_path,
@@ -402,9 +404,11 @@ class GrobidClient(ApiClient):
                 verbose,
             )
             processed_files_count += batch_processed
+            errors_files_count += batch_errors
 
         # Log final statistics
         self.logger.info(f"Processing completed: {processed_files_count} out of {total_files} files processed")
+        self.logger.info(f"Errors: {errors_files_count} out of {total_files} files processed")
 
     def process_batch(
             self,
@@ -428,6 +432,7 @@ class GrobidClient(ApiClient):
             self.logger.info(f"{len(input_files)} files to process in current batch")
 
         processed_count = 0
+        error_count = 0
 
         # we use ThreadPoolExecutor and not ProcessPoolExecutor because it is an I/O intensive process
         with concurrent.futures.ThreadPoolExecutor(max_workers=n) as executor:
@@ -468,10 +473,10 @@ class GrobidClient(ApiClient):
         for r in concurrent.futures.as_completed(results):
             input_file, status, text = r.result()
             filename = self._output_file_name(input_file, input_path, output)
-            processed_count += 1
 
             if status != 200 or text is None:
                 self.logger.error(f"Processing of {input_file} failed with error {status}: {text}")
+                error_count += 1
                 # writing error file with suffixed error code
                 try:
                     pathlib.Path(os.path.dirname(filename)).mkdir(parents=True, exist_ok=True)
@@ -485,6 +490,7 @@ class GrobidClient(ApiClient):
                 except OSError as e:
                     self.logger.error(f"Failed to write error file {filename}: {str(e)}")
             else:
+                processed_count += 1
                 # writing TEI file
                 try:
                     pathlib.Path(os.path.dirname(filename)).mkdir(parents=True, exist_ok=True)
@@ -494,7 +500,7 @@ class GrobidClient(ApiClient):
                 except OSError as e:
                     self.logger.error(f"Failed to write TEI XML file {filename}: {str(e)}")
 
-        return processed_count
+        return processed_count, error_count
 
     def process_pdf(
             self,

{grobid_client_python-0.0.16 → grobid_client_python-0.0.18}/grobid_client_python.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: grobid-client-python
-Version: 0.0.16
+Version: 0.0.18
 Summary: Simple python client for GROBID REST services
 Author-email: Patrice Lopez <patrice.lopez@science-miner.com>
 Maintainer-email: Patrice Lopez <patrice.lopez@science-miner.com>, Luca Foppiano <lucanoro@duck.com>
@@ -254,7 +254,7 @@ A simple, efficient Python client for [GROBID](https://github.com/kermitt2/grobi
   - Local installation: [GROBID Documentation](http://grobid.readthedocs.io/)
   - Docker: `docker run -t --rm -p 8070:8070 lfoppiano/grobid:0.8.2`
   - Default server: `http://localhost:8070`
-  - Online demo: https://lfoppiano-grobid.hf.space (usage limits apply)
+  - Online demo: https://lfoppiano-grobid.hf.space (usage limits apply), more details [here](https://grobid.readthedocs.io/en/latest/getting_started/#using-grobid-from-the-cloud).
 
 > [!IMPORTANT]
 > GROBID supports Windows only through Docker containers. See the [Docker documentation](https://grobid.readthedocs.io/en/latest/Grobid-docker/) for details.
@@ -314,39 +314,39 @@ grobid_client [OPTIONS] SERVICE
 
 #### Available Services
 
-| Service | Description | Input Format |
-|---------|-------------|--------------|
-| `processFulltextDocument` | Extract full document structure | PDF files |
-| `processHeaderDocument` | Extract document metadata | PDF files |
-| `processReferences` | Extract bibliographic references | PDF files |
-| `processCitationList` | Parse citation strings | Text files (one citation per line) |
-| `processCitationPatentST36` | Process patent citations | XML ST36 format |
-| `processCitationPatentPDF` | Process patent PDFs | PDF files |
+| Service                     | Description                       | Input Format                       |
+|-----------------------------|-----------------------------------|------------------------------------|
+| `processFulltextDocument`   | Extract full document structure   | PDF files                          |
+| `processHeaderDocument`     | Extract document metadata         | PDF files                          |
+| `processReferences`         | Extract bibliographic references  | PDF files                          |
+| `processCitationList`       | Parse citation strings            | Text files (one citation per line) |
+| `processCitationPatentST36` | Process patent citations          | XML ST36 format                    |
+| `processCitationPatentPDF`  | Process patent PDFs               | PDF files                          |
 
 #### Common Options
 
-| Option | Description | Default |
-|--------|-------------|---------|
-| `--input` | Input directory path | Required |
-| `--output` | Output directory path | Same as input |
-| `--server` | GROBID server URL | `http://localhost:8070` |
-| `--n` | Concurrency level | 10 |
-| `--config` | Config file path | Optional |
-| `--force` | Overwrite existing files | False |
-| `--verbose` | Enable verbose logging | False |
+| Option      | Description              | Default                 |
+|-------------|--------------------------|-------------------------|
+| `--input`   | Input directory path     | Required                |
+| `--output`  | Output directory path    | Same as input           |
+| `--server`  | GROBID server URL        | `http://localhost:8070` |
+| `--n`       | Concurrency level        | 10                      |
+| `--config`  | Config file path         | Optional                |
+| `--force`   | Overwrite existing files | False                   |
+| `--verbose` | Enable verbose logging   | False                   |
 
 #### Processing Options
 
-| Option | Description |
-|--------|-------------|
-| `--generateIDs` | Generate random XML IDs |
-| `--consolidate_header` | Consolidate header metadata |
-| `--consolidate_citations` | Consolidate bibliographic references |
-| `--include_raw_citations` | Include raw citation text |
-| `--include_raw_affiliations` | Include raw affiliation text |
-| `--teiCoordinates` | Add PDF coordinates to XML |
-| `--segmentSentences` | Segment sentences with coordinates |
-| `--flavor` | Processing flavor for fulltext extraction |
+| Option                       | Description                               |
+|------------------------------|-------------------------------------------|
+| `--generateIDs`              | Generate random XML IDs                   |
+| `--consolidate_header`       | Consolidate header metadata               |
+| `--consolidate_citations`    | Consolidate bibliographic references      |
+| `--include_raw_citations`    | Include raw citation text                 |
+| `--include_raw_affiliations` | Include raw affiliation text              |
+| `--teiCoordinates`           | Add PDF coordinates to XML                |
+| `--segmentSentences`         | Segment sentences with coordinates        |
+| `--flavor`                   | Processing flavor for fulltext extraction |
 
 #### Examples
 
@@ -430,13 +430,13 @@ Configuration can be provided via a JSON file. When using the CLI, the `--server
 
 ### Configuration Parameters
 
-| Parameter | Description | Default |
-|-----------|-------------|---------|
-| `grobid_server` | GROBID server URL | `http://localhost:8070` |
-| `batch_size` | Thread pool size | 1000 |
-| `sleep_time` | Wait time when server is busy (seconds) | 5 |
-| `timeout` | Client-side timeout (seconds) | 60 |
-| `coordinates` | XML elements for coordinate extraction | See above |
+| Parameter       | Description                                                                                                      | Default                 |
+|-----------------|------------------------------------------------------------------------------------------------------------------|-------------------------|
+| `grobid_server` | GROBID server URL                                                                                                | `http://localhost:8070` |
+| `batch_size`    | Thread pool size. **Tune carefully: a large batch size will result in the data being written less frequently**   | 1000                    |
+| `sleep_time`    | Wait time when server is busy (seconds)                                                                          | 5                       |
+| `timeout`       | Client-side timeout (seconds)                                                                                    | 180                     |
+| `coordinates`   | XML elements for coordinate extraction                                                                           | See above               |
 
 > [!TIP]
 > Since version 0.0.12, the config file is optional. The client will use default localhost settings if no configuration is provided.

{grobid_client_python-0.0.16 → grobid_client_python-0.0.18}/pyproject.toml

@@ -3,7 +3,7 @@ requires = ["setuptools", "setuptools-scm"]
 build-backend = "setuptools.build_meta"
 
 [tool.bumpversion]
-current_version = "0.0.16"
+current_version = "0.0.18"
 commit = "true"
 tag = "true"
 tag_name = "v{new_version}"

{grobid_client_python-0.0.16 → grobid_client_python-0.0.18}/tests/test_config.json

@@ -6,7 +6,7 @@
     "timeout": 30,
     "logging": {
         "level": "DEBUG",
-        "format": "%(asctime)s - %(name)s - %(levelname)s - %(message)s",
+        "format": "%(asctime)s - %(levelname)s - %(message)s",
         "console": true,
         "file": null,
         "max_file_size": "10MB",

{grobid_client_python-0.0.16 → grobid_client_python-0.0.18}/tests/test_grobid_client.py

@@ -26,7 +26,7 @@ class TestGrobidClient:
             'timeout': 60,
             'logging': {
                 'level': 'INFO',
-                'format': '%(asctime)s - %(name)s - %(levelname)s - %(message)s',
+                'format': '%(asctime)s - %(levelname)s - %(message)s',
                 'console': True,
                 'file': None
             }
@@ -41,7 +41,7 @@ class TestGrobidClient:
         client = GrobidClient(check_server=False)
 
         assert client.config['grobid_server'] == 'http://localhost:8070'
-        assert client.config['batch_size'] == 1000
+        assert client.config['batch_size'] == 10
         assert client.config['sleep_time'] == 5
         assert client.config['timeout'] == 180
         assert 'persName' in client.config['coordinates']
@@ -258,7 +258,7 @@ class TestGrobidClient:
         with patch('grobid_client.grobid_client.GrobidClient._test_server_connection'):
             with patch('grobid_client.grobid_client.GrobidClient._configure_logging'):
                 with patch('grobid_client.grobid_client.GrobidClient.process_batch') as mock_batch:
-                    mock_batch.return_value = 2
+                    mock_batch.return_value = (2, 0)  # Return tuple as expected
                     client = GrobidClient(check_server=False)
                     client.logger = Mock()
 
@@ -421,7 +421,7 @@ class TestGrobidClient:
                                 verbose=False
                             )
 
-                            assert result == 1  # One file processed
+                            assert result == (1, 0)  # One file processed, zero errors
 
 
 class TestServerUnavailableException:

{grobid_client_python-0.0.16 → grobid_client_python-0.0.18}/tests/test_integration.py

@@ -181,7 +181,7 @@ class TestGrobidClientIntegration:
                                         force=True
                                     )
 
-                                    assert processed_count == 5
+                                    assert processed_count == (5, 0)
 
     def test_error_handling_and_recovery(self):
         """Test error handling and recovery mechanisms."""
@@ -342,5 +342,4 @@ class TestGrobidClientIntegration:
                                         force=True
                                     )
 
-                                    assert processed_count == 20
-
+                                    assert processed_count == (20, 0)