grobid-client-python 0.0.16__py3-none-any.whl → 0.0.18__py3-none-any.whl

grobid_client/grobid_client.py

@@ -41,7 +41,7 @@ class GrobidClient(ApiClient):
     # Default configuration values
     DEFAULT_CONFIG = {
         'grobid_server': 'http://localhost:8070',
-        'batch_size': 1000,
+        'batch_size': 10,
         'sleep_time': 5,
         'timeout': 180,
         'coordinates': [
@@ -60,7 +60,7 @@ class GrobidClient(ApiClient):
         ],
         'logging': {
             'level': 'INFO',
-            'format': '%(asctime)s - %(name)s - %(levelname)s - %(message)s',
+            'format': '%(asctime)s - %(levelname)s - %(message)s',
             'console': True,
             'file': None,  # Disabled by default
             'max_file_size': '10MB',
@@ -133,7 +133,7 @@ class GrobidClient(ApiClient):
         log_level = getattr(logging, log_level_str, logging.INFO)
 
         # Parse log format
-        log_format = log_config.get('format', '%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+        log_format = log_config.get('format', '%(asctime)s - %(levelname)s - %(message)s')
 
         # Create formatter
         formatter = logging.Formatter(log_format)
@@ -347,6 +347,7 @@ class GrobidClient(ApiClient):
 
         # Counter for actually processed files
         processed_files_count = 0
+        errors_files_count = 0
         input_files = []
 
         for input_file in all_input_files:
@@ -363,7 +364,7 @@ class GrobidClient(ApiClient):
             input_files.append(input_file)
 
             if len(input_files) == batch_size_pdf:
-                batch_processed = self.process_batch(
+                batch_processed, batch_errors = self.process_batch(
                     service,
                     input_files,
                     input_path,
@@ -381,11 +382,12 @@ class GrobidClient(ApiClient):
                     flavor
                 )
                 processed_files_count += batch_processed
+                errors_files_count += batch_errors
                 input_files = []
 
         # last batch
         if len(input_files) > 0:
-            batch_processed = self.process_batch(
+            batch_processed, batch_errors = self.process_batch(
                 service,
                 input_files,
                 input_path,
@@ -402,9 +404,11 @@ class GrobidClient(ApiClient):
                 verbose,
             )
             processed_files_count += batch_processed
+            errors_files_count += batch_errors
 
         # Log final statistics
         self.logger.info(f"Processing completed: {processed_files_count} out of {total_files} files processed")
+        self.logger.info(f"Errors: {errors_files_count} out of {total_files} files processed")
 
     def process_batch(
             self,
@@ -428,6 +432,7 @@ class GrobidClient(ApiClient):
             self.logger.info(f"{len(input_files)} files to process in current batch")
 
         processed_count = 0
+        error_count = 0
 
         # we use ThreadPoolExecutor and not ProcessPoolExecutor because it is an I/O intensive process
         with concurrent.futures.ThreadPoolExecutor(max_workers=n) as executor:
@@ -468,10 +473,10 @@ class GrobidClient(ApiClient):
         for r in concurrent.futures.as_completed(results):
             input_file, status, text = r.result()
             filename = self._output_file_name(input_file, input_path, output)
-            processed_count += 1
 
             if status != 200 or text is None:
                 self.logger.error(f"Processing of {input_file} failed with error {status}: {text}")
+                error_count += 1
                 # writing error file with suffixed error code
                 try:
                     pathlib.Path(os.path.dirname(filename)).mkdir(parents=True, exist_ok=True)
@@ -485,6 +490,7 @@ class GrobidClient(ApiClient):
                 except OSError as e:
                     self.logger.error(f"Failed to write error file {filename}: {str(e)}")
             else:
+                processed_count += 1
                 # writing TEI file
                 try:
                     pathlib.Path(os.path.dirname(filename)).mkdir(parents=True, exist_ok=True)
@@ -494,7 +500,7 @@ class GrobidClient(ApiClient):
                 except OSError as e:
                     self.logger.error(f"Failed to write TEI XML file {filename}: {str(e)}")
 
-        return processed_count
+        return processed_count, error_count
 
     def process_pdf(
             self,

{grobid_client_python-0.0.16.dist-info → grobid_client_python-0.0.18.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: grobid-client-python
-Version: 0.0.16
+Version: 0.0.18
 Summary: Simple python client for GROBID REST services
 Author-email: Patrice Lopez <patrice.lopez@science-miner.com>
 Maintainer-email: Patrice Lopez <patrice.lopez@science-miner.com>, Luca Foppiano <lucanoro@duck.com>
@@ -254,7 +254,7 @@ A simple, efficient Python client for [GROBID](https://github.com/kermitt2/grobi
   - Local installation: [GROBID Documentation](http://grobid.readthedocs.io/)
   - Docker: `docker run -t --rm -p 8070:8070 lfoppiano/grobid:0.8.2`
   - Default server: `http://localhost:8070`
-  - Online demo: https://lfoppiano-grobid.hf.space (usage limits apply)
+  - Online demo: https://lfoppiano-grobid.hf.space (usage limits apply), more details [here](https://grobid.readthedocs.io/en/latest/getting_started/#using-grobid-from-the-cloud).
 
 > [!IMPORTANT]
 > GROBID supports Windows only through Docker containers. See the [Docker documentation](https://grobid.readthedocs.io/en/latest/Grobid-docker/) for details.
@@ -314,39 +314,39 @@ grobid_client [OPTIONS] SERVICE
 
 #### Available Services
 
-| Service | Description | Input Format |
-|---------|-------------|--------------|
-| `processFulltextDocument` | Extract full document structure | PDF files |
-| `processHeaderDocument` | Extract document metadata | PDF files |
-| `processReferences` | Extract bibliographic references | PDF files |
-| `processCitationList` | Parse citation strings | Text files (one citation per line) |
-| `processCitationPatentST36` | Process patent citations | XML ST36 format |
-| `processCitationPatentPDF` | Process patent PDFs | PDF files |
+| Service                     | Description                       | Input Format                       |
+|-----------------------------|-----------------------------------|------------------------------------|
+| `processFulltextDocument`   | Extract full document structure   | PDF files                          |
+| `processHeaderDocument`     | Extract document metadata         | PDF files                          |
+| `processReferences`         | Extract bibliographic references  | PDF files                          |
+| `processCitationList`       | Parse citation strings            | Text files (one citation per line) |
+| `processCitationPatentST36` | Process patent citations          | XML ST36 format                    |
+| `processCitationPatentPDF`  | Process patent PDFs               | PDF files                          |
 
 #### Common Options
 
-| Option | Description | Default |
-|--------|-------------|---------|
-| `--input` | Input directory path | Required |
-| `--output` | Output directory path | Same as input |
-| `--server` | GROBID server URL | `http://localhost:8070` |
-| `--n` | Concurrency level | 10 |
-| `--config` | Config file path | Optional |
-| `--force` | Overwrite existing files | False |
-| `--verbose` | Enable verbose logging | False |
+| Option      | Description              | Default                 |
+|-------------|--------------------------|-------------------------|
+| `--input`   | Input directory path     | Required                |
+| `--output`  | Output directory path    | Same as input           |
+| `--server`  | GROBID server URL        | `http://localhost:8070` |
+| `--n`       | Concurrency level        | 10                      |
+| `--config`  | Config file path         | Optional                |
+| `--force`   | Overwrite existing files | False                   |
+| `--verbose` | Enable verbose logging   | False                   |
 
 #### Processing Options
 
-| Option | Description |
-|--------|-------------|
-| `--generateIDs` | Generate random XML IDs |
-| `--consolidate_header` | Consolidate header metadata |
-| `--consolidate_citations` | Consolidate bibliographic references |
-| `--include_raw_citations` | Include raw citation text |
-| `--include_raw_affiliations` | Include raw affiliation text |
-| `--teiCoordinates` | Add PDF coordinates to XML |
-| `--segmentSentences` | Segment sentences with coordinates |
-| `--flavor` | Processing flavor for fulltext extraction |
+| Option                       | Description                               |
+|------------------------------|-------------------------------------------|
+| `--generateIDs`              | Generate random XML IDs                   |
+| `--consolidate_header`       | Consolidate header metadata               |
+| `--consolidate_citations`    | Consolidate bibliographic references      |
+| `--include_raw_citations`    | Include raw citation text                 |
+| `--include_raw_affiliations` | Include raw affiliation text              |
+| `--teiCoordinates`           | Add PDF coordinates to XML                |
+| `--segmentSentences`         | Segment sentences with coordinates        |
+| `--flavor`                   | Processing flavor for fulltext extraction |
 
 #### Examples
 
@@ -430,13 +430,13 @@ Configuration can be provided via a JSON file. When using the CLI, the `--server
 
 ### Configuration Parameters
 
-| Parameter | Description | Default |
-|-----------|-------------|---------|
-| `grobid_server` | GROBID server URL | `http://localhost:8070` |
-| `batch_size` | Thread pool size | 1000 |
-| `sleep_time` | Wait time when server is busy (seconds) | 5 |
-| `timeout` | Client-side timeout (seconds) | 60 |
-| `coordinates` | XML elements for coordinate extraction | See above |
+| Parameter       | Description                                                                                                      | Default                 |
+|-----------------|------------------------------------------------------------------------------------------------------------------|-------------------------|
+| `grobid_server` | GROBID server URL                                                                                                | `http://localhost:8070` |
+| `batch_size`    | Thread pool size. **Tune carefully: a large batch size will result in the data being written less frequently**   | 1000                    |
+| `sleep_time`    | Wait time when server is busy (seconds)                                                                          | 5                       |
+| `timeout`       | Client-side timeout (seconds)                                                                                    | 180                     |
+| `coordinates`   | XML elements for coordinate extraction                                                                           | See above               |
 
 > [!TIP]
 > Since version 0.0.12, the config file is optional. The client will use default localhost settings if no configuration is provided.

grobid_client_python-0.0.18.dist-info/RECORD

@@ -0,0 +1,8 @@
+grobid_client/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+grobid_client/client.py,sha256=crvUiVnnkn6tFwO4lKQg2cTxAHZxCBAmeNSVtuL4ZA0,6009
+grobid_client/grobid_client.py,sha256=2oKtZw7UAoCqakHJo9LBMiSkQLTMpEXOfTV8h-rmcPk,31365
+grobid_client_python-0.0.18.dist-info/METADATA,sha256=iJGqV8JOKyM1cbIp6meysadW4aqtoLFIi1cSqyE1H7w,26273
+grobid_client_python-0.0.18.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+grobid_client_python-0.0.18.dist-info/entry_points.txt,sha256=-WwPi8D8eXEbDB6QvitvHjxSGBeyLCyFvAoZcaDGswQ,67
+grobid_client_python-0.0.18.dist-info/top_level.txt,sha256=HnjgWxdJNKTLIN86XkgRAfVeEK4C71fc9X4fe7qdMvA,14
+grobid_client_python-0.0.18.dist-info/RECORD,,

grobid_client_python-0.0.16.dist-info/RECORD

@@ -1,8 +0,0 @@
-grobid_client/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-grobid_client/client.py,sha256=crvUiVnnkn6tFwO4lKQg2cTxAHZxCBAmeNSVtuL4ZA0,6009
-grobid_client/grobid_client.py,sha256=3XlEXrg8v0TWeXWE1jhcYicrLOu5BOWE7l5FBO7YvjE,31063
-grobid_client_python-0.0.16.dist-info/METADATA,sha256=H2mnjLSnP8xjj5NN_6iONKghxMmPTfGZ-EKoDKWjNcY,24633
-grobid_client_python-0.0.16.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-grobid_client_python-0.0.16.dist-info/entry_points.txt,sha256=-WwPi8D8eXEbDB6QvitvHjxSGBeyLCyFvAoZcaDGswQ,67
-grobid_client_python-0.0.16.dist-info/top_level.txt,sha256=HnjgWxdJNKTLIN86XkgRAfVeEK4C71fc9X4fe7qdMvA,14
-grobid_client_python-0.0.16.dist-info/RECORD,,