grobid-client-python 0.0.4__py3-none-any.whl → 0.0.7__py3-none-any.whl

grobid_client/grobid_client.py

@@ -33,7 +33,6 @@ class ServerUnavailableException(Exception):
 class GrobidClient(ApiClient):
 
     def __init__(self, grobid_server='localhost', 
-                 grobid_port='8070',
                  batch_size=1000, 
                  coordinates=["persName", "figure", "ref", "biblStruct", "formula", "s" ], 
                  sleep_time=5,
@@ -42,7 +41,6 @@ class GrobidClient(ApiClient):
                  check_server=True):
         self.config = {
             'grobid_server': grobid_server,
-            'grobid_port': grobid_port,
             'batch_size': batch_size,
             'coordinates': coordinates,
             'sleep_time': sleep_time,
@@ -62,7 +60,7 @@ class GrobidClient(ApiClient):
 
     def _test_server_connection(self):
         """Test if the server is up and running."""
-        the_url = self.config['grobid_server'] + "/api/isalive"
+        the_url = self.get_server_url("isalive")
         try:
             r = requests.get(the_url)
         except:
@@ -81,13 +79,13 @@ class GrobidClient(ApiClient):
         if output is not None:
             input_file_name = str(os.path.relpath(os.path.abspath(input_file), input_path))
             filename = os.path.join(
-                output, os.path.splitext(input_file_name)[0] + ".tei.xml"
+                output, os.path.splitext(input_file_name)[0] + ".grobid.tei.xml"
             )
         else:
             input_file_name = ntpath.basename(input_file)
             filename = os.path.join(
                 ntpath.dirname(input_file),
-                os.path.splitext(input_file_name)[0] + ".tei.xml",
+                os.path.splitext(input_file_name)[0] + ".grobid.tei.xml",
             )
 
         return filename
@@ -214,8 +212,18 @@ class GrobidClient(ApiClient):
             input_file, status, text = r.result()
             filename = self._output_file_name(input_file, input_path, output)
 
-            if text is None:
-                print("Processing of", input_file, "failed with error", str(status))
+            if status != 200 or text is None:
+                print("Processing of", input_file, "failed with error", str(status), ",", text)
+                # writing error file with suffixed error code
+                try:
+                    pathlib.Path(os.path.dirname(filename)).mkdir(parents=True, exist_ok=True)
+                    with open(filename.replace(".grobid.tei.xml", "_"+str(status)+".txt"), 'w', encoding='utf8') as tei_file:
+                        if text is not None:
+                            tei_file.write(text)
+                        else:
+                            tei_file.write("")
+                except OSError:
+                    print("Writing resulting TEI XML file", filename, "failed")
             else:
                 # writing TEI file
                 try:
@@ -247,7 +255,7 @@ class GrobidClient(ApiClient):
             )
         }
         
-        the_url = self.config['grobid_server'] + "/api/"+service
+        the_url = self.get_server_url(service)
 
         # set the GROBID parameters
         the_data = {}
@@ -291,6 +299,9 @@ class GrobidClient(ApiClient):
         pdf_handle.close()
         return (pdf_file, status, res.text)
 
+    def get_server_url(self, service):
+        return self.config['grobid_server'] + "/api/" + service
+
     def process_txt(
         self,
         service,
@@ -308,10 +319,7 @@ class GrobidClient(ApiClient):
         with open(txt_file) as f:
             references = [line.rstrip() for line in f]
 
-        the_url = "http://" + self.config["grobid_server"]
-        if len(self.config["grobid_port"]) > 0:
-            the_url += ":" + self.config["grobid_port"]
-        the_url += "/api/" + service
+        the_url = self.get_server_url(service)
 
         # set the GROBID parameters
         the_data = {}

{grobid_client_python-0.0.4.dist-info → grobid_client_python-0.0.7.dist-info}/LICENSE

@@ -187,7 +187,7 @@
      same "printed page" as the copyright notice for easier
      identification within third-party archives.
 
-  Copyright 2018-2021 The Contributors
+  Copyright 2018-2023 The Contributors
 
   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.

{grobid_client_python-0.0.4.dist-info → grobid_client_python-0.0.7.dist-info}/METADATA

@@ -1,12 +1,12 @@
 Metadata-Version: 2.1
 Name: grobid-client-python
-Version: 0.0.4
+Version: 0.0.7
 Summary: Simple python client for GROBID REST services
 Home-page: https://github.com/kermitt2/grobid_client_python
 Author: kermitt2
 License: LICENSE
-Platform: UNKNOWN
 Description-Content-Type: text/markdown
+License-File: LICENSE
 Requires-Dist: requests
 
 [![PyPI version](https://badge.fury.io/py/grobid_client_python.svg)](https://badge.fury.io/py/grobid_client_python)
@@ -17,9 +17,17 @@ Requires-Dist: requests
 
 This Python client can be used to process in an efficient concurrent manner a set of PDF in a given directory by the [GROBID](https://github.com/kermitt2/grobid) service. It includes a command line for processing PDF on a file system and write results in a given output directory and a library for import in other python scripts. The client can also process similarly a list of files with reference strings (one per line).
 
+## Before you start
+
+Please be aware that, at the moment, [grobid does not support Windows](https://grobid.readthedocs.io/en/latest/Troubleshooting/#windows-related-issues).
+If you are a Windows user, don't worry. You can still [run grobid 
+via Docker](https://grobid.readthedocs.io/en/latest/Grobid-docker/).
+
 ## Build and run
 
-You need first to install and start the *grobid* service, latest stable version, see the [documentation](http://grobid.readthedocs.io/). It is assumed that the server will run on the address `http://localhost:8070`. You can change the server address by editing the file `config.json`.
+You need first a running *grobid* service, latest stable version, see the [documentation](http://grobid.readthedocs.io/) for installation. 
+By default, it is assumed that the server will run on the address `http://localhost:8070`. 
+You can change the server address by editing the file `config.json`, see below.
 
 ## Requirements
 
@@ -29,7 +37,7 @@ This client has been developed and was tested with Python `3.5`-`3.8` and should
 
 Get the github repo:
 
-```
+```console
 git clone https://github.com/kermitt2/grobid_client_python
 cd grobid_client_python
 python3 setup.py install
@@ -37,6 +45,7 @@ python3 setup.py install
 
 There is nothing more needed to start using the python command lines, see the next section. 
 
+
 ## Usage and options
 
 ```
@@ -88,25 +97,33 @@ optional arguments:
 
 Examples:
 
+```console
 > grobid_client --input ~/tmp/in2 --output ~/tmp/out processFulltextDocument
+```
 
-This command will process all the PDF files present under the input directory recursively (files with extension `.pdf` only) with the `processFulltextDocument` service of GROBID, and write the resulting XML TEI files under the output directory, reusing the file name with a different file extension (`.tei.xml`), using the default `10` concurrent workers. 
+This command will process all the PDF files present under the input directory recursively (files with extension `.pdf` only) with the `processFulltextDocument` service of GROBID, and write the resulting XML TEI files under the output directory, reusing the file name with a different file extension (`.grobid.tei.xml`), using the default `10` concurrent workers. 
 
 If `--output` is omitted, the resulting XML TEI documents will be produced alongside the PDF in the `--input` directory.
 
+```console
 > grobid_client --input ~/tmp/in2 --output ~/tmp/out --n 20 processHeaderDocument
+```
 
-This command will process all the PDF files present in the input directory (files with extension `.pdf` only) with the `processHeaderDocument` service of GROBID, and write the resulting XML TEI files under the output directory, reusing the file name with a different file extension (`.tei.xml`), using `20` concurrent workers. 
+This command will process all the PDF files present in the input directory (files with extension `.pdf` only) with the `processHeaderDocument` service of GROBID, and write the resulting XML TEI files under the output directory, reusing the file name with a different file extension (`.grobid.tei.xml`), using `20` concurrent workers. 
 
-By default if an existing `.tei.xml` file is present in the output directory corresponding to a PDF in the input directory, this PDF will be skipped to avoid reprocessing several times the same PDF. To force the processing of PDF and over-write of existing TEI files, use the parameter `--force`.   
+By default if an existing `.grobid.tei.xml` file is present in the output directory corresponding to a PDF in the input directory, this PDF will be skipped to avoid reprocessing several times the same PDF. To force the processing of PDF and over-write of existing TEI files, use the parameter `--force`.   
 
 `processCitationList` does not take a repertory of PDF as input, but a repertory of `.txt` files, with one reference raw string per line, for example:
 
+```console
 > grobid_client --input resources/test_txt/ --output resources/test_out/ --n 20 processCitationList
+```
 
 The following command example will process all the PDF files present in the input directory and add bounding box coordinates (`--teiCoordinates`) relative to the original PDFs for the elements listed in the config file. It will also segment the sentences (`--segmentSentences`, this is a "layout aware" sentence segmentation) in the identified paragraphs with bounding box coordinates for the sentences. 
 
+```console
 > grobid_client --input ~/tmp/in2 --output ~/tmp/out --teiCoordinates --segmentSentences processFulltextDocument
+```
 
 The file `example.py` gives an example of usage as a library, from a another python script. 
 
@@ -114,7 +131,7 @@ The file `example.py` gives an example of usage as a library, from a another pyt
 
 Import and call the client as follow:
 
-```
+```python
 from grobid_client.grobid_client import GrobidClient
 
 client = GrobidClient(config_path="./config.json")
@@ -123,6 +140,32 @@ client.process("processFulltextDocument", "/mnt/data/covid/pdfs", n=20)
 
 See also `example.py`.
 
+## Configuration of the client
+
+There are a few parameters that can be set with the `config.json` file. 
+
+- `grobid_server` indicates the URL of the GROBID server to be used by the client. 
+
+- `batch_size` is the the size of the pool of threads used by ThreadPoolExecutor, you normally don't want to change this. This should be a high number (default 1000) - but not too high to protect the memory on the machine running the client. This should not be confused with the concurrency parameter `n` which indicates how many parallel requests can be send to GROBID.
+
+- `sleep_time` indicates in seconds the time to wait for sending a new request to GROBID when the server indicates that all its threads are currently used. The client need to re-send the query after a wait time that will allow the server to free some threads. This wait time usually depends on the service and the capacities of the server, we suggest 5-10 seconds for the `processFulltextDocument` service and 2 seconds for `processHeaderDocument` service.
+
+- `timeout` is a client side timeout - the process on server side will still be running until the server finished the task or the server timeout is reached.
+
+- `coordinates` indicates the structure XML elements that should contains PDF coordinates when the parameters `--teiCoordinates` is used see [here](https://grobid.readthedocs.io/en/latest/Coordinates-in-PDF/) for more details.
+
+Here is the default `config.json` file for the client:
+
+```
+{
+    "grobid_server": "http://localhost:8070",
+    "batch_size": 1000,
+    "sleep_time": 5,
+    "timeout": 60,
+    "coordinates": [ "persName", "figure", "ref", "biblStruct", "formula", "s" ]
+}
+```
+
 ## Benchmarking
 
 Full text processing of __136 PDF__ (total 3443 pages, in average 25 pages per PDF) on Intel Core i7-4790K CPU 4.00GHz, 4 cores (8 threads), 16GB memory, `n` being the concurrency parameter:
@@ -150,5 +193,3 @@ Processing of 3500 raw bibliographical take 4.3 s with `n=10` (814 references pa
 Distributed under [Apache 2.0 license](http://www.apache.org/licenses/LICENSE-2.0). 
 
 Main author and contact: Patrice Lopez (<patrice.lopez@science-miner.com>)
-
-

grobid_client_python-0.0.7.dist-info/RECORD

@@ -0,0 +1,9 @@
+grobid_client/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+grobid_client/client.py,sha256=crvUiVnnkn6tFwO4lKQg2cTxAHZxCBAmeNSVtuL4ZA0,6009
+grobid_client/grobid_client.py,sha256=wHgfrA-HJUPaXgKz7LjKBJ7ol84Tj_vuDtA5HRbs6k4,16720
+grobid_client_python-0.0.7.dist-info/LICENSE,sha256=L35Ao91pn_wG-mYDM3RfG7WKUjOnRmrCNHFSUQwXhKI,11184
+grobid_client_python-0.0.7.dist-info/METADATA,sha256=vLlG2E2D4BNPGLmJJ187zfpp1IO14p9Ww6gkILugaUM,9862
+grobid_client_python-0.0.7.dist-info/WHEEL,sha256=2wepM1nk4DS4eFpYrW1TTqPcoGNfHhhO_i5m4cOimbo,92
+grobid_client_python-0.0.7.dist-info/entry_points.txt,sha256=-WwPi8D8eXEbDB6QvitvHjxSGBeyLCyFvAoZcaDGswQ,67
+grobid_client_python-0.0.7.dist-info/top_level.txt,sha256=HnjgWxdJNKTLIN86XkgRAfVeEK4C71fc9X4fe7qdMvA,14
+grobid_client_python-0.0.7.dist-info/RECORD,,

{grobid_client_python-0.0.4.dist-info → grobid_client_python-0.0.7.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.34.2)
+Generator: bdist_wheel (0.38.4)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

{grobid_client_python-0.0.4.dist-info → grobid_client_python-0.0.7.dist-info}/entry_points.txt

@@ -1,3 +1,2 @@
 [console_scripts]
 grobid_client = grobid_client.grobid_client:main
-

grobid_client_python-0.0.4.dist-info/RECORD

@@ -1,9 +0,0 @@
-grobid_client/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-grobid_client/client.py,sha256=crvUiVnnkn6tFwO4lKQg2cTxAHZxCBAmeNSVtuL4ZA0,6009
-grobid_client/grobid_client.py,sha256=cEjSial_8ubdOYcE7h1BkwIw0jGWuKqXat4XDeMki_s,16250
-grobid_client_python-0.0.4.dist-info/LICENSE,sha256=O3ECu2kCEzpF8fBgQabIWw1muFjkZeYP_szjjhzOBW4,11184
-grobid_client_python-0.0.4.dist-info/METADATA,sha256=BVyrwD78Kqi3mdcv6J1SBMck9g9YFTOaDbVTS5K4PlA,7760
-grobid_client_python-0.0.4.dist-info/WHEEL,sha256=g4nMs7d-Xl9-xC9XovUrsDHGXt-FT0E17Yqo92DEfvY,92
-grobid_client_python-0.0.4.dist-info/entry_points.txt,sha256=IfhMcY5YhUem5uQ4ku1HbfAWjNLvptr0wJbMUubrdCs,68
-grobid_client_python-0.0.4.dist-info/top_level.txt,sha256=HnjgWxdJNKTLIN86XkgRAfVeEK4C71fc9X4fe7qdMvA,14
-grobid_client_python-0.0.4.dist-info/RECORD,,