grobid-client-python 0.1.4__tar.gz → 0.2.0__tar.gz

grobid_client_python-0.2.0/.github/dependabot.yml

@@ -0,0 +1,9 @@
+version: 2
+updates:
+  # Keep GitHub Actions used in the workflows up to date
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    commit-message:
+      prefix: "ci"

{grobid_client_python-0.1.4 → grobid_client_python-0.2.0}/.github/workflows/ci-build.yml

@@ -15,9 +15,9 @@ jobs:
       fail-fast: true
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v7
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v7
         with:
           python-version: ${{ matrix.python-version }}
           cache: 'pip'
@@ -30,7 +30,7 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install --upgrade flake8 pytest pycodestyle pytest-cov pytest-mock
+          pip install --upgrade flake8 pytest pycodestyle pytest-cov pytest-mock "mypy==2.3.*"
           if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
       - name: Lint with flake8
         run: |
@@ -38,12 +38,16 @@ jobs:
           flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
           # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
           flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+      - name: Type check with mypy
+        if: matrix.python-version == '3.11'
+        run: |
+          mypy grobid_client
       - name: Test with pytest
         run: |
           pytest tests/ -v --cov=grobid_client --cov-report=xml --cov-report=term-missing
       - name: Upload coverage to Codecov
         if: matrix.python-version == '3.11'
-        uses: codecov/codecov-action@v4
+        uses: codecov/codecov-action@v7
         with:
           file: ./coverage.xml
           flags: unittests

{grobid_client_python-0.1.4 → grobid_client_python-0.2.0}/.github/workflows/ci-release.yml

@@ -14,9 +14,9 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v7
       - name: Set up Python 3.9
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v7
         with:
           python-version: "3.9"
           cache: 'pip'

grobid_client_python-0.2.0/MANIFEST.in

@@ -0,0 +1,2 @@
+include Readme.md
+include grobid_client/py.typed

{grobid_client_python-0.1.4 → grobid_client_python-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: grobid-client-python
-Version: 0.1.4
+Version: 0.2.0
 Summary: Simple python client for GROBID REST services
 Author-email: Patrice Lopez <patrice.lopez@science-miner.com>
 Maintainer-email: Patrice Lopez <patrice.lopez@science-miner.com>, Luca Foppiano <lucanoro@duck.com>
@@ -216,6 +216,9 @@ Requires-Dist: requests
 Requires-Dist: dateparser
 Requires-Dist: beautifulsoup4
 Requires-Dist: lxml
+Provides-Extra: s3
+Requires-Dist: smart_open[s3]>=6.0; extra == "s3"
+Requires-Dist: boto3; extra == "s3"
 
 # GROBID Client Python
 
@@ -252,6 +255,10 @@ concurrent processing capabilities for PDF documents, reference strings, and pat
 - **Sentence Segmentation**: Layout-aware sentence segmentation capabilities
 - **JSON Output**: Convert TEI XML output to structured JSON format with CORD-19-like structure
 - **Markdown Output**: Convert TEI XML output to clean Markdown format with structured sections
+- **Type Hints**: Ships inline type annotations and a `py.typed` marker (PEP 561) for static type checking
+- **Archive Streaming**: Process files directly from `.zip`/`.tar`/`.tar.gz` archives without fully decompressing them
+- **S3 Streaming**: Read PDFs and zips straight from `s3://` (range-streamed, no full download) with the optional `[s3]` extra
+- **In-Memory Documents**: Send PDFs held as bytes straight to GROBID, without writing them to disk first
 
 ## 📋 Prerequisites
 
@@ -276,6 +283,9 @@ Choose one of the following installation methods:
 
 ```bash
 pip install grobid-client-python
+
+# to stream inputs directly from S3 (s3:// URIs), install the optional 's3' extra:
+pip install "grobid-client-python[s3]"
 ```
 
 ### Development Version
@@ -339,27 +349,28 @@ grobid_client [OPTIONS] SERVICE
 
 #### Common Options
 
-| Option      | Description              | Default                 |
-|-------------|--------------------------|-------------------------|
-| `--input`   | Input directory path     | Required                |
-| `--output`  | Output directory path    | Same as input           |
-| `--server`  | GROBID server URL        | `http://localhost:8070` |
-| `--n`       | Concurrency level        | 10                      |
-| `--config`  | Config file path         | Optional                |
-| `--force`   | Overwrite existing files | False                   |
-| `--verbose` | Enable verbose logging   | False                   |
+| Option          | Description                                       | Default                 |
+|-----------------|---------------------------------------------------|-------------------------|
+| `--input`       | Input directory path                              | Required                |
+| `--output`      | Output directory path                             | Same as input           |
+| `--server`      | GROBID server URL                                 | `http://localhost:8070` |
+| `--n`           | Concurrency level                                 | 10                      |
+| `--config`      | Config file path                                  | Optional                |
+| `--force`       | Overwrite existing files                          | False                   |
+| `--skip_errors` | Also skip documents that failed in a previous run | False                   |
+| `--verbose`     | Enable verbose logging                            | False                   |
 
 #### Processing Options
 
 | Option                       | Description                               |
 |------------------------------|-------------------------------------------|
-| `--generateIDs`              | Generate random XML IDs                   |
+| `--generate_ids`             | Generate random XML IDs                   |
 | `--consolidate_header`       | Consolidate header metadata               |
 | `--consolidate_citations`    | Consolidate bibliographic references      |
 | `--include_raw_citations`    | Include raw citation text                 |
 | `--include_raw_affiliations` | Include raw affiliation text              |
-| `--teiCoordinates`           | Add PDF coordinates to XML                |
-| `--segmentSentences`         | Segment sentences with coordinates        |
+| `--tei_coordinates`          | Add PDF coordinates to XML                |
+| `--segment_sentences`        | Segment sentences with coordinates        |
 | `--flavor`                   | Processing flavor for fulltext extraction |
 | `--json`                     | Convert TEI output to JSON format         |
 | `--markdown`                 | Convert TEI output to Markdown format     |
@@ -372,7 +383,7 @@ grobid_client [OPTIONS] SERVICE
 grobid_client --input ~/documents --output ~/results processFulltextDocument
 
 # High concurrency with coordinates
-grobid_client --input ~/pdfs --output ~/tei --n 20 --teiCoordinates processFulltextDocument
+grobid_client --input ~/pdfs --output ~/tei --n 20 --tei_coordinates processFulltextDocument
 
 # Process with JSON output
 grobid_client --input ~/pdfs --output ~/results --json processFulltextDocument
@@ -384,9 +395,50 @@ grobid_client --input ~/pdfs --output ~/results --markdown processFulltextDocume
 grobid_client --server https://grobid.example.com --input ~/citations.txt processCitationList
 
 # Force reprocessing with sentence segmentation and JSON output
-grobid_client --input ~/docs --force --segmentSentences --json processFulltextDocument
+grobid_client --input ~/docs --force --segment_sentences --json processFulltextDocument
+
+# Resume an interrupted run without retrying the documents that already failed
+grobid_client --input ~/docs --output ~/results --skip_errors processFulltextDocument
+
+# Process PDFs directly from a zip or tar.gz archive (streamed, not fully decompressed)
+grobid_client --input ~/papers.zip --output ~/results processFulltextDocument
+grobid_client --input ~/papers.tar.gz --output ~/results processFulltextDocument
+
+# --input also accepts glob patterns (quote them so the shell does not expand them)
+grobid_client --input "~/papers/*.zip"    --output ~/results processFulltextDocument   # many archives
+grobid_client --input "~/data/**/*.pdf"   --output ~/results processFulltextDocument   # PDFs in subdirectories
 ```
 
+> [!NOTE]
+> `--input` accepts a directory, a single file, an **archive**, or a **glob pattern**:
+> - **Archives** (`.zip`, `.tar`, `.tar.gz`/`.tgz`, `.tar.bz2`/`.tbz2`) are streamed: eligible entries are read into
+>   memory in chunks of `queue_size` and sent to GROBID straight from there — the archive is never fully decompressed and
+>   nothing but the results in `--output` ever touches the disk. If `--output` is omitted, results go to a directory named
+>   after the archive (e.g. `papers.zip` → `papers/`).
+> - **Glob patterns** (`paper.zip`, `paper*.zip`, `**/paper*.zip`, `**/*.pdf`, …) are expanded with `**` recursion; each
+>   match is handled by type (archive → streamed, directory → recursed, file → processed). Quote the pattern so your shell
+>   passes it through to the client unexpanded.
+> - **S3** (requires `pip install "grobid-client-python[s3]"`): pass an `s3://` object, prefix or glob. A remote zip is
+>   **range-streamed** (only its central directory and the entries are fetched — never the whole object); loose remote
+>   PDFs are fetched a batch at a time, directly into memory without being written to a local file first. Credentials use
+>   the standard AWS chain (env vars / `~/.aws` / IAM role).
+>   ```bash
+>   grobid_client --input "s3://my-bucket/papers/2021.zip"  --output ~/out processFulltextDocument   # one remote zip
+>   grobid_client --input "s3://my-bucket/pdfs/*.pdf"        --output ~/out processFulltextDocument   # loose PDFs
+>   grobid_client --input "s3://my-bucket/zips/"             --output ~/out processFulltextDocument   # every object under a prefix
+>   ```
+>
+> A **manifest of paths** (local, glob or `s3://`, one per line, `#` comments allowed) can be processed together via
+> `--input-list paths.txt` (combinable with `--input`).
+
+> [!NOTE]
+> **Skipping already handled documents.** By default a re-run skips a document only when its TEI output already exists,
+> so documents that failed are sent to GROBID again. Since a failed document generally fails again unless something
+> changed, `--skip_errors` also skips the documents for which a previous run left an error file
+> (`<name>_<status>.txt`, e.g. `paper_500.txt`) next to the expected TEI output. Drop the flag (or use `--force`) to
+> retry them. Error files are kept in sync automatically: the marker is deleted once the document is processed
+> successfully, and replaced when the same document fails again with a different status code.
+
 ### Python Library
 
 #### Basic Usage
@@ -421,10 +473,10 @@ client.process(
     input_path="/path/to/pdfs",
     output_path="/path/to/output",
     n=10,
-    generateIDs=True,
+    generate_ids=True,
     consolidate_header=True,
-    teiCoordinates=True,
-    segmentSentences=True
+    tei_coordinates=True,
+    segment_sentences=True
 )
 
 # Process with JSON output
@@ -443,6 +495,15 @@ client.process(
     markdown_output=True
 )
 
+# Re-run without retrying the documents that failed before
+client.process(
+    service="processFulltextDocument",
+    input_path="/path/to/pdfs",
+    output_path="/path/to/output",
+    force=False,
+    skip_errors=True
+)
+
 ```python
 # Process citation lists
 client.process(
@@ -452,6 +513,63 @@ client.process(
 )
 ```
 
+#### Processing a PDF from memory
+
+A PDF that is already in memory - downloaded from an API, read from a database or an object store - can be sent
+directly, without writing it to a temporary file first. `process_pdf` takes either a path or the document itself, as
+`bytes` or as any binary stream, and returns the TEI as a string:
+
+```python
+import io
+import requests
+
+pdf = io.BytesIO(requests.get("https://example.org/paper.pdf").content)
+pdf.name = "paper.pdf"          # optional, see below
+
+name, status, tei = client.process_pdf(
+    service="processFulltextDocument",
+    pdf_file=pdf,
+    consolidate_header=True,
+    tei_coordinates=True
+)
+
+if status == 200:
+    print(tei)
+```
+
+There is no flag to say where the document comes from: the object itself says it. A document also carries its own name,
+taken from the `name` attribute that `open()` sets on files and that can be set on anything else, `io.BytesIO` included.
+The name identifies the document in the request sent to GROBID, in the logs, and as the first element of the result, so
+documents processed this way stay distinguishable. Bytes passed on their own have nothing to be named after and fall
+back to `document.pdf`.
+
+Several documents can be sent concurrently with `process_documents`, which runs them through the same thread pool the
+file-based processing uses:
+
+```python
+results = client.process_documents(
+    service="processFulltextDocument",
+    documents=[pdf1, pdf2, "/path/to/paper3.pdf"],
+    n=10                            # documents sent concurrently
+)
+
+for name, status, tei in results:
+    ...
+```
+
+Documents that do not name themselves are named `document-1.pdf`, `document-2.pdf`, ... after their position. Results
+come back **in the order the documents were given**, not in completion order, so they can be zipped back onto whatever
+the caller has them keyed by. A document that fails does not stop the others: its own entry carries the error status.
+
+Before an in-memory run starts (this includes archive and `s3://` streaming), the client asks the server's `/api/health`
+how many engines it actually has and logs a warning when the requested concurrency `n` exceeds them - the surplus
+requests would only queue on the server or bounce as 503 - and an info message when engines would sit idle. The check is
+advisory: a server without the endpoint (older GROBID) never blocks the run.
+
+> [!NOTE]
+> Both return the TEI instead of writing it to disk, so the caller decides what to do with it. Use `process()` for the
+> directory-oriented processing with resume and JSON/Markdown conversion.
+
 ### Standalone Conversion Tools
 
 The library includes standalone scripts to convert TEI XML files to other formats without using the main client or server.
@@ -488,7 +606,6 @@ settings.
 ```json
 {
   "grobid_server": "http://localhost:8070",
-  "batch_size": 1000,
   "sleep_time": 5,
   "timeout": 60,
   "coordinates": [
@@ -507,7 +624,7 @@ settings.
 | Parameter       | Description                                                                                                      | Default                 |
 |-----------------|------------------------------------------------------------------------------------------------------------------|-------------------------|
 | `grobid_server` | GROBID server URL                                                                                                | `http://localhost:8070` |
-| `batch_size`    | Thread pool size. **Tune carefully: a large batch size will result in the data being written less frequently**   | 1000                    |
+| `queue_size`    | Number of files queued per processing chunk. See [Choosing a queue size](#choosing-a-queue-size). | 1000 for local directories, 1.2 × `n` for archives and S3 |
 | `sleep_time`    | Wait time when server is busy (seconds)                                                                          | 5                       |
 | `timeout`       | Client-side timeout (seconds)                                                                                    | 180                     |
 | `coordinates`   | XML elements for coordinate extraction                                                                           | See above               |
@@ -517,6 +634,32 @@ settings.
 > Since version 0.0.12, the config file is optional. The client will use default localhost settings if no configuration
 > is provided.
 
+### Choosing a queue size
+
+`queue_size` controls how many files are grouped into one processing chunk. It is a memory/durability knob, not a
+concurrency one: parallelism toward the GROBID server is set by `-n`, and each chunk is processed `n` files at a time.
+When `queue_size` is not set, the client picks a sensible default per input type (see the table above); set it
+explicitly only if you need to override that. A few guidelines:
+
+- **Never set it below `n`.** Effective parallelism is `min(n, queue_size)`: a queue smaller than the thread pool
+  leaves workers idle. A bit of headroom above `n` (the default streaming value is 1.2 × `n`) keeps the pool busy.
+- **Results are written per chunk.** Output files land on disk only once a whole chunk has been processed, so a
+  larger queue means results are written less frequently and an interrupted run loses at most one chunk of work
+  (already-written results are skipped on re-run unless `--force` is used).
+- **Local PDF directories:** the queue holds only file paths, so large values are essentially free — the default is
+  1000. Lower it if you want results flushed to disk more often on long runs.
+- **Archives (zip/tar) and S3:** each chunk is read or downloaded *into memory* before processing starts, so peak RAM
+  grows with `queue_size × average file size`. Keep it moderate — the 1.2 × `n` default is a safe floor; going up to
+  a few multiples of `n` (e.g. 2–5 ×) trades memory for slightly better throughput around chunk boundaries.
+
+> [!WARNING]
+> **Citation consolidation and the `timeout` setting.** When `--consolidate_citations` (or `consolidate_citations=True`)
+> is enabled, GROBID queries external services (e.g. CrossRef) to enrich the extracted references. This is considerably
+> slower than a plain extraction, and a low `timeout` frequently causes `HTTP 408 (Request Timeout)` errors.
+> Set the `timeout` to at least **120 seconds (2-3 minutes recommended)** when consolidating citations. The client emits
+> a warning when consolidation is requested with a timeout below 120 seconds.
+> See [issue #54](https://github.com/grobidOrg/grobid-client-python/issues/54).
+
 ### Logging Configuration
 
 The client provides configurable logging with different verbosity levels. By default, only essential statistics and warnings are shown.
@@ -673,7 +816,7 @@ When using the `--json` flag, the client converts TEI XML output to a structured
 grobid_client --input pdfs/ --output results/ --json processFulltextDocument
 
 # JSON output with coordinates and sentence segmentation
-grobid_client --input pdfs/ --output results/ --json --teiCoordinates --segmentSentences processFulltextDocument
+grobid_client --input pdfs/ --output results/ --json --tei_coordinates --segment_sentences processFulltextDocument
 ```
 
 ```python
@@ -754,7 +897,7 @@ Competing interests statement...
 grobid_client --input pdfs/ --output results/ --markdown processFulltextDocument
 
 # Markdown output with coordinates and sentence segmentation
-grobid_client --input pdfs/ --output results/ --markdown --teiCoordinates --segmentSentences processFulltextDocument
+grobid_client --input pdfs/ --output results/ --markdown --tei_coordinates --segment_sentences processFulltextDocument
 ```
 
 ```python

{grobid_client_python-0.1.4 → grobid_client_python-0.2.0}/Readme.md

@@ -33,6 +33,10 @@ concurrent processing capabilities for PDF documents, reference strings, and pat
 - **Sentence Segmentation**: Layout-aware sentence segmentation capabilities
 - **JSON Output**: Convert TEI XML output to structured JSON format with CORD-19-like structure
 - **Markdown Output**: Convert TEI XML output to clean Markdown format with structured sections
+- **Type Hints**: Ships inline type annotations and a `py.typed` marker (PEP 561) for static type checking
+- **Archive Streaming**: Process files directly from `.zip`/`.tar`/`.tar.gz` archives without fully decompressing them
+- **S3 Streaming**: Read PDFs and zips straight from `s3://` (range-streamed, no full download) with the optional `[s3]` extra
+- **In-Memory Documents**: Send PDFs held as bytes straight to GROBID, without writing them to disk first
 
 ## 📋 Prerequisites
 
@@ -57,6 +61,9 @@ Choose one of the following installation methods:
 
 ```bash
 pip install grobid-client-python
+
+# to stream inputs directly from S3 (s3:// URIs), install the optional 's3' extra:
+pip install "grobid-client-python[s3]"
 ```
 
 ### Development Version
@@ -120,27 +127,28 @@ grobid_client [OPTIONS] SERVICE
 
 #### Common Options
 
-| Option      | Description              | Default                 |
-|-------------|--------------------------|-------------------------|
-| `--input`   | Input directory path     | Required                |
-| `--output`  | Output directory path    | Same as input           |
-| `--server`  | GROBID server URL        | `http://localhost:8070` |
-| `--n`       | Concurrency level        | 10                      |
-| `--config`  | Config file path         | Optional                |
-| `--force`   | Overwrite existing files | False                   |
-| `--verbose` | Enable verbose logging   | False                   |
+| Option          | Description                                       | Default                 |
+|-----------------|---------------------------------------------------|-------------------------|
+| `--input`       | Input directory path                              | Required                |
+| `--output`      | Output directory path                             | Same as input           |
+| `--server`      | GROBID server URL                                 | `http://localhost:8070` |
+| `--n`           | Concurrency level                                 | 10                      |
+| `--config`      | Config file path                                  | Optional                |
+| `--force`       | Overwrite existing files                          | False                   |
+| `--skip_errors` | Also skip documents that failed in a previous run | False                   |
+| `--verbose`     | Enable verbose logging                            | False                   |
 
 #### Processing Options
 
 | Option                       | Description                               |
 |------------------------------|-------------------------------------------|
-| `--generateIDs`              | Generate random XML IDs                   |
+| `--generate_ids`             | Generate random XML IDs                   |
 | `--consolidate_header`       | Consolidate header metadata               |
 | `--consolidate_citations`    | Consolidate bibliographic references      |
 | `--include_raw_citations`    | Include raw citation text                 |
 | `--include_raw_affiliations` | Include raw affiliation text              |
-| `--teiCoordinates`           | Add PDF coordinates to XML                |
-| `--segmentSentences`         | Segment sentences with coordinates        |
+| `--tei_coordinates`          | Add PDF coordinates to XML                |
+| `--segment_sentences`        | Segment sentences with coordinates        |
 | `--flavor`                   | Processing flavor for fulltext extraction |
 | `--json`                     | Convert TEI output to JSON format         |
 | `--markdown`                 | Convert TEI output to Markdown format     |
@@ -153,7 +161,7 @@ grobid_client [OPTIONS] SERVICE
 grobid_client --input ~/documents --output ~/results processFulltextDocument
 
 # High concurrency with coordinates
-grobid_client --input ~/pdfs --output ~/tei --n 20 --teiCoordinates processFulltextDocument
+grobid_client --input ~/pdfs --output ~/tei --n 20 --tei_coordinates processFulltextDocument
 
 # Process with JSON output
 grobid_client --input ~/pdfs --output ~/results --json processFulltextDocument
@@ -165,9 +173,50 @@ grobid_client --input ~/pdfs --output ~/results --markdown processFulltextDocume
 grobid_client --server https://grobid.example.com --input ~/citations.txt processCitationList
 
 # Force reprocessing with sentence segmentation and JSON output
-grobid_client --input ~/docs --force --segmentSentences --json processFulltextDocument
+grobid_client --input ~/docs --force --segment_sentences --json processFulltextDocument
+
+# Resume an interrupted run without retrying the documents that already failed
+grobid_client --input ~/docs --output ~/results --skip_errors processFulltextDocument
+
+# Process PDFs directly from a zip or tar.gz archive (streamed, not fully decompressed)
+grobid_client --input ~/papers.zip --output ~/results processFulltextDocument
+grobid_client --input ~/papers.tar.gz --output ~/results processFulltextDocument
+
+# --input also accepts glob patterns (quote them so the shell does not expand them)
+grobid_client --input "~/papers/*.zip"    --output ~/results processFulltextDocument   # many archives
+grobid_client --input "~/data/**/*.pdf"   --output ~/results processFulltextDocument   # PDFs in subdirectories
 ```
 
+> [!NOTE]
+> `--input` accepts a directory, a single file, an **archive**, or a **glob pattern**:
+> - **Archives** (`.zip`, `.tar`, `.tar.gz`/`.tgz`, `.tar.bz2`/`.tbz2`) are streamed: eligible entries are read into
+>   memory in chunks of `queue_size` and sent to GROBID straight from there — the archive is never fully decompressed and
+>   nothing but the results in `--output` ever touches the disk. If `--output` is omitted, results go to a directory named
+>   after the archive (e.g. `papers.zip` → `papers/`).
+> - **Glob patterns** (`paper.zip`, `paper*.zip`, `**/paper*.zip`, `**/*.pdf`, …) are expanded with `**` recursion; each
+>   match is handled by type (archive → streamed, directory → recursed, file → processed). Quote the pattern so your shell
+>   passes it through to the client unexpanded.
+> - **S3** (requires `pip install "grobid-client-python[s3]"`): pass an `s3://` object, prefix or glob. A remote zip is
+>   **range-streamed** (only its central directory and the entries are fetched — never the whole object); loose remote
+>   PDFs are fetched a batch at a time, directly into memory without being written to a local file first. Credentials use
+>   the standard AWS chain (env vars / `~/.aws` / IAM role).
+>   ```bash
+>   grobid_client --input "s3://my-bucket/papers/2021.zip"  --output ~/out processFulltextDocument   # one remote zip
+>   grobid_client --input "s3://my-bucket/pdfs/*.pdf"        --output ~/out processFulltextDocument   # loose PDFs
+>   grobid_client --input "s3://my-bucket/zips/"             --output ~/out processFulltextDocument   # every object under a prefix
+>   ```
+>
+> A **manifest of paths** (local, glob or `s3://`, one per line, `#` comments allowed) can be processed together via
+> `--input-list paths.txt` (combinable with `--input`).
+
+> [!NOTE]
+> **Skipping already handled documents.** By default a re-run skips a document only when its TEI output already exists,
+> so documents that failed are sent to GROBID again. Since a failed document generally fails again unless something
+> changed, `--skip_errors` also skips the documents for which a previous run left an error file
+> (`<name>_<status>.txt`, e.g. `paper_500.txt`) next to the expected TEI output. Drop the flag (or use `--force`) to
+> retry them. Error files are kept in sync automatically: the marker is deleted once the document is processed
+> successfully, and replaced when the same document fails again with a different status code.
+
 ### Python Library
 
 #### Basic Usage
@@ -202,10 +251,10 @@ client.process(
     input_path="/path/to/pdfs",
     output_path="/path/to/output",
     n=10,
-    generateIDs=True,
+    generate_ids=True,
     consolidate_header=True,
-    teiCoordinates=True,
-    segmentSentences=True
+    tei_coordinates=True,
+    segment_sentences=True
 )
 
 # Process with JSON output
@@ -224,6 +273,15 @@ client.process(
     markdown_output=True
 )
 
+# Re-run without retrying the documents that failed before
+client.process(
+    service="processFulltextDocument",
+    input_path="/path/to/pdfs",
+    output_path="/path/to/output",
+    force=False,
+    skip_errors=True
+)
+
 ```python
 # Process citation lists
 client.process(
@@ -233,6 +291,63 @@ client.process(
 )
 ```
 
+#### Processing a PDF from memory
+
+A PDF that is already in memory - downloaded from an API, read from a database or an object store - can be sent
+directly, without writing it to a temporary file first. `process_pdf` takes either a path or the document itself, as
+`bytes` or as any binary stream, and returns the TEI as a string:
+
+```python
+import io
+import requests
+
+pdf = io.BytesIO(requests.get("https://example.org/paper.pdf").content)
+pdf.name = "paper.pdf"          # optional, see below
+
+name, status, tei = client.process_pdf(
+    service="processFulltextDocument",
+    pdf_file=pdf,
+    consolidate_header=True,
+    tei_coordinates=True
+)
+
+if status == 200:
+    print(tei)
+```
+
+There is no flag to say where the document comes from: the object itself says it. A document also carries its own name,
+taken from the `name` attribute that `open()` sets on files and that can be set on anything else, `io.BytesIO` included.
+The name identifies the document in the request sent to GROBID, in the logs, and as the first element of the result, so
+documents processed this way stay distinguishable. Bytes passed on their own have nothing to be named after and fall
+back to `document.pdf`.
+
+Several documents can be sent concurrently with `process_documents`, which runs them through the same thread pool the
+file-based processing uses:
+
+```python
+results = client.process_documents(
+    service="processFulltextDocument",
+    documents=[pdf1, pdf2, "/path/to/paper3.pdf"],
+    n=10                            # documents sent concurrently
+)
+
+for name, status, tei in results:
+    ...
+```
+
+Documents that do not name themselves are named `document-1.pdf`, `document-2.pdf`, ... after their position. Results
+come back **in the order the documents were given**, not in completion order, so they can be zipped back onto whatever
+the caller has them keyed by. A document that fails does not stop the others: its own entry carries the error status.
+
+Before an in-memory run starts (this includes archive and `s3://` streaming), the client asks the server's `/api/health`
+how many engines it actually has and logs a warning when the requested concurrency `n` exceeds them - the surplus
+requests would only queue on the server or bounce as 503 - and an info message when engines would sit idle. The check is
+advisory: a server without the endpoint (older GROBID) never blocks the run.
+
+> [!NOTE]
+> Both return the TEI instead of writing it to disk, so the caller decides what to do with it. Use `process()` for the
+> directory-oriented processing with resume and JSON/Markdown conversion.
+
 ### Standalone Conversion Tools
 
 The library includes standalone scripts to convert TEI XML files to other formats without using the main client or server.
@@ -269,7 +384,6 @@ settings.
 ```json
 {
   "grobid_server": "http://localhost:8070",
-  "batch_size": 1000,
   "sleep_time": 5,
   "timeout": 60,
   "coordinates": [
@@ -288,7 +402,7 @@ settings.
 | Parameter       | Description                                                                                                      | Default                 |
 |-----------------|------------------------------------------------------------------------------------------------------------------|-------------------------|
 | `grobid_server` | GROBID server URL                                                                                                | `http://localhost:8070` |
-| `batch_size`    | Thread pool size. **Tune carefully: a large batch size will result in the data being written less frequently**   | 1000                    |
+| `queue_size`    | Number of files queued per processing chunk. See [Choosing a queue size](#choosing-a-queue-size). | 1000 for local directories, 1.2 × `n` for archives and S3 |
 | `sleep_time`    | Wait time when server is busy (seconds)                                                                          | 5                       |
 | `timeout`       | Client-side timeout (seconds)                                                                                    | 180                     |
 | `coordinates`   | XML elements for coordinate extraction                                                                           | See above               |
@@ -298,6 +412,32 @@ settings.
 > Since version 0.0.12, the config file is optional. The client will use default localhost settings if no configuration
 > is provided.
 
+### Choosing a queue size
+
+`queue_size` controls how many files are grouped into one processing chunk. It is a memory/durability knob, not a
+concurrency one: parallelism toward the GROBID server is set by `-n`, and each chunk is processed `n` files at a time.
+When `queue_size` is not set, the client picks a sensible default per input type (see the table above); set it
+explicitly only if you need to override that. A few guidelines:
+
+- **Never set it below `n`.** Effective parallelism is `min(n, queue_size)`: a queue smaller than the thread pool
+  leaves workers idle. A bit of headroom above `n` (the default streaming value is 1.2 × `n`) keeps the pool busy.
+- **Results are written per chunk.** Output files land on disk only once a whole chunk has been processed, so a
+  larger queue means results are written less frequently and an interrupted run loses at most one chunk of work
+  (already-written results are skipped on re-run unless `--force` is used).
+- **Local PDF directories:** the queue holds only file paths, so large values are essentially free — the default is
+  1000. Lower it if you want results flushed to disk more often on long runs.
+- **Archives (zip/tar) and S3:** each chunk is read or downloaded *into memory* before processing starts, so peak RAM
+  grows with `queue_size × average file size`. Keep it moderate — the 1.2 × `n` default is a safe floor; going up to
+  a few multiples of `n` (e.g. 2–5 ×) trades memory for slightly better throughput around chunk boundaries.
+
+> [!WARNING]
+> **Citation consolidation and the `timeout` setting.** When `--consolidate_citations` (or `consolidate_citations=True`)
+> is enabled, GROBID queries external services (e.g. CrossRef) to enrich the extracted references. This is considerably
+> slower than a plain extraction, and a low `timeout` frequently causes `HTTP 408 (Request Timeout)` errors.
+> Set the `timeout` to at least **120 seconds (2-3 minutes recommended)** when consolidating citations. The client emits
+> a warning when consolidation is requested with a timeout below 120 seconds.
+> See [issue #54](https://github.com/grobidOrg/grobid-client-python/issues/54).
+
 ### Logging Configuration
 
 The client provides configurable logging with different verbosity levels. By default, only essential statistics and warnings are shown.
@@ -454,7 +594,7 @@ When using the `--json` flag, the client converts TEI XML output to a structured
 grobid_client --input pdfs/ --output results/ --json processFulltextDocument
 
 # JSON output with coordinates and sentence segmentation
-grobid_client --input pdfs/ --output results/ --json --teiCoordinates --segmentSentences processFulltextDocument
+grobid_client --input pdfs/ --output results/ --json --tei_coordinates --segment_sentences processFulltextDocument
 ```
 
 ```python
@@ -535,7 +675,7 @@ Competing interests statement...
 grobid_client --input pdfs/ --output results/ --markdown processFulltextDocument
 
 # Markdown output with coordinates and sentence segmentation
-grobid_client --input pdfs/ --output results/ --markdown --teiCoordinates --segmentSentences processFulltextDocument
+grobid_client --input pdfs/ --output results/ --markdown --tei_coordinates --segment_sentences processFulltextDocument
 ```
 
 ```python

{grobid_client_python-0.1.4 → grobid_client_python-0.2.0}/config.json

@@ -1,6 +1,5 @@
 {
   "grobid_server": "http://localhost:8070",
-  "batch_size": 1000,
   "timeout": 180,
   "sleep_time": 5,
   "coordinates": [