grobid-client-python 0.1.2__tar.gz → 0.1.4__tar.gz

{grobid_client_python-0.1.2 → grobid_client_python-0.1.4}/.github/workflows/ci-build.yml

@@ -22,7 +22,11 @@ jobs:
           python-version: ${{ matrix.python-version }}
           cache: 'pip'
       - name: Cleanup more disk space
-        run: sudo rm -rf /usr/share/dotnet && sudo rm -rf /opt/ghc && sudo rm -rf "/usr/local/share/boost" && sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+        run: |
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /opt/ghc 
+          sudo rm -rf "/usr/local/share/boost"
+          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip

{grobid_client_python-0.1.2 → grobid_client_python-0.1.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: grobid-client-python
-Version: 0.1.2
+Version: 0.1.4
 Summary: Simple python client for GROBID REST services
 Author-email: Patrice Lopez <patrice.lopez@science-miner.com>
 Maintainer-email: Patrice Lopez <patrice.lopez@science-miner.com>, Luca Foppiano <lucanoro@duck.com>
@@ -443,6 +443,7 @@ client.process(
     markdown_output=True
 )
 
+```python
 # Process citation lists
 client.process(
     service="processCitationList",
@@ -451,6 +452,32 @@ client.process(
 )
 ```
 
+### Standalone Conversion Tools
+
+The library includes standalone scripts to convert TEI XML files to other formats without using the main client or server.
+
+#### TEI to JSON Converter
+
+Converts TEI XML files to the structured JSON format (similar to `--json` option).
+
+```bash
+# Convert a single file
+python -m grobid_client.format.TEI2LossyJSON_cli --input path/to/file.tei.xml --output path/to/output.json
+
+# Convert with verbose logging
+python -m grobid_client.format.TEI2LossyJSON_cli --input path/to/file.tei.xml --verbose
+```
+
+#### TEI to Markdown Converter
+
+Converts TEI XML files to Markdown format (similar to `--markdown` option).
+
+```bash
+# Convert a single file
+python -m grobid_client.format.TEI2Markdown_cli --input path/to/file.tei.xml --output path/to/output.md
+```
+
+
 ## ⚙️ Configuration
 
 Configuration can be provided via a JSON file. When using the CLI, the `--server` argument overrides the config file

{grobid_client_python-0.1.2 → grobid_client_python-0.1.4}/Readme.md

@@ -224,6 +224,7 @@ client.process(
     markdown_output=True
 )
 
+```python
 # Process citation lists
 client.process(
     service="processCitationList",
@@ -232,6 +233,32 @@ client.process(
 )
 ```
 
+### Standalone Conversion Tools
+
+The library includes standalone scripts to convert TEI XML files to other formats without using the main client or server.
+
+#### TEI to JSON Converter
+
+Converts TEI XML files to the structured JSON format (similar to `--json` option).
+
+```bash
+# Convert a single file
+python -m grobid_client.format.TEI2LossyJSON_cli --input path/to/file.tei.xml --output path/to/output.json
+
+# Convert with verbose logging
+python -m grobid_client.format.TEI2LossyJSON_cli --input path/to/file.tei.xml --verbose
+```
+
+#### TEI to Markdown Converter
+
+Converts TEI XML files to Markdown format (similar to `--markdown` option).
+
+```bash
+# Convert a single file
+python -m grobid_client.format.TEI2Markdown_cli --input path/to/file.tei.xml --output path/to/output.md
+```
+
+
 ## ⚙️ Configuration
 
 Configuration can be provided via a JSON file. When using the CLI, the `--server` argument overrides the config file

{grobid_client_python-0.1.2 → grobid_client_python-0.1.4}/grobid_client/format/TEI2LossyJSON.py

@@ -9,6 +9,8 @@ import os
 import uuid
 from collections import OrderedDict
 from concurrent.futures import ProcessPoolExecutor, as_completed
+import html
+import re
 from pathlib import Path
 from typing import Dict, Union, BinaryIO, Iterator
 
@@ -41,8 +43,15 @@ class TEI2LossyJSONConverter:
         If stream=False returns the full document dict (same shape as original function).
         """
         # Load with BeautifulSoup but avoid building huge structures when streaming
-        with open(tei_file, 'r') as f:
-            content = f.read()
+        if hasattr(tei_file, 'read'):
+            # File-like object (BinaryIO/StringIO)
+            content = tei_file.read()
+            if isinstance(content, bytes):
+                content = content.decode('utf-8')
+        else:
+            # Path-like object
+            with open(tei_file, 'r', encoding='utf-8') as f:
+                content = f.read()
         soup = BeautifulSoup(content, 'xml')
 
         if soup.TEI is None:
@@ -222,7 +231,6 @@ class TEI2LossyJSONConverter:
         Extract detailed bibliographic information from TEI biblStruct elements.
         Implements comprehensive parsing for all standard TEI bibliographic components.
         """
-        import re
 
         citation_data = OrderedDict()
         citation_data['id'] = f"b{index}"
@@ -430,7 +438,6 @@ class TEI2LossyJSONConverter:
 
     def _process_imprint_details(self, imprint_element: Tag, publication_metadata: Dict):
         """Extract and process imprint information including publisher, dates, and page ranges."""
-        import re
 
         # Extract publisher information
         publisher_elements = imprint_element.find_all("publisher")
@@ -557,7 +564,6 @@ class TEI2LossyJSONConverter:
         Extract person data (author/editor) from TEI persName or author elements.
         Handles various name formats and affiliations.
         """
-        import re
 
         person_data = {}
 
@@ -628,11 +634,9 @@ class TEI2LossyJSONConverter:
                     text = text.decode('utf-8', errors='ignore')
 
         # Normalize whitespace and strip
-        import re
         text = re.sub(r'\s+', ' ', text.strip())
 
         # Remove any potential XML/HTML entities
-        import html
         text = html.unescape(text)
 
         return text
@@ -665,14 +669,33 @@ class TEI2LossyJSONConverter:
 
                 div_type = div.get("type")
 
+                # Check if this is a header-only div (no content, no nested divs)
+                # If so, capture its header as context for subsequent divs
+                head = div.find("head")
+                direct_p_nodes = [c for c in div.children if hasattr(c, 'name') and c.name == "p"]
+                direct_formula_nodes = [c for c in div.children if hasattr(c, 'name') and c.name == "formula"]
+                nested_divs = [c for c in div.children if hasattr(c, 'name') and (c.name == "div" or (c.name and c.name.endswith(":div")))]
+                has_direct_content = len(direct_p_nodes) > 0 or len(direct_formula_nodes) > 0
+                
+                if head and not has_direct_content and len(nested_divs) == 0:
+                    # This is a header-only div with no nested content
+                    # Capture the header for the next div
+                    head_paragraph = self._clean_text(head.get_text())
+                    continue  # Skip to next div, the header will be used by subsequent sibling
+
                 # Process this div and potentially nested divs
                 for passage in self._process_div_with_nested_content(div, passage_level, head_paragraph):
                     yield passage
+                
+                # Reset head_paragraph after it's been used by a content-bearing div
+                head_paragraph = None
+
 
     def _process_div_with_nested_content(self, div: Tag, passage_level: str, head_paragraph: str = None) -> Iterator[Dict[str, Union[str, Dict[str, str]]]]:
         """
         Process a div and its nested content, handling various back section types.
         Supports nested divs for complex back sections like annex with multiple subsections.
+        Also handles formula elements that are direct children of divs.
         """
         head = div.find("head")
         p_nodes = div.find_all("p")
@@ -687,10 +710,12 @@ class TEI2LossyJSONConverter:
                 if child.name == "div" or child.name.endswith(":div"):
                     nested_divs.append(child)
 
-        # Count only direct child paragraphs, not those in nested divs
+        # Count only direct child paragraphs and formulas, not those in nested divs
         direct_p_nodes = [child for child in div.children if hasattr(child, 'name') and child.name == "p"]
+        direct_formula_nodes = [child for child in div.children if hasattr(child, 'name') and child.name == "formula"]
+        has_direct_content = len(direct_p_nodes) > 0 or len(direct_formula_nodes) > 0
 
-        if len(nested_divs) > 0 and len(direct_p_nodes) == 0:
+        if len(nested_divs) > 0 and not has_direct_content:
             # This is a container div - process each nested div independently
             for nested_div in nested_divs:
                 # Skip references divs
@@ -703,11 +728,11 @@ class TEI2LossyJSONConverter:
 
         # Determine the section header and content type for divs with content
         if head:
-            if len(direct_p_nodes) == 0:
-                # This div has only a head, no paragraphs (standalone head)
+            if not has_direct_content:
+                # This div has only a head, no paragraphs or formulas (standalone head)
                 current_head_paragraph = self._clean_text(head.get_text())
             else:
-                # This div has both head and paragraphs - head is the section header
+                # This div has both head and content - head is the section header
                 head_section = self._clean_text(head.get_text())
         else:
             # If no head element, try to use the type attribute as head_section
@@ -722,7 +747,7 @@ class TEI2LossyJSONConverter:
                     head_section = "Author Contributions"
                 elif div_type == "availability":
                     # Only set as default if this div has its own content
-                    if len(direct_p_nodes) > 0:
+                    if has_direct_content:
                         head_section = "Data Availability"
                 elif div_type == "annex":
                     head_section = "Annex"
@@ -730,27 +755,60 @@ class TEI2LossyJSONConverter:
                     # Generic handling - capitalize and format
                     head_section = div_type.replace("_", " ").title()
 
-        # Process paragraphs in this div
-        if len(direct_p_nodes) > 0:
-            for id_p, p in enumerate(direct_p_nodes):
+        # Process direct children (paragraphs and formulas) in document order
+        for child in div.children:
+            if not hasattr(child, 'name') or not child.name:
+                continue
+
+            if child.name == "p":
                 paragraph_id = get_random_id(prefix="p_")
 
                 if passage_level == "sentence":
-                    for id_s, sentence in enumerate(p.find_all("s")):
+                    for id_s, sentence in enumerate(child.find_all("s")):
                         struct = get_formatted_passage(current_head_paragraph or head_paragraph, head_section, paragraph_id, sentence)
                         if self.validate_refs:
                             for ref in struct['refs']:
-                                assert "Wrong offsets", ref['offset_start'] < ref['offset_end']
-                                assert "Cannot apply offsets", struct['text'][ref['offset_start']:ref['offset_end']] == ref['text']
+                                assert ref['offset_start'] < ref['offset_end'], "Wrong offsets"
+                                assert struct['text'][ref['offset_start']:ref['offset_end']] == ref['text'], "Cannot apply offsets"
                         yield struct
                 else:
-                    struct = get_formatted_passage(current_head_paragraph or head_paragraph, head_section, paragraph_id, p)
+                    struct = get_formatted_passage(current_head_paragraph or head_paragraph, head_section, paragraph_id, child)
                     if self.validate_refs:
                         for ref in struct['refs']:
-                            assert "Wrong offsets", ref['offset_start'] < ref['offset_end']
-                            assert "Cannot apply offsets", struct['text'][ref['offset_start']:ref['offset_end']] == ref['text']
+                            assert ref['offset_start'] < ref['offset_end'], "Wrong offsets"
+                            assert struct['text'][ref['offset_start']:ref['offset_end']] == ref['text'], "Cannot apply offsets"
                     yield struct
 
+            elif child.name == "formula":
+                # Process formula elements as passages
+                formula_id = get_random_id(prefix="f_")
+                formula_text = self._clean_text(child.get_text())
+                
+                if formula_text:
+                    # Create a passage structure for the formula
+                    formula_passage = {
+                        "id": formula_id,
+                        "text": formula_text,
+                        "coords": [
+                            box_to_dict(coord.split(","))
+                            for coord in child.get("coords", "").split(";")
+                        ] if child.has_attr("coords") else [],
+                        "refs": [],
+                        "type": "formula"
+                    }
+                    
+                    if current_head_paragraph or head_paragraph:
+                        formula_passage["head_paragraph"] = current_head_paragraph or head_paragraph
+                    if head_section:
+                        formula_passage["head_section"] = head_section
+                    
+                    # Extract formula label if present
+                    label = child.find("label")
+                    if label:
+                        formula_passage["label"] = self._clean_text(label.get_text())
+                    
+                    yield formula_passage
+
         # Update head_paragraph for potential next div
         if current_head_paragraph is not None:
             head_paragraph = current_head_paragraph
@@ -817,8 +875,6 @@ def get_refs_with_offsets(element):
     def _clean_text(text: str) -> str:
         if not text:
             return ""
-        import re
-        import html
         text = re.sub(r'\s+', ' ', text.strip())
         text = html.unescape(text)
         return text
@@ -903,8 +959,6 @@ def get_formatted_passage(head_paragraph, head_section, paragraph_id, element):
     def _clean_text_local(text: str) -> str:
         if not text:
             return ""
-        import re
-        import html
         text = re.sub(r'\s+', ' ', text.strip())
         text = html.unescape(text)
         return text

grobid_client_python-0.1.4/grobid_client/format/TEI2LossyJSON_cli.py

@@ -0,0 +1,131 @@
+#!/usr/bin/env python3
+"""
+Standalone CLI for TEI2LossyJSON converter.
+
+This script provides a command-line interface for converting TEI XML files to JSON format
+using the TEI2LossyJSONConverter.
+"""
+import argparse
+import json
+import logging
+import sys
+from pathlib import Path
+
+from .TEI2LossyJSON import TEI2LossyJSONConverter
+
+
+def setup_logging(verbose: bool = False):
+    """Setup logging configuration."""
+    level = logging.INFO if verbose else logging.WARNING
+    logging.basicConfig(
+        level=level,
+        format='%(asctime)s - %(levelname)s - %(message)s',
+        datefmt='%Y-%m-%d %H:%M:%S'
+    )
+
+
+def convert_single_file(input_file: Path, output_file: Path, verbose: bool = False) -> bool:
+    """Convert a single TEI file to JSON format."""
+    try:
+        if verbose:
+            logging.info(f"Converting {input_file} to {output_file}")
+
+        converter = TEI2LossyJSONConverter()
+        result = converter.convert_tei_file(input_file, stream=False)
+
+        if result is None:
+            logging.error(f"Failed to convert {input_file}: TEI file is not well-formed or empty")
+            return False
+
+        # Ensure output directory exists
+        output_file.parent.mkdir(parents=True, exist_ok=True)
+
+        # Write JSON output
+        with open(output_file, 'w', encoding='utf-8') as f:
+            json.dump(result, f, indent=2, ensure_ascii=False)
+
+        if verbose:
+            logging.info(f"Successfully converted {input_file} to {output_file}")
+
+        return True
+
+    except Exception as e:
+        logging.error(f"Error converting {input_file}: {str(e)}")
+        return False
+
+
+def main():
+    """Main CLI entry point."""
+    parser = argparse.ArgumentParser(
+        description="Convert TEI XML files to JSON format using TEI2LossyJSON converter",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+Examples:
+  # Convert a single TEI file
+  python -m grobid_client.format.TEI2LossyJSON --input input.tei.xml --output output.json
+
+  # Convert with verbose logging
+  python -m grobid_client.format.TEI2LossyJSON --input input.tei.xml --output output.json --verbose
+
+  # Convert and output to stdout
+  python -m grobid_client.format.TEI2LossyJSON --input input.tei.xml
+        """
+    )
+
+    parser.add_argument(
+        "--input", "-i",
+        type=Path,
+        required=True,
+        help="Input TEI XML file to convert"
+    )
+
+    parser.add_argument(
+        "--output", "-o",
+        type=Path,
+        help="Output JSON file (if not specified, prints to stdout)"
+    )
+
+    parser.add_argument(
+        "--verbose", "-v",
+        action="store_true",
+        help="Enable verbose logging"
+    )
+
+    args = parser.parse_args()
+
+    # Setup logging
+    setup_logging(args.verbose)
+
+    # Validate input file
+    if not args.input.exists():
+        logging.error(f"Input file does not exist: {args.input}")
+        sys.exit(1)
+
+    if not args.input.is_file():
+        logging.error(f"Input path is not a file: {args.input}")
+        sys.exit(1)
+
+    # Convert the file
+    if args.output:
+        success = convert_single_file(args.input, args.output, args.verbose)
+        sys.exit(0 if success else 1)
+    else:
+        # Output to stdout
+        try:
+            converter = TEI2LossyJSONConverter()
+            result = converter.convert_tei_file(args.input, stream=False)
+
+            if result is None:
+                logging.error(f"Failed to convert {args.input}: TEI file is not well-formed or empty")
+                sys.exit(1)
+
+            # Print JSON to stdout
+            print(json.dumps(result, indent=2, ensure_ascii=False))
+
+        except Exception as e:
+            logging.error(f"Error converting {args.input}: {str(e)}")
+            sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

{grobid_client_python-0.1.2 → grobid_client_python-0.1.4}/grobid_client/format/TEI2Markdown.py

@@ -11,8 +11,7 @@ Markdown format with the following sections:
 - Annex
 - References
 """
-import os
-import uuid
+import re
 from pathlib import Path
 from typing import List, Dict, Union, Optional, BinaryIO
 from bs4 import BeautifulSoup, NavigableString, Tag
@@ -44,9 +43,12 @@ class TEI2MarkdownConverter:
         try:
             # Load with BeautifulSoup
             if isinstance(tei_file, (str, Path)):
-                content = open(tei_file, 'r', encoding='utf-8').read()
+                with open(tei_file, 'r', encoding='utf-8') as f:
+                    content = f.read()
             else:
                 content = tei_file.read()
+                if isinstance(content, bytes):
+                    content = content.decode('utf-8')
                 
             soup = BeautifulSoup(content, 'xml')
 
@@ -77,7 +79,7 @@ class TEI2MarkdownConverter:
             # Extract publication date
             pub_date = self._extract_publication_date(soup)
             if pub_date:
-                markdown_sections.append(f"Publishd on {pub_date}\n\n")
+                markdown_sections.append(f"Published on {pub_date}\n\n")
 
             # Extract abstract
             abstract = self._extract_abstract(soup)
@@ -211,14 +213,24 @@ class TEI2MarkdownConverter:
             head = div.find("head")
             if head:
                 section_title = head.get_text().strip()
-                fulltext_sections.append(f"### {section_title}\n")
-
-            # Get paragraphs
-            paragraphs = div.find_all("p")
-            for p in paragraphs:
-                paragraph_text = self._process_paragraph(p)
-                if paragraph_text.strip():
-                    fulltext_sections.append(f"{paragraph_text}\n\n")
+                if section_title:
+                    fulltext_sections.append(f"### {section_title}\n")
+
+            # Process direct children of the div in document order
+            # This captures paragraphs, formulas, and other elements as they appear
+            for child in div.children:
+                if not hasattr(child, 'name') or not child.name:
+                    continue
+                    
+                if child.name == "p":
+                    paragraph_text = self._process_paragraph(child)
+                    if paragraph_text.strip():
+                        fulltext_sections.append(f"{paragraph_text}\n\n")
+                elif child.name == "formula":
+                    # Handle formula elements - extract text and optional label
+                    formula_text = self._process_formula(child)
+                    if formula_text.strip():
+                        fulltext_sections.append(f"{formula_text}\n\n")
         
         return "".join(fulltext_sections)
 
@@ -270,16 +282,23 @@ class TEI2MarkdownConverter:
             if header_text not in annex_sections:
                 annex_sections.append(header_text)
 
-        # Process paragraphs that are direct children of this div (not in nested divs)
+        # Process direct children of this div in document order
+        # This captures paragraphs, formulas, and other elements as they appear
         for child in div.children:
-            if hasattr(child, 'name') and child.name == "p":
+            if not hasattr(child, 'name') or not child.name:
+                continue
+                
+            if child.name == "p":
                 paragraph_text = self._process_paragraph(child)
                 if paragraph_text.strip():
                     annex_sections.append(f"{paragraph_text}\n\n")
-
-        # Process nested div elements
-        for child in div.children:
-            if hasattr(child, 'name') and child.name == "div":
+            elif child.name == "formula":
+                # Handle formula elements
+                formula_text = self._process_formula(child)
+                if formula_text.strip():
+                    annex_sections.append(f"{formula_text}\n\n")
+            elif child.name == "div":
+                # Process nested div elements
                 self._process_div_and_nested_divs(child, annex_sections)
 
     def _extract_references(self, soup: BeautifulSoup) -> str:
@@ -336,6 +355,34 @@ class TEI2MarkdownConverter:
         
         return "".join(text_parts).strip()
 
+    def _process_formula(self, formula_element: Tag) -> str:
+        """Process a formula element and convert to markdown.
+        
+        Formulas are rendered as italicized text with optional equation label.
+        """
+        # Get the main formula text (excluding the label)
+        formula_text_parts = []
+        label_text = ""
+        
+        for child in formula_element.children:
+            if hasattr(child, 'name') and child.name == "label":
+                # Extract equation label (e.g., "(1)", "(2)")
+                label_text = child.get_text().strip()
+            elif isinstance(child, NavigableString):
+                formula_text_parts.append(str(child))
+            else:
+                # Other elements within formula - get their text
+                formula_text_parts.append(child.get_text())
+        
+        formula_text = "".join(formula_text_parts).strip()
+        
+        if formula_text:
+            # Format as: *formula text* (label) if label exists
+            if label_text:
+                return f"*{formula_text}* {label_text}"
+            return f"*{formula_text}*"
+        return ""
+
     def _table_to_markdown(self, table_element: Tag) -> str:
         """Convert a table element to simple markdown."""
         markdown_lines = []
@@ -511,17 +558,25 @@ class TEI2MarkdownConverter:
             unit = bibl_scope.get("unit", "").lower()
             text = bibl_scope.get_text().strip()
 
-            if unit == "vol" and text:
+            if unit in ["vol", "volume"] and text:
                 bib_data['volume'] = text
             elif unit == "issue" and text:
                 bib_data['issue'] = text
             elif unit == "page" and text:
                 # Handle page ranges
-                if "from" in bibl_scope.attrs:
-                    bib_data['pages'] = f"{text}-"
-                elif "to" in bibl_scope.attrs and bib_data.get('pages'):
-                    bib_data['pages'] += text
-                else:
+                from_val = bibl_scope.get("from")
+                to_val = bibl_scope.get("to")
+                if from_val and to_val:
+                    # Both from and to in same element
+                    bib_data['pages'] = f"{from_val}-{to_val}"
+                elif from_val:
+                    # Only from specified, may get combined with another element
+                    bib_data['pages'] = f"{from_val}-"
+                elif to_val and bib_data.get('pages'):
+                    # Only to specified, append to existing from
+                    bib_data['pages'] = bib_data['pages'] + to_val
+                elif text and not bib_data.get('pages'):
+                    # Plain text, no from/to attributes
                     bib_data['pages'] = text
 
     def _extract_author_info(self, author: Tag) -> dict:
@@ -593,8 +648,6 @@ class TEI2MarkdownConverter:
 
     def _extract_year(self, date_text: str) -> str:
         """Extract year from date text, handling various formats."""
-        import re
-
         # Look for 4-digit year patterns
         year_match = re.search(r'\b(19|20)\d{2}\b', date_text)
         if year_match:
@@ -629,6 +682,9 @@ class TEI2MarkdownConverter:
         """Build publication details string from extracted data."""
         details = []
 
+        if ref_data.get('year'):
+            details.append(f"({ref_data['year']})")
+
         if ref_data.get('volume'):
             details.append(ref_data['volume'])
 
@@ -684,7 +740,6 @@ class TEI2MarkdownConverter:
         raw_text = bibl_struct.get_text().strip()
 
         # Remove reference number if present
-        import re
         raw_text = re.sub(r'^\[\d+\]\s*', '', raw_text)
 
         # Clean up excessive whitespace