scitex 2.5.0__py3-none-any.whl → 2.7.0__py3-none-any.whl

scitex/io/_load_modules/_bibtex.py

@@ -4,6 +4,7 @@
 # File: ./src/scitex/io/_load_modules/_bibtex.py
 # ----------------------------------------
 import os
+
 __FILE__ = __file__
 __DIR__ = os.path.dirname(__FILE__)
 # ----------------------------------------
@@ -26,166 +27,164 @@ logger = logging.getLogger(__name__)
 def _load_bibtex(lpath: str, **kwargs) -> List[Dict[str, Any]]:
     """
     Load BibTeX file and parse entries.
-    
+
     Args:
         lpath: Path to the .bib file
         **kwargs: Additional arguments
             - parse_mode: 'full' (default) or 'basic'
             - encoding: File encoding (default: utf-8)
-    
+
     Returns:
         List of dictionaries, each representing a BibTeX entry
     """
-    parse_mode = kwargs.get('parse_mode', 'full')
-    encoding = kwargs.get('encoding', 'utf-8')
-    
+    parse_mode = kwargs.get("parse_mode", "full")
+    encoding = kwargs.get("encoding", "utf-8")
+
     try:
         # Read file content
-        with open(lpath, 'r', encoding=encoding) as f:
+        with open(lpath, "r", encoding=encoding) as f:
             content = f.read()
-        
+
         # Parse BibTeX entries
         entries = _parse_bibtex_content(content, parse_mode)
-        
+
         logger.info(f"Loaded {len(entries)} BibTeX entries from {lpath}")
         return entries
-        
+
     except Exception as e:
         raise ValueError(f"Error loading BibTeX file {lpath}: {str(e)}")
 
 
-def _parse_bibtex_content(content: str, parse_mode: str = 'full') -> List[Dict[str, Any]]:
+def _parse_bibtex_content(
+    content: str, parse_mode: str = "full"
+) -> List[Dict[str, Any]]:
     """
     Parse BibTeX content into structured entries.
-    
+
     Args:
         content: BibTeX file content
         parse_mode: 'full' for complete parsing, 'basic' for minimal parsing
-    
+
     Returns:
         List of parsed entries
     """
     entries = []
-    
+
     # Pattern to match BibTeX entries
-    entry_pattern = r'@(\w+)\s*\{\s*([^,]+)\s*,(.*?)\n\s*\}'
+    entry_pattern = r"@(\w+)\s*\{\s*([^,]+)\s*,(.*?)\n\s*\}"
     matches = re.finditer(entry_pattern, content, re.DOTALL | re.IGNORECASE)
-    
+
     for match in matches:
         entry_type = match.group(1).lower()
         entry_key = match.group(2).strip()
         entry_body = match.group(3)
-        
+
         # Parse entry fields
-        entry = {
-            'entry_type': entry_type,
-            'key': entry_key,
-            'fields': {}
-        }
-        
-        if parse_mode == 'full':
-            entry['fields'] = _parse_bibtex_fields(entry_body)
+        entry = {"entry_type": entry_type, "key": entry_key, "fields": {}}
+
+        if parse_mode == "full":
+            entry["fields"] = _parse_bibtex_fields(entry_body)
         else:
             # Basic parsing - just extract key-value pairs
-            entry['fields'] = _parse_bibtex_fields_basic(entry_body)
-        
+            entry["fields"] = _parse_bibtex_fields_basic(entry_body)
+
         entries.append(entry)
-    
+
     return entries
 
 
 def _parse_bibtex_fields(body: str) -> Dict[str, str]:
     """
     Parse BibTeX entry fields with proper handling of nested braces.
-    
+
     Args:
         body: The body of a BibTeX entry
-    
+
     Returns:
         Dictionary of field names to values
     """
     fields = {}
-    
+
     # Pattern to match field = value pairs
     # Handles both {braced} and "quoted" values
     field_pattern = r'(\w+)\s*=\s*(?:\{([^{}]*(?:\{[^{}]*\}[^{}]*)*)\}|"([^"]*)")'
-    
+
     for match in re.finditer(field_pattern, body):
         field_name = match.group(1).lower()
         # Get value from either braced or quoted group
         field_value = match.group(2) if match.group(2) is not None else match.group(3)
-        
+
         # Clean up the value
         field_value = field_value.strip()
-        
+
         # Handle special characters
         field_value = _unescape_bibtex(field_value)
-        
+
         fields[field_name] = field_value
-    
+
     return fields
 
 
 def _parse_bibtex_fields_basic(body: str) -> Dict[str, str]:
     """
     Basic parsing of BibTeX fields without special character handling.
-    
+
     Args:
         body: The body of a BibTeX entry
-    
+
     Returns:
         Dictionary of field names to values
     """
     fields = {}
-    
+
     # Simple pattern for basic parsing
-    lines = body.strip().split(',')
-    
+    lines = body.strip().split(",")
+
     for line in lines:
-        if '=' in line:
-            parts = line.split('=', 1)
+        if "=" in line:
+            parts = line.split("=", 1)
             if len(parts) == 2:
                 field_name = parts[0].strip().lower()
                 field_value = parts[1].strip()
-                
+
                 # Remove surrounding braces or quotes
-                if field_value.startswith('{') and field_value.endswith('}'):
+                if field_value.startswith("{") and field_value.endswith("}"):
                     field_value = field_value[1:-1]
                 elif field_value.startswith('"') and field_value.endswith('"'):
                     field_value = field_value[1:-1]
-                
+
                 fields[field_name] = field_value.strip()
-    
+
     return fields
 
 
 def _unescape_bibtex(text: str) -> str:
     """
     Unescape BibTeX special characters.
-    
+
     Args:
         text: BibTeX text with escaped characters
-    
+
     Returns:
         Unescaped text
     """
     # Common BibTeX escapes
     replacements = {
-        r'\&': '&',
-        r'\%': '%',
-        r'\$': '$',
-        r'\#': '#',
-        r'\_': '_',
-        r'\{': '{',
-        r'\}': '}',
-        r'~': ' ',  # Non-breaking space
-        r'--': '–',  # En dash
-        r'---': '—',  # Em dash
+        r"\&": "&",
+        r"\%": "%",
+        r"\$": "$",
+        r"\#": "#",
+        r"\_": "_",
+        r"\{": "{",
+        r"\}": "}",
+        r"~": " ",  # Non-breaking space
+        r"--": "–",  # En dash
+        r"---": "—",  # Em dash
     }
-    
+
     for escaped, unescaped in replacements.items():
         text = text.replace(escaped, unescaped)
-    
+
     return text
 
 
@@ -193,15 +192,15 @@ def _unescape_bibtex(text: str) -> str:
 def load_bibtex(filepath: str, **kwargs) -> List[Dict[str, Any]]:
     """
     Load and parse a BibTeX file.
-    
+
     Args:
         filepath: Path to the .bib file
         **kwargs: Additional arguments passed to _load_bibtex
-    
+
     Returns:
         List of parsed BibTeX entries
     """
     return _load_bibtex(filepath, **kwargs)
 
 
-# EOF
+# EOF

scitex/io/_load_modules/_canvas.py

@@ -69,23 +69,17 @@ def _load_canvas(
 
     # Validate path
     if not str(lpath).endswith(".canvas"):
-        raise ValueError(
-            f"Canvas path must end with .canvas extension: {lpath}"
-        )
+        raise ValueError(f"Canvas path must end with .canvas extension: {lpath}")
 
     if not lpath.exists():
         raise FileNotFoundError(f"Canvas directory not found: {lpath}")
 
     if not lpath.is_dir():
-        raise ValueError(
-            f"Canvas path must be a directory: {lpath}"
-        )
+        raise ValueError(f"Canvas path must be a directory: {lpath}")
 
     json_path = lpath / "canvas.json"
     if not json_path.exists():
-        raise FileNotFoundError(
-            f"canvas.json not found in canvas directory: {lpath}"
-        )
+        raise FileNotFoundError(f"canvas.json not found in canvas directory: {lpath}")
 
     # Load canvas.json
     with open(json_path, "r") as f:
@@ -107,6 +101,7 @@ def _load_canvas(
     if not as_dict:
         try:
             from scitex.vis.canvas import Canvas
+
             canvas_obj = Canvas.from_dict(canvas_dict)
             # Store reference to original directory for copying
             canvas_obj._canvas_dir = str(lpath)

scitex/io/_load_modules/_catboost.py

@@ -7,12 +7,15 @@ from typing import Union
 
 try:
     from catboost import CatBoostClassifier, CatBoostRegressor
+
     CATBOOST_AVAILABLE = True
 except ImportError:
     CATBOOST_AVAILABLE = False
+
     # Create placeholder classes for testing
     class CatBoostClassifier:
         pass
+
     class CatBoostRegressor:
         pass
 
@@ -50,8 +53,10 @@ def _load_catboost(
     >>> predictions = model.predict(X_test)
     """
     if not CATBOOST_AVAILABLE:
-        raise ImportError("CatBoost is not installed. Please install with: pip install catboost")
-    
+        raise ImportError(
+            "CatBoost is not installed. Please install with: pip install catboost"
+        )
+
     if not (lpath.endswith(".cbm") or lpath.endswith(".CBM")):
         raise ValueError("File must have .cbm extension")
 

scitex/io/_load_modules/_hdf5.py

@@ -4,6 +4,7 @@
 # File: /ssh:sp:/home/ywatanabe/proj/scitex_repo/src/scitex/io/_load_modules/_hdf5.py
 # ----------------------------------------
 import os
+
 __FILE__ = __file__
 __DIR__ = os.path.dirname(__FILE__)
 # ----------------------------------------
@@ -68,4 +69,5 @@ def _load_h5_object(h5_obj):
     else:
         return h5_obj
 
+
 # EOF

scitex/io/_load_modules/_image.py

@@ -11,7 +11,9 @@ from PIL import Image
 logger = logging.getLogger(__name__)
 
 
-def _load_image(lpath: str, metadata: bool = True, verbose: bool = False, **kwargs) -> Union[Any, Tuple[Any, Optional[Dict]]]:
+def _load_image(
+    lpath: str, metadata: bool = True, verbose: bool = False, **kwargs
+) -> Union[Any, Tuple[Any, Optional[Dict]]]:
     """
     Load image file.
 
@@ -41,6 +43,7 @@ def _load_image(lpath: str, metadata: bool = True, verbose: bool = False, **kwar
 
     try:
         from .._metadata import read_metadata
+
         metadata_dict = read_metadata(lpath)
 
         if verbose:
@@ -56,6 +59,7 @@ def _load_image(lpath: str, metadata: bool = True, verbose: bool = False, **kwar
     except Exception as e:
         # If metadata reading fails, return None
         import warnings
+
         warnings.warn(f"Failed to read metadata: {e}")
         return img, None
 

scitex/io/_load_modules/_matlab.py

@@ -5,7 +5,9 @@
 # ----------------------------------------
 import os
 
-__FILE__ = "/ssh:sp:/home/ywatanabe/proj/scitex_repo/src/scitex/io/_load_modules/_matlab.py"
+__FILE__ = (
+    "/ssh:sp:/home/ywatanabe/proj/scitex_repo/src/scitex/io/_load_modules/_matlab.py"
+)
 __DIR__ = os.path.dirname(__FILE__)
 # ----------------------------------------
 

scitex/io/_load_modules/_optuna.py

@@ -109,7 +109,6 @@ def load_yaml_as_an_optuna_dict(fpath_yaml, trial):
     _d = _load_yaml(fpath_yaml)
 
     for k, v in _d.items():
-
         dist = v["distribution"]
 
         if dist == "categorical":

scitex/io/_load_modules/_pdf.py

@@ -5,6 +5,7 @@
 # ----------------------------------------
 from __future__ import annotations
 import os
+
 __FILE__ = __file__
 __DIR__ = os.path.dirname(__FILE__)
 # ----------------------------------------
@@ -183,6 +184,7 @@ def _load_pdf(lpath: str, mode: str = "full", metadata: bool = False, **kwargs)
     if metadata:
         try:
             from .._metadata import read_metadata
+
             metadata_dict = read_metadata(lpath)
             return result, metadata_dict
         except Exception:
@@ -293,9 +295,7 @@ def _extract_text_pdfplumber(lpath: str, clean: bool) -> str:
         return full_text
 
     except Exception as e:
-        logger.error(
-            f"Error extracting text with pdfplumber from {lpath}: {e}"
-        )
+        logger.error(f"Error extracting text with pdfplumber from {lpath}: {e}")
         raise
 
 
@@ -363,9 +363,7 @@ def _extract_tables(
                         if table and len(table) > 0:
                             # First row as header if it looks like headers
                             if len(table) > 1 and all(
-                                isinstance(cell, str)
-                                for cell in table[0]
-                                if cell
+                                isinstance(cell, str) for cell in table[0] if cell
                             ):
                                 df = pd.DataFrame(table[1:], columns=table[0])
                             else:
@@ -453,26 +451,37 @@ def _extract_images(
                             img_pil = Image.open(io.BytesIO(image_bytes))
 
                             # Convert RGBA to RGB if necessary
-                            if img_pil.mode in ('RGBA', 'LA', 'P'):
+                            if img_pil.mode in ("RGBA", "LA", "P"):
                                 # Create a white background
-                                background = Image.new('RGB', img_pil.size, (255, 255, 255))
-                                if img_pil.mode == 'P':
-                                    img_pil = img_pil.convert('RGBA')
-                                background.paste(img_pil, mask=img_pil.split()[-1] if img_pil.mode == 'RGBA' else None)
+                                background = Image.new(
+                                    "RGB", img_pil.size, (255, 255, 255)
+                                )
+                                if img_pil.mode == "P":
+                                    img_pil = img_pil.convert("RGBA")
+                                background.paste(
+                                    img_pil,
+                                    mask=img_pil.split()[-1]
+                                    if img_pil.mode == "RGBA"
+                                    else None,
+                                )
                                 img_pil = background
-                            elif img_pil.mode != 'RGB':
-                                img_pil = img_pil.convert('RGB')
+                            elif img_pil.mode != "RGB":
+                                img_pil = img_pil.convert("RGB")
 
                             # Save as JPG
                             filename = f"page_{page_num + 1}_img_{img_index}.jpg"
                             filepath = os.path.join(output_dir, filename)
-                            img_pil.save(filepath, 'JPEG', quality=95)
+                            img_pil.save(filepath, "JPEG", quality=95)
 
                             image_info["ext"] = "jpg"
                         except ImportError:
-                            logger.warning("PIL not available for image conversion. Install with: pip install Pillow")
+                            logger.warning(
+                                "PIL not available for image conversion. Install with: pip install Pillow"
+                            )
                             # Fall back to original format
-                            filename = f"page_{page_num + 1}_img_{img_index}.{original_ext}"
+                            filename = (
+                                f"page_{page_num + 1}_img_{img_index}.{original_ext}"
+                            )
                             filepath = os.path.join(output_dir, filename)
                             with open(filepath, "wb") as img_file:
                                 img_file.write(image_bytes)
@@ -616,6 +625,7 @@ def _extract_metadata(lpath: str, backend: str) -> Dict[str, Any]:
             if subject:
                 try:
                     import json
+
                     # Check if subject is JSON (scitex metadata)
                     parsed_subject = json.loads(subject)
                     if isinstance(parsed_subject, dict):
@@ -648,6 +658,7 @@ def _extract_metadata(lpath: str, backend: str) -> Dict[str, Any]:
                     if subject:
                         try:
                             import json
+
                             parsed_subject = json.loads(subject)
                             if isinstance(parsed_subject, dict):
                                 # Merge parsed scitex metadata with standard PDF metadata
@@ -673,12 +684,8 @@ def _extract_metadata(lpath: str, backend: str) -> Dict[str, Any]:
                         "subject": reader.metadata.get("/Subject", ""),
                         "creator": reader.metadata.get("/Creator", ""),
                         "producer": reader.metadata.get("/Producer", ""),
-                        "creation_date": str(
-                            reader.metadata.get("/CreationDate", "")
-                        ),
-                        "modification_date": str(
-                            reader.metadata.get("/ModDate", "")
-                        ),
+                        "creation_date": str(reader.metadata.get("/CreationDate", "")),
+                        "modification_date": str(reader.metadata.get("/ModDate", "")),
                     }
                 )
 
@@ -690,6 +697,7 @@ def _extract_metadata(lpath: str, backend: str) -> Dict[str, Any]:
             if subject:
                 try:
                     import json
+
                     parsed_subject = json.loads(subject)
                     if isinstance(parsed_subject, dict):
                         # Merge parsed scitex metadata with standard PDF metadata
@@ -709,9 +717,7 @@ def _extract_metadata(lpath: str, backend: str) -> Dict[str, Any]:
     return metadata
 
 
-def _extract_pages(
-    lpath: str, backend: str, clean: bool
-) -> List[Dict[str, Any]]:
+def _extract_pages(lpath: str, backend: str, clean: bool) -> List[Dict[str, Any]]:
     """Extract content page by page."""
     pages = []
 
@@ -774,7 +780,11 @@ def _extract_pages(
 
 
 def _extract_scientific(
-    lpath: str, clean_text: bool, output_dir: str, table_settings: Dict, save_as_jpg: bool = True
+    lpath: str,
+    clean_text: bool,
+    output_dir: str,
+    table_settings: Dict,
+    save_as_jpg: bool = True,
 ) -> DotDict:
     """
     Optimized extraction for scientific papers.
@@ -823,9 +833,7 @@ def _extract_scientific(
             "total_words": len(result["text"].split()),
             "total_pages": result["metadata"].get("pages", 0),
             "num_sections": len(result["sections"]),
-            "num_tables": sum(
-                len(tables) for tables in result["tables"].values()
-            ),
+            "num_tables": sum(len(tables) for tables in result["tables"].values()),
             "num_images": len(result["images"]),
         }
 
@@ -946,4 +954,5 @@ def _calculate_file_hash(lpath: str) -> str:
             hash_md5.update(chunk)
     return hash_md5.hexdigest()
 
+
 # EOF

scitex/io/_load_modules/_sqlite3.py

@@ -4,6 +4,7 @@
 # File: /ssh:sp:/home/ywatanabe/proj/scitex_repo/src/scitex/io/_load_modules/_sqlite3.py
 # ----------------------------------------
 import os
+
 __FILE__ = __file__
 __DIR__ = os.path.dirname(__FILE__)
 # ----------------------------------------

scitex/io/_load_modules/_txt.py

@@ -4,6 +4,7 @@
 # File: /ssh:sp:/home/ywatanabe/proj/scitex_repo/src/scitex/io/_load_modules/_txt.py
 # ----------------------------------------
 import os
+
 __FILE__ = __file__
 __DIR__ = os.path.dirname(__FILE__)
 # ----------------------------------------
@@ -87,6 +88,7 @@ def _load_txt(lpath, strip=True, as_lines=True):
     """
     # Convert Path object to string if needed
     from pathlib import Path
+
     if isinstance(lpath, Path):
         lpath = str(lpath)
 

scitex/io/_load_modules/_xml.py

@@ -8,22 +8,22 @@ def _load_xml(lpath, **kwargs):
     """Load XML file and convert to dict."""
     if not lpath.endswith(".xml"):
         raise ValueError("File must have .xml extension")
-    
+
     # Import xml2dict locally to avoid circular imports
     from xml.etree import cElementTree as ElementTree
-    
+
     # Inline the xml2dict functionality to avoid circular import
     tree = ElementTree.parse(lpath)
     root = tree.getroot()
-    
+
     # Simplified XML to dict conversion - basic implementation
     def xml_element_to_dict(element):
         result = {}
-        
+
         # Add attributes
         if element.attrib:
             result.update(element.attrib)
-        
+
         # Handle child elements
         for child in element:
             if child.tag in result:
@@ -33,16 +33,16 @@ def _load_xml(lpath, **kwargs):
                 result[child.tag].append(xml_element_to_dict(child))
             else:
                 result[child.tag] = xml_element_to_dict(child)
-        
+
         # Handle text content
         if element.text and element.text.strip():
             if result:
-                result['text'] = element.text.strip()
+                result["text"] = element.text.strip()
             else:
                 return element.text.strip()
-        
+
         return result
-    
+
     return xml_element_to_dict(root)
 
 

scitex/io/_load_modules/_zarr.py

@@ -4,6 +4,7 @@
 # File: /ssh:sp:/home/ywatanabe/proj/scitex_repo/src/scitex/io/_load_modules/_zarr.py
 # ----------------------------------------
 import os
+
 __FILE__ = __file__
 __DIR__ = os.path.dirname(__FILE__)
 # ----------------------------------------
@@ -29,17 +30,17 @@ def _load_zarr(lpath: str, key: Optional[str] = None, **kwargs) -> Any:
     Any : Loaded data
     """
     from pathlib import Path
-    
+
     lpath_obj = Path(lpath)
-    
+
     try:
         # Check if it's a zip store
-        if lpath.endswith('.zip') or lpath.endswith('.zarr.zip'):
-            store = zarr.ZipStore(lpath, mode='r')
-            root = zarr.open(store, mode='r')
+        if lpath.endswith(".zip") or lpath.endswith(".zarr.zip"):
+            store = zarr.ZipStore(lpath, mode="r")
+            root = zarr.open(store, mode="r")
         # Check if it's consolidated
-        elif lpath_obj.joinpath('.zmetadata').exists():
-            root = zarr.open_consolidated(lpath, mode='r')
+        elif lpath_obj.joinpath(".zmetadata").exists():
+            root = zarr.open_consolidated(lpath, mode="r")
         else:
             # Regular directory store
             root = zarr.open(lpath, mode="r")
@@ -102,11 +103,11 @@ def _load_zarr_dataset(dataset):
     if hasattr(dataset, "attrs") and dataset.attrs.get("_type") == "pickled":
         pickled_bytes = dataset[:]
         return pickle.loads(pickled_bytes.tobytes())
-    
+
     # Check if it's a string
     if hasattr(dataset, "attrs") and dataset.attrs.get("_type") == "string":
         data = dataset[:]
-        return data.decode('utf-8') if hasattr(data, 'decode') else str(data)
+        return data.decode("utf-8") if hasattr(data, "decode") else str(data)
 
     # Regular data
     # Handle 0-d arrays (scalars) differently
@@ -119,8 +120,9 @@ def _load_zarr_dataset(dataset):
     if dataset.dtype.kind == "U":  # Unicode string
         return str(data) if dataset.ndim == 0 else data
     elif dataset.dtype.kind == "S":  # Byte string
-        return data.decode('utf-8') if hasattr(data, 'decode') else str(data)
+        return data.decode("utf-8") if hasattr(data, "decode") else str(data)
 
     return data
 
+
 # EOF