HowdenParser 1.0.0__tar.gz → 1.0.1__tar.gz

{howdenparser-1.0.0 → howdenparser-1.0.1}/HowdenParser/parameter/llamaparser.py

@@ -3,6 +3,6 @@ from typing import Literal
 from pydantic import BaseModel
 
 class Parameter(BaseModel):
-    model: str = "llamaparser:"
+    provider_and_model: str = "llamaparser:"
     result_type: Literal["md"] = "md"
     mode: bool = False

{howdenparser-1.0.0 → howdenparser-1.0.1}/HowdenParser/parser.py

@@ -4,8 +4,10 @@ import logging
 from pathlib import Path
 from PyPDF2 import PdfReader
 import dotenv
+import logging
 
 dotenv.load_dotenv()
+logger = logging.getLogger(__name__)
 
 
 class BaseParser(ABC):
@@ -27,7 +29,7 @@ class Parser(BaseParser):
     """Factory + registry interface for all parsers."""
 
     @classmethod
-    def available_parsers(cls) -> dict[str, list[str]]:
+    def available_parsers(cls) -> None:
         """Return registered parsers and their init arguments."""
         import inspect
         result = {}
@@ -35,7 +37,8 @@ class Parser(BaseParser):
             sig = inspect.signature(parser_cls.__init__)
             result[name] = [p for p in sig.parameters if p != "self"]
         result.pop('', None)
-        return result
+        for key, values in result.items():
+            print(f"{key} with parameters: {values}")
 
     @classmethod
     def create(cls, config: dict | None = None, **kwargs) -> BaseParser:
@@ -57,9 +60,6 @@ class Parser(BaseParser):
 
         return parser_cls(**valid_args)
 
-
-
-
     @abstractmethod
     def parse(self, text: str):
         pass
@@ -107,10 +107,10 @@ class MistralOCRParser(BaseParser, name="mistralocr"):
 
 
 class LangChainParser(BaseParser, name="langchain"):
-    def __init__(self, model: str):
+    def __init__(self, provider_and_model: str):
         from langchain.llms import OpenAI
-        self.model_name = model
-        self.model = OpenAI(model_name=model)
+        self.model = provider_and_model.split(":")[1]
+        self.model = OpenAI(model_name=self.model)
 
     def parse(self, text: str) -> dict:
         response = self.model(text)
@@ -138,36 +138,37 @@ class LlamaParser(BaseParser, name="llamaparser"):
 
 
 class HuggingFaceParser(BaseParser, name="huggingface"):
-    def __init__(self, model: str, result_type: str) -> None:
-        from transformers import pipeline
-
-        if result_type.lower() in ("md", "markdown"):
-            self.result_type = "markdown"
-        else:
-            self.result_type = "text"
+    def __init__(self, provider_and_model: str) -> None:
+        from transformers import TrOCRProcessor, VisionEncoderDecoderModel
 
         api_key = os.getenv("HF-API-TOKEN")
         if not api_key:
             raise EnvironmentError("Missing HF-API-TOKEN in .env file.")
 
-        self.parser = pipeline(
-            task="document-question-answering",
-            model=model,
-            use_auth_token=api_key
-        )
+        model_name = provider_and_model.split(":")[1]
+        logger.info(f"Loading Hugging Face OCR model: {model_name}")
+
+        self.processor = TrOCRProcessor.from_pretrained(model_name, token=api_key)
+        self.model = VisionEncoderDecoderModel.from_pretrained(model_name, token=api_key)
+
+        logger.info("Model and processor loaded successfully.")
 
     def parse(self, file_path: Path) -> str:
-        import fitz
-        pdf_doc = fitz.open(file_path)
-        output_parts = []
-
-        for page in pdf_doc:
-            pix = page.get_pixmap(dpi=200)
-            img_bytes = pix.tobytes("png")
-            response = self.parser(img_bytes, question="Extract all text")
-            if response and "answer" in response[0]:
-                output_parts.append(response[0]["answer"])
-
-        return "\n\n".join(output_parts) if self.result_type == "markdown" else " ".join(output_parts)
+        from pdf2image import convert_from_path
+        logger.info(f"Converting PDF to images: {file_path}")
+        pages = convert_from_path(file_path, dpi=300)
+        logger.info(f"PDF conversion complete. Total pages: {len(pages)}")
+
+        all_text = ""
+        for i, page in enumerate(pages, start=1):
+            logger.info(f"Running OCR on page {i}/{len(pages)}")
+            pixel_values = self.processor(page, return_tensors="pt").pixel_values
+            generated_ids = self.model.generate(pixel_values)
+            text = self.processor.batch_decode(generated_ids, skip_special_tokens=True)[0]
+            all_text += text + "\n"
+            logger.debug(f"OCR text (page {i}): {text[:100]}...")  # preview first 100 chars
+
+        logger.info("OCR completed for all pages.")
+        return all_text
 
 

{howdenparser-1.0.0 → howdenparser-1.0.1}/PKG-INFO

@@ -1,22 +1,27 @@
 Metadata-Version: 2.3
 Name: HowdenParser
-Version: 1.0.0
+Version: 1.0.1
 Summary: A simple configuration manager with Pydantic and JSON export.
 License: MIT
 Keywords: config,configuration,pydantic,json
 Author: JesperThoftIllemannJ
 Author-email: jesper.jaeger@howdendanmark.dk
-Requires-Python: >=3.12,<4.0
+Requires-Python: >=3.12,<3.14
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Requires-Dist: fitz (>=0.0.1.dev2,<0.0.2)
+Requires-Dist: hf-xet (>=1.1.7,<2.0.0)
 Requires-Dist: howdenconfig (>=0.1.13,<0.2.0)
 Requires-Dist: langchain (>=0.3.27,<0.4.0)
 Requires-Dist: llama-parse (>=0.6.58,<0.7.0)
 Requires-Dist: mistralai (>=1.9.3,<2.0.0)
+Requires-Dist: pdf2image (>=1.17.0,<2.0.0)
 Requires-Dist: pypdf2 (>=3.0.1,<4.0.0)
+Requires-Dist: torch (>=2.8.0,<3.0.0)
+Requires-Dist: torchaudio (>=2.8.0,<3.0.0)
+Requires-Dist: torchvision (>=0.23.0,<0.24.0)
 Requires-Dist: transformers (>=4.55.2,<5.0.0)
 Project-URL: Documentation, https://github.com/yourusername/config
 Project-URL: Homepage, https://github.com/yourusername/config

{howdenparser-1.0.0 → howdenparser-1.0.1}/pyproject.toml

@@ -2,8 +2,8 @@
 name = "HowdenParser"
 description = ""
 readme = "README.md"
-requires-python = ">=3.12,<4.0"
-dependencies = [ "mistralai (>=1.9.3,<2.0.0)", "llama-parse (>=0.6.58,<0.7.0)", "langchain (>=0.3.27,<0.4.0)", "transformers (>=4.55.2,<5.0.0)", "fitz (>=0.0.1.dev2,<0.0.2)", "howdenconfig (>=0.1.13,<0.2.0)", "pypdf2 (>=3.0.1,<4.0.0)",]
+requires-python = ">=3.12,<3.14"
+dependencies = [ "mistralai (>=1.9.3,<2.0.0)", "llama-parse (>=0.6.58,<0.7.0)", "langchain (>=0.3.27,<0.4.0)", "transformers (>=4.55.2,<5.0.0)", "fitz (>=0.0.1.dev2,<0.0.2)", "howdenconfig (>=0.1.13,<0.2.0)", "pypdf2 (>=3.0.1,<4.0.0)", "pdf2image (>=1.17.0,<2.0.0)", "torch (>=2.8.0,<3.0.0)", "torchvision (>=0.23.0,<0.24.0)", "torchaudio (>=2.8.0,<3.0.0)", "hf-xet (>=1.1.7,<2.0.0)",]
 [[project.authors]]
 name = "JesperThoftIllemannJ"
 email = "jesper.jaeger@howdendanmark.dk"
@@ -14,7 +14,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "HowdenParser"
-version = "1.0.0"
+version = "1.0.1"
 description = "A simple configuration manager with Pydantic and JSON export."
 authors = [ "JesperThoftIllemannJ <jesper.jaeger@howdendanmark.dk>",]
 readme = "README.md"
@@ -26,5 +26,14 @@ documentation = "https://github.com/yourusername/config"
 [[tool.poetry.packages]]
 include = "HowdenParser"
 
+[tool.poetry.dependencies.torch]
+source = "pypi"
+
+[tool.poetry.dependencies.torchvision]
+source = "pypi"
+
+[tool.poetry.dependencies.torchaudio]
+source = "pypi"
+
 [tool.poetry.group.dev.dependencies]
 toml = "^0.10.2"