HowdenParser 1.0.0__tar.gz → 2.0.0__tar.gz

{howdenparser-1.0.0 → howdenparser-2.0.0}/HowdenParser/parameter/llamaparser.py

@@ -3,6 +3,6 @@ from typing import Literal
 from pydantic import BaseModel
 
 class Parameter(BaseModel):
-    model: str = "llamaparser:"
+    provider_and_model: str = "llamaparser:"
     result_type: Literal["md"] = "md"
     mode: bool = False

{howdenparser-1.0.0 → howdenparser-2.0.0}/HowdenParser/parser.py

@@ -4,8 +4,11 @@ import logging
 from pathlib import Path
 from PyPDF2 import PdfReader
 import dotenv
+import logging
+from inspect import signature
 
 dotenv.load_dotenv()
+logger = logging.getLogger(__name__)
 
 
 class BaseParser(ABC):
@@ -27,38 +30,61 @@ class Parser(BaseParser):
     """Factory + registry interface for all parsers."""
 
     @classmethod
-    def available_parsers(cls) -> dict[str, list[str]]:
-        """Return registered parsers and their init arguments."""
+    def available_parsers(cls) -> None:
+        """Print registered parsers and their init arguments."""
         import inspect
         result = {}
         for name, parser_cls in BaseParser._registry.items():
             sig = inspect.signature(parser_cls.__init__)
             result[name] = [p for p in sig.parameters if p != "self"]
         result.pop('', None)
-        return result
+        for key, values in result.items():
+            print(f"{key} with parameters: {values}")
 
     @classmethod
-    def create(cls, config: dict | None = None, **kwargs) -> BaseParser:
-        provider = kwargs["provider_and_model"].split(":")[0].lower()
-        model = kwargs["provider_and_model"].split(":")[1].lower()
+    def create(cls, config_or_dict: "Parameter | dict", **kwargs) -> BaseParser:
+        """
+        Dynamically create parser instances from a config object or dict.
+        Only passes arguments accepted by the parser constructor.
+        """
+        # Convert Parameter -> dict if needed
+        if hasattr(config_or_dict, "model_dump"):
+            config_dict = config_or_dict.model_dump()
+        elif isinstance(config_or_dict, dict):
+            config_dict = config_or_dict
+        else:
+            raise TypeError("Expected Parameter instance or dict for config_or_dict")
+
+        if "provider_and_model" not in kwargs and "provider_and_model" not in config_dict:
+            raise ValueError("provider_and_model must be specified")
+
+        # Merge dict + kwargs (kwargs take precedence)
+        merged_args = {**config_dict, **kwargs}
+
+        # Extract provider/model info
+        provider_and_model = merged_args.get("provider_and_model")
+        provider, model = provider_and_model.split(":")
+        provider = provider.lower()
+        model = model.lower()
+
         if provider not in BaseParser._registry:
-            raise ValueError(f"Unknown parser '{provider}'. "
-                             f"Available: {cls.available_parsers()}")
+            raise ValueError(f"Unknown parser '{provider}'. Available: {list(BaseParser._registry)}")
 
         parser_cls = BaseParser._registry[provider]
-        import inspect
-        merged_args = {**(config or {}), **kwargs}
-        if "model" in inspect.signature(parser_cls.__init__).parameters:
-            merged_args["model"] = model
 
-        # Remove keys not in constructor
-        sig = inspect.signature(parser_cls.__init__)
+        # Inspect constructor and only pass valid arguments
+        sig = signature(parser_cls.__init__)
         valid_args = {k: v for k, v in merged_args.items() if k in sig.parameters and k != "self"}
 
-        return parser_cls(**valid_args)
-
+        # Auto-insert 'model' if required
+        if "model" in sig.parameters and "model" not in valid_args:
+            valid_args["model"] = model
 
+        # Auto-insert 'provider_and_model' if required
+        if "provider_and_model" in sig.parameters and "provider_and_model" not in valid_args:
+            valid_args["provider_and_model"] = provider_and_model
 
+        return parser_cls(**valid_args)
 
     @abstractmethod
     def parse(self, text: str):
@@ -107,10 +133,10 @@ class MistralOCRParser(BaseParser, name="mistralocr"):
 
 
 class LangChainParser(BaseParser, name="langchain"):
-    def __init__(self, model: str):
+    def __init__(self, provider_and_model: str):
         from langchain.llms import OpenAI
-        self.model_name = model
-        self.model = OpenAI(model_name=model)
+        self.model = provider_and_model.split(":")[1]
+        self.model = OpenAI(model_name=self.model)
 
     def parse(self, text: str) -> dict:
         response = self.model(text)
@@ -138,36 +164,37 @@ class LlamaParser(BaseParser, name="llamaparser"):
 
 
 class HuggingFaceParser(BaseParser, name="huggingface"):
-    def __init__(self, model: str, result_type: str) -> None:
-        from transformers import pipeline
-
-        if result_type.lower() in ("md", "markdown"):
-            self.result_type = "markdown"
-        else:
-            self.result_type = "text"
+    def __init__(self, provider_and_model: str) -> None:
+        from transformers import TrOCRProcessor, VisionEncoderDecoderModel
 
         api_key = os.getenv("HF-API-TOKEN")
         if not api_key:
             raise EnvironmentError("Missing HF-API-TOKEN in .env file.")
 
-        self.parser = pipeline(
-            task="document-question-answering",
-            model=model,
-            use_auth_token=api_key
-        )
+        model_name = provider_and_model.split(":")[1]
+        logger.info(f"Loading Hugging Face OCR model: {model_name}")
+
+        self.processor = TrOCRProcessor.from_pretrained(model_name, token=api_key)
+        self.model = VisionEncoderDecoderModel.from_pretrained(model_name, token=api_key)
+
+        logger.info("Model and processor loaded successfully.")
 
     def parse(self, file_path: Path) -> str:
-        import fitz
-        pdf_doc = fitz.open(file_path)
-        output_parts = []
-
-        for page in pdf_doc:
-            pix = page.get_pixmap(dpi=200)
-            img_bytes = pix.tobytes("png")
-            response = self.parser(img_bytes, question="Extract all text")
-            if response and "answer" in response[0]:
-                output_parts.append(response[0]["answer"])
-
-        return "\n\n".join(output_parts) if self.result_type == "markdown" else " ".join(output_parts)
+        from pdf2image import convert_from_path
+        logger.info(f"Converting PDF to images: {file_path}")
+        pages = convert_from_path(file_path, dpi=300)
+        logger.info(f"PDF conversion complete. Total pages: {len(pages)}")
+
+        all_text = ""
+        for i, page in enumerate(pages, start=1):
+            logger.info(f"Running OCR on page {i}/{len(pages)}")
+            pixel_values = self.processor(page, return_tensors="pt").pixel_values
+            generated_ids = self.model.generate(pixel_values)
+            text = self.processor.batch_decode(generated_ids, skip_special_tokens=True)[0]
+            all_text += text + "\n"
+            logger.debug(f"OCR text (page {i}): {text[:100]}...")  # preview first 100 chars
+
+        logger.info("OCR completed for all pages.")
+        return all_text
 
 

{howdenparser-1.0.0 → howdenparser-2.0.0}/PKG-INFO

@@ -1,22 +1,27 @@
 Metadata-Version: 2.3
 Name: HowdenParser
-Version: 1.0.0
+Version: 2.0.0
 Summary: A simple configuration manager with Pydantic and JSON export.
 License: MIT
 Keywords: config,configuration,pydantic,json
 Author: JesperThoftIllemannJ
 Author-email: jesper.jaeger@howdendanmark.dk
-Requires-Python: >=3.12,<4.0
+Requires-Python: >=3.12,<3.14
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Requires-Dist: fitz (>=0.0.1.dev2,<0.0.2)
+Requires-Dist: hf-xet (>=1.1.7,<2.0.0)
 Requires-Dist: howdenconfig (>=0.1.13,<0.2.0)
 Requires-Dist: langchain (>=0.3.27,<0.4.0)
 Requires-Dist: llama-parse (>=0.6.58,<0.7.0)
 Requires-Dist: mistralai (>=1.9.3,<2.0.0)
+Requires-Dist: pdf2image (>=1.17.0,<2.0.0)
 Requires-Dist: pypdf2 (>=3.0.1,<4.0.0)
+Requires-Dist: torch (>=2.8.0,<3.0.0)
+Requires-Dist: torchaudio (>=2.8.0,<3.0.0)
+Requires-Dist: torchvision (>=0.23.0,<0.24.0)
 Requires-Dist: transformers (>=4.55.2,<5.0.0)
 Project-URL: Documentation, https://github.com/yourusername/config
 Project-URL: Homepage, https://github.com/yourusername/config
@@ -73,8 +78,9 @@ print(text)
 
 if HowdenConfig package being used
 
+config: Config = Config(parameter=Parameter())
 
-parser = ParserFactory.get_parser("mistralocr:", **config.parameter.dump_model()) 
+parser = Parser.create(config.parameter)
 
 text = parser.parse(Path("document.pdf"))
 

{howdenparser-1.0.0 → howdenparser-2.0.0}/README.md

@@ -48,8 +48,9 @@ print(text)
 
 if HowdenConfig package being used
 
+config: Config = Config(parameter=Parameter())
 
-parser = ParserFactory.get_parser("mistralocr:", **config.parameter.dump_model()) 
+parser = Parser.create(config.parameter)
 
 text = parser.parse(Path("document.pdf"))
 

{howdenparser-1.0.0 → howdenparser-2.0.0}/pyproject.toml

@@ -2,8 +2,8 @@
 name = "HowdenParser"
 description = ""
 readme = "README.md"
-requires-python = ">=3.12,<4.0"
-dependencies = [ "mistralai (>=1.9.3,<2.0.0)", "llama-parse (>=0.6.58,<0.7.0)", "langchain (>=0.3.27,<0.4.0)", "transformers (>=4.55.2,<5.0.0)", "fitz (>=0.0.1.dev2,<0.0.2)", "howdenconfig (>=0.1.13,<0.2.0)", "pypdf2 (>=3.0.1,<4.0.0)",]
+requires-python = ">=3.12,<3.14"
+dependencies = [ "mistralai (>=1.9.3,<2.0.0)", "llama-parse (>=0.6.58,<0.7.0)", "langchain (>=0.3.27,<0.4.0)", "transformers (>=4.55.2,<5.0.0)", "fitz (>=0.0.1.dev2,<0.0.2)", "howdenconfig (>=0.1.13,<0.2.0)", "pypdf2 (>=3.0.1,<4.0.0)", "pdf2image (>=1.17.0,<2.0.0)", "torch (>=2.8.0,<3.0.0)", "torchvision (>=0.23.0,<0.24.0)", "torchaudio (>=2.8.0,<3.0.0)", "hf-xet (>=1.1.7,<2.0.0)",]
 [[project.authors]]
 name = "JesperThoftIllemannJ"
 email = "jesper.jaeger@howdendanmark.dk"
@@ -14,7 +14,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "HowdenParser"
-version = "1.0.0"
+version = "2.0.0"
 description = "A simple configuration manager with Pydantic and JSON export."
 authors = [ "JesperThoftIllemannJ <jesper.jaeger@howdendanmark.dk>",]
 readme = "README.md"
@@ -26,5 +26,14 @@ documentation = "https://github.com/yourusername/config"
 [[tool.poetry.packages]]
 include = "HowdenParser"
 
+[tool.poetry.dependencies.torch]
+source = "pypi"
+
+[tool.poetry.dependencies.torchvision]
+source = "pypi"
+
+[tool.poetry.dependencies.torchaudio]
+source = "pypi"
+
 [tool.poetry.group.dev.dependencies]
 toml = "^0.10.2"