data-science-document-ai 1.24.0__tar.gz → 1.43.6__tar.gz

{data_science_document_ai-1.24.0 → data_science_document_ai-1.43.6}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.3
+Metadata-Version: 2.4
 Name: data-science-document-ai
-Version: 1.24.0
+Version: 1.43.6
 Summary: "Document AI repo for data science"
 Author: Naomi Nguyen
 Author-email: naomi.nguyen@forto.com
@@ -38,6 +38,7 @@ Requires-Dist: pdf2image (>=1.17.0,<2.0.0)
 Requires-Dist: pgzip (>=0.3.5,<0.4.0)
 Requires-Dist: pyarrow (==16.1.0)
 Requires-Dist: pymupdf (>=1.23.26,<2.0.0)
+Requires-Dist: pypdf (>=6.1.2,<7.0.0)
 Requires-Dist: python-dotenv (>=1.0.0,<2.0.0)
 Requires-Dist: python-multipart (>=0.0.7,<0.0.8)
 Requires-Dist: rapidfuzz (>=3.12.2,<4.0.0)

{data_science_document_ai-1.24.0 → data_science_document_ai-1.43.6}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "data-science-document-ai"
-version = "1.24.0"
+version = "1.43.6"
 description = "\"Document AI repo for data science\""
 authors = ["Naomi Nguyen <naomi.nguyen@forto.com>", "Kumar Rajendrababu <kumar.rajendrababu@forto.com>", "Igor Tonko <igor.tonko@forto.com>", "Osman Demirel <osman.demirel@forto.com>"]
 packages = [
@@ -48,6 +48,7 @@ rapidfuzz = "^3.12.2"
 fuzzywuzzy = "^0.18.0"
 nltk = "^3.9.1"
 pgzip = "^0.3.5"
+pypdf = "^6.1.2"
 
 [tool.poetry.dev-dependencies]
 jupyter = "^1.0.0"

{data_science_document_ai-1.24.0 → data_science_document_ai-1.43.6}/src/constants.py

@@ -33,11 +33,23 @@ project_parameters = {
     # models metadata (confidence),
     "g_model_data_folder": "models",
     "local_model_data_folder": "data",
+    "released_doc_types": {
+        "bookingConfirmation",
+        "packingList",
+        "commercialInvoice",
+        "finalMbL",
+        "draftMbl",
+        "arrivalNotice",
+        "shippingInstruction",
+        "customsAssessment",
+        "deliveryOrder",
+        "partnerInvoice",
+        "customsInvoice",
+        "bundeskasse",
+    },
     "model_selector": {
         "stable": {
             "bookingConfirmation": 1,
-            "packingList": 0,
-            "commercialInvoice": 0,
             "finalMbL": 0,
             "draftMbl": 0,
             "arrivalNotice": 0,
@@ -55,11 +67,19 @@ project_parameters = {
     # LLM model parameters
     "gemini_params": {
         "temperature": 0,
-        "maxOutputTokens": 8000,
+        "maxOutputTokens": 65536,
+        "top_p": 0.8,
+        "top_k": 40,
+        "seed": 42,
+        "model_id": "gemini-2.5-pro",
+    },
+    "gemini_flash_params": {
+        "temperature": 0,
+        "maxOutputTokens": 65536,
         "top_p": 0.8,
         "top_k": 40,
         "seed": 42,
-        "model_id": "gemini-2.0-flash",
+        "model_id": "gemini-2.5-flash",
     },
     # Key to combine the LLM results with the Doc Ai results
     "key_to_combine": {
@@ -71,6 +91,8 @@ project_parameters = {
         "commercialInvoice": ["skus"],
         "shippingInstruction": ["containers"],
         "partnerInvoice": ["lineItem"],
+        "customsInvoice": ["lineItem"],
+        "bundeskasse": ["lineItem"],
     },
 }
 

{data_science_document_ai-1.24.0 → data_science_document_ai-1.43.6}/src/docai.py

@@ -3,11 +3,16 @@ import re
 
 from google.cloud import documentai
 
-from src.io import delete_folder_from_bucket, logger, upload_pdf_to_bucket
+from src.io import (
+    delete_folder_from_bucket,
+    get_gcp_labels,
+    logger,
+    upload_pdf_to_bucket,
+)
 from src.utils import cache_on_disk
 
 
-async def _process_pdf_w_docai(image_content, client, processor_name):
+async def _process_pdf_w_docai(image_content, client, processor_name, doc_type=None):
     """Process the PDF using Document AI.
 
     Args:
@@ -15,6 +20,7 @@ async def _process_pdf_w_docai(image_content, client, processor_name):
         client: The Document AI client.
         processor_name (str): The name of the processor to be used.
                             e.g.: projects/{project_id}/locations/{location}/processor/{processor_id}
+        doc_type (str, optional): Document type for cost tracking labels.
 
     Returns:
         The processed document.
@@ -24,10 +30,11 @@ async def _process_pdf_w_docai(image_content, client, processor_name):
         content=image_content, mime_type="application/pdf"
     )
 
-    # Configure the process request
+    # Configure the process request with labels for cost tracking
     request = documentai.ProcessRequest(
         name=processor_name,
         raw_document=raw_document,  # field_mask=field_mask
+        labels=get_gcp_labels(doc_type=doc_type),
     )
     result = await cache_on_disk(client.process_document, request=request)
 
@@ -35,7 +42,7 @@ async def _process_pdf_w_docai(image_content, client, processor_name):
 
 
 async def _batch_process_pdf_w_docai(
-    params, image_content, client, processor_name, timeout=1200
+    params, image_content, client, processor_name, timeout=1200, doc_type=None
 ):
     """Process the PDF using Document AI Batch Process API.
 
@@ -45,6 +52,7 @@ async def _batch_process_pdf_w_docai(
         processor_name (str): The name of the processor to be used.
                             e.g.: projects/{project_id}/locations/{location}/processor/{processor_id}
         timeout (int, optional): The timeout in seconds. Defaults to 1200.
+        doc_type (str, optional): Document type for cost tracking labels.
 
     Returns:
         The processed document.
@@ -72,11 +80,12 @@ async def _batch_process_pdf_w_docai(
     # Where to write results
     output_config = documentai.DocumentOutputConfig(gcs_output_config=gcs_output_config)
 
-    # The full resource name of the processor
+    # The full resource name of the processor with labels for cost tracking
     request = documentai.BatchProcessRequest(
         name=processor_name,
         input_documents=input_config,
         document_output_config=output_config,
+        labels=get_gcp_labels(doc_type=doc_type),
     )
 
     # BatchProcess returns a Long Running Operation (LRO)

{data_science_document_ai-1.24.0 → data_science_document_ai-1.43.6}/src/docai_processor_config.yaml

@@ -13,20 +13,6 @@ model_config:
           author: "igor.tonko@forto.com"
           created_date: ""
 
-    packingList:
-      - id: "d967005bd9d45aeb"
-        details:
-          display_name: "doc_cap_packingList"
-          author: "kumar.rajendrababu@forto.com"
-          created_date: ""
-
-    commercialInvoice:
-      - id: "7d37236207f75758"
-        details:
-          display_name: "doc_cap_commercialInvoice"
-          author: "kumar.rajendrababu@forto.com"
-          created_date: ""
-
     finalMbL:
       - id: "1eda2f22d64b1b89"
         details:
@@ -48,7 +34,6 @@ model_config:
           author: "kumar.rajendrababu@forto.com"
           created_date: ""
 
-
     arrivalNotice:
       - id: "748b2e2b9161dcf3"
         details:
@@ -70,13 +55,6 @@ model_config:
           author: "igor.tonko@forto.com"
           created_date: ""
 
-    partnerInvoice:
-      - id: "17d103181e745a05"
-        details:
-          display_name: "doc_cap_partnerInvoice"
-          author: "osman.demirel@forto.com"
-          created_date: ""
-
   beta:
     bookingConfirmation:
       - id: "3c280b11bdb3ed89"

{data_science_document_ai-1.24.0 → data_science_document_ai-1.43.6}/src/excel_processing.py

@@ -2,21 +2,28 @@
 # flake8: noqa: E402
 import logging
 
+from ddtrace import tracer
+
+from src.postprocessing.common import llm_prediction_to_tuples
+
 logger = logging.getLogger(__name__)
 
 import asyncio
-import json
 
 import numpy as np
 import pandas as pd
 
 from src.llm import prompt_excel_extraction
-from src.utils import generate_schema_structure, get_excel_sheets
+from src.prompts.prompt_library import prompt_library
+from src.utils import estimate_page_count, generate_schema_structure, get_excel_sheets
 
 
-async def extract_data_from_sheet(params, sheet_name, sheet, response_schema):
+async def extract_data_from_sheet(
+    llm_client, sheet_name, sheet, response_schema, doc_type=None
+):
     logger.info(f"Processing sheet: {sheet_name}")
-    excel_content = pd.DataFrame(sheet.values)
+    excel_content = pd.DataFrame(sheet.values).dropna(how="all", axis=1)
+
     # Convert to Markdown format for the LLM model
     worksheet = (
         "This is from a excel. Pay attention to the cell position:\n"
@@ -27,9 +34,10 @@ async def extract_data_from_sheet(params, sheet_name, sheet, response_schema):
     prompt_docai = prompt_excel_extraction(worksheet)
 
     try:
-        result = await params["LlmClient"].get_unified_json_genai(
+        result = await llm_client.get_unified_json_genai(
             prompt_docai,
             response_schema=response_schema,
+            doc_type=doc_type,
         )
     except Exception as e:
         result = {}
@@ -42,8 +50,8 @@ async def extract_data_from_excel(
     params,
     input_doc_type,
     file_content,
-    schema_client,
     mime_type,
+    llm_client,
 ):
     """Extract data from the Excel file.
 
@@ -51,8 +59,8 @@ async def extract_data_from_excel(
         params (dict): Parameters for the data extraction process.
         input_doc_type (str): The type of the document.
         file_content (bytes): The content of the Excel file to process.
-        schema_client (DocumentSchemaClient): Client for the Document AI schema.
         mime_type (str): The MIME type of the file.
+        llm_client: The LLM client to use for data extraction.
 
     Returns:
         formatted_data (list): A list of dictionaries containing the extracted data.
@@ -61,20 +69,42 @@ async def extract_data_from_excel(
 
     """
     # Generate the response structure
-    response_schema = generate_schema_structure(params, input_doc_type)
+    response_schema = (
+        prompt_library.library[input_doc_type]["other"]["placeholders"]
+        if input_doc_type
+        in [
+            "partnerInvoice",
+            "customsInvoice",
+            "bundeskasse",
+            "commercialInvoice",
+            "packingList",
+        ]
+        else generate_schema_structure(params, input_doc_type)
+    )
 
     # Load the Excel file and get ONLY the "visible" sheet names
     sheets, workbook = get_excel_sheets(file_content, mime_type)
 
+    # Track the number of sheets in dd-trace
+    span = tracer.current_span()
+    if span:
+        estimated_page_counts = [
+            estimate_page_count(workbook[sheet]) for sheet in sheets
+        ]
+        est_page_count = sum(estimated_page_counts)
+        span.set_metric("est_page_count", est_page_count)
+
     # Excel files may contain multiple sheets. Extract data from each sheet
     sheet_extract_tasks = [
         extract_data_from_sheet(
-            params, sheet_name, workbook[sheet_name], response_schema
+            llm_client,
+            sheet_name,
+            workbook[sheet_name],
+            response_schema,
+            doc_type=input_doc_type,
         )
         for sheet_name in sheets
     ]
     extracted_data = {k: v for k, v in await asyncio.gather(*sheet_extract_tasks)}
 
-    stored_data = json.dumps(extracted_data)
-
-    return extracted_data, stored_data, params["gemini_params"]["model_id"]
+    return extracted_data, extracted_data, llm_client.model_id

{data_science_document_ai-1.24.0 → data_science_document_ai-1.43.6}/src/io.py

@@ -12,13 +12,55 @@ from pathlib import Path
 from google.cloud import bigquery, storage
 
 
+def get_gcp_labels(**extra_labels):
+    """Generate standardized GCP labels for cost tracking.
+
+    Args:
+        **extra_labels: Additional custom labels
+
+    Returns:
+        dict: Labels dictionary with keys normalized (lowercase, hyphens, max 63 chars)
+    """
+    project_name = os.getenv("PROJECT_NAME")
+
+    # If not set, detect once and cache it
+    if not project_name:
+        # Try pyproject.toml first
+        try:
+            import toml
+
+            pyproject_path = Path(__file__).parent.parent / "pyproject.toml"
+            if pyproject_path.exists():
+                config = toml.load(pyproject_path)
+                project_name = config.get("tool", {}).get("poetry", {}).get("name")
+        except Exception:
+            pass
+
+        # Fallback to unknown
+        if not project_name:
+            project_name = "unknown"
+
+        # Cache it
+        os.environ["PROJECT_NAME"] = project_name
+
+    labels = {
+        "ds-project-name": project_name.lower(),
+        "ds-env": os.getenv("CLUSTER", "local").lower(),
+    }
+
+    # Add any extra labels
+    labels.update({k.lower(): str(v).lower() for k, v in extra_labels.items()})
+
+    return labels
+
+
 def get_bq_client(params):
     """Get Google BigQuery client."""
     bq_client = bigquery.Client(project=params["g_ai_project_name"])
     job_config = bigquery.QueryJobConfig(
         allow_large_results=True,
         # flatten_results=True,
-        labels={"project-name": params["project_name"]},
+        labels=get_gcp_labels(),
     )
     return bq_client, job_config
 
@@ -112,3 +154,6 @@ def download_dir_from_bucket(bucket, directory_cloud, directory_local) -> bool:
         Path(directory).mkdir(parents=True, exist_ok=True)
         blob.download_to_filename(directory_local / Path(blob.name))
     return result
+
+
+# type: ignore

{data_science_document_ai-1.24.0 → data_science_document_ai-1.43.6}/src/llm.py

@@ -15,6 +15,7 @@ from vertexai.generative_models import (
     Part,
 )
 
+from src.io import get_gcp_labels
 from src.utils import cache_on_disk
 
 
@@ -28,12 +29,12 @@ class LlmClient:
         # Initialize the model parameters
         self.model_params = {
             "temperature": parameters.get("temperature", 0),
-            "max_output_tokens": parameters.get("maxOutputTokens", 8000),
+            "max_output_tokens": parameters.get("maxOutputTokens", 65536),
             "top_p": parameters.get("top_p", 0.8),
             "top_k": parameters.get("top_k", 40),
             "seed": parameters.get("seed", 42),
         }
-        self.model_id = parameters.get("model_id", "gemini-1.5-pro-001")
+        self.model_id = parameters.get("model_id", "gemini-2.5-flash")
         # Initialize the safety configuration
         self.safety_config = {
             HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT: HarmBlockThreshold.BLOCK_NONE,
@@ -69,6 +70,7 @@ class LlmClient:
         document: str = None,
         response_schema: dict = None,
         response_mime_type: str = "application/json",
+        doc_type: str = None,
     ):
         """Ask the Gemini model a question.
 
@@ -76,6 +78,7 @@ class LlmClient:
             prompt (str): The prompt to send to the model.
             document (str, optional): An optional document to provide context.
             response_schema (dict, optional): Defines a specific response schema for the model.
+            doc_type (str, optional): Document type for cost tracking labels.
 
         Returns:
             str: The response from the model.
@@ -96,12 +99,13 @@ class LlmClient:
             # Prepare inputs for the model
             inputs = [document, prompt] if document else prompt
 
-            # Generate the response
+            # Generate the response with labels for cost tracking
             model_response = await cache_on_disk(
                 self.geminy_client.generate_content_async,
                 contents=inputs,
                 generation_config=config,
                 safety_settings=self.safety_config,
+                labels=get_gcp_labels(doc_type=doc_type),
             )
 
             response_text = model_response.text
@@ -113,7 +117,7 @@ class LlmClient:
             return "{}"
 
     async def get_unified_json_genai(
-        self, prompt, document=None, response_schema=None, model="gemini"
+        self, prompt, document=None, response_schema=None, model="gemini", doc_type=None
     ):
         """Send a prompt to a Google Cloud AI Platform model and returns the generated json.
 
@@ -122,6 +126,7 @@ class LlmClient:
             document: Content of the PDF document
             response_schema: The schema to use for the response
             model (str): The model to use for the response ["gemini" or "chatGPT"]. Default is "gemini".
+            doc_type (str, optional): Document type for cost tracking labels.
 
         Returns:
             dict: The generated json from the model.
@@ -131,7 +136,9 @@ class LlmClient:
             response = await self.ask_chatgpt(prompt, document, response_schema)
         else:
             # Default to Gemini
-            response = await self.ask_gemini(prompt, document, response_schema)
+            response = await self.ask_gemini(
+                prompt, document, response_schema, doc_type=doc_type
+            )
 
         try:
             return json.loads(response)