data-science-document-ai 1.41.0__tar.gz → 1.45.2__tar.gz

{data_science_document_ai-1.41.0 → data_science_document_ai-1.45.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: data-science-document-ai
-Version: 1.41.0
+Version: 1.45.2
 Summary: "Document AI repo for data science"
 Author: Naomi Nguyen
 Author-email: naomi.nguyen@forto.com
@@ -38,7 +38,7 @@ Requires-Dist: pdf2image (>=1.17.0,<2.0.0)
 Requires-Dist: pgzip (>=0.3.5,<0.4.0)
 Requires-Dist: pyarrow (==16.1.0)
 Requires-Dist: pymupdf (>=1.23.26,<2.0.0)
-Requires-Dist: pypdf2 (>=3.0.1,<4.0.0)
+Requires-Dist: pypdf (>=6.1.2,<7.0.0)
 Requires-Dist: python-dotenv (>=1.0.0,<2.0.0)
 Requires-Dist: python-multipart (>=0.0.7,<0.0.8)
 Requires-Dist: rapidfuzz (>=3.12.2,<4.0.0)

{data_science_document_ai-1.41.0 → data_science_document_ai-1.45.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "data-science-document-ai"
-version = "1.41.0"
+version = "1.45.2"
 description = "\"Document AI repo for data science\""
 authors = ["Naomi Nguyen <naomi.nguyen@forto.com>", "Kumar Rajendrababu <kumar.rajendrababu@forto.com>", "Igor Tonko <igor.tonko@forto.com>", "Osman Demirel <osman.demirel@forto.com>"]
 packages = [
@@ -48,7 +48,7 @@ rapidfuzz = "^3.12.2"
 fuzzywuzzy = "^0.18.0"
 nltk = "^3.9.1"
 pgzip = "^0.3.5"
-pypdf2 = "^3.0.1"
+pypdf = "^6.1.2"
 
 [tool.poetry.dev-dependencies]
 jupyter = "^1.0.0"

{data_science_document_ai-1.41.0 → data_science_document_ai-1.45.2}/src/constants.py

@@ -23,9 +23,12 @@ project_parameters = {
     "invoice_classification_lookup": "invoice_classification.json",
     "reverse_charge_sentence_lookup": "reverse_charge_sentences.json",
     # Fuzzy logic params
-    "fuzzy_threshold_item_code": 70,
+    "fuzzy_threshold_item_code": 90,
     "fuzzy_threshold_reverse_charge": 80,
     "fuzzy_threshold_invoice_classification": 70,
+    # Chunking params
+    "chunk_size": 1,  # page (do not change this without changing the page number logic)
+    "chunk_after": 10,  # pages
     # Big Query
     "g_ai_gbq_db_schema": "document_ai",
     "g_ai_gbq_db_table_out": "document_ai_api_calls_v1",
@@ -50,8 +53,6 @@ project_parameters = {
     "model_selector": {
         "stable": {
             "bookingConfirmation": 1,
-            "packingList": 0,
-            "commercialInvoice": 0,
             "finalMbL": 0,
             "draftMbl": 0,
             "arrivalNotice": 0,

{data_science_document_ai-1.41.0 → data_science_document_ai-1.45.2}/src/docai.py

@@ -3,11 +3,16 @@ import re
 
 from google.cloud import documentai
 
-from src.io import delete_folder_from_bucket, logger, upload_pdf_to_bucket
+from src.io import (
+    delete_folder_from_bucket,
+    get_gcp_labels,
+    logger,
+    upload_pdf_to_bucket,
+)
 from src.utils import cache_on_disk
 
 
-async def _process_pdf_w_docai(image_content, client, processor_name):
+async def _process_pdf_w_docai(image_content, client, processor_name, doc_type=None):
     """Process the PDF using Document AI.
 
     Args:
@@ -15,6 +20,7 @@ async def _process_pdf_w_docai(image_content, client, processor_name):
         client: The Document AI client.
         processor_name (str): The name of the processor to be used.
                             e.g.: projects/{project_id}/locations/{location}/processor/{processor_id}
+        doc_type (str, optional): Document type for cost tracking labels.
 
     Returns:
         The processed document.
@@ -24,10 +30,11 @@ async def _process_pdf_w_docai(image_content, client, processor_name):
         content=image_content, mime_type="application/pdf"
     )
 
-    # Configure the process request
+    # Configure the process request with labels for cost tracking
     request = documentai.ProcessRequest(
         name=processor_name,
         raw_document=raw_document,  # field_mask=field_mask
+        labels=get_gcp_labels(doc_type=doc_type),
     )
     result = await cache_on_disk(client.process_document, request=request)
 
@@ -35,7 +42,7 @@ async def _process_pdf_w_docai(image_content, client, processor_name):
 
 
 async def _batch_process_pdf_w_docai(
-    params, image_content, client, processor_name, timeout=1200
+    params, image_content, client, processor_name, timeout=1200, doc_type=None
 ):
     """Process the PDF using Document AI Batch Process API.
 
@@ -45,6 +52,7 @@ async def _batch_process_pdf_w_docai(
         processor_name (str): The name of the processor to be used.
                             e.g.: projects/{project_id}/locations/{location}/processor/{processor_id}
         timeout (int, optional): The timeout in seconds. Defaults to 1200.
+        doc_type (str, optional): Document type for cost tracking labels.
 
     Returns:
         The processed document.
@@ -72,11 +80,12 @@ async def _batch_process_pdf_w_docai(
     # Where to write results
     output_config = documentai.DocumentOutputConfig(gcs_output_config=gcs_output_config)
 
-    # The full resource name of the processor
+    # The full resource name of the processor with labels for cost tracking
     request = documentai.BatchProcessRequest(
         name=processor_name,
         input_documents=input_config,
         document_output_config=output_config,
+        labels=get_gcp_labels(doc_type=doc_type),
     )
 
     # BatchProcess returns a Long Running Operation (LRO)

{data_science_document_ai-1.41.0 → data_science_document_ai-1.45.2}/src/docai_processor_config.yaml

@@ -13,20 +13,6 @@ model_config:
           author: "igor.tonko@forto.com"
           created_date: ""
 
-    packingList:
-      - id: "d967005bd9d45aeb"
-        details:
-          display_name: "doc_cap_packingList"
-          author: "kumar.rajendrababu@forto.com"
-          created_date: ""
-
-    commercialInvoice:
-      - id: "7d37236207f75758"
-        details:
-          display_name: "doc_cap_commercialInvoice"
-          author: "kumar.rajendrababu@forto.com"
-          created_date: ""
-
     finalMbL:
       - id: "1eda2f22d64b1b89"
         details:

{data_science_document_ai-1.41.0 → data_science_document_ai-1.45.2}/src/excel_processing.py

@@ -2,23 +2,26 @@
 # flake8: noqa: E402
 import logging
 
-from src.postprocessing.common import llm_prediction_to_tuples
+from ddtrace import tracer
 
 logger = logging.getLogger(__name__)
 
 import asyncio
-import json
 
 import numpy as np
 import pandas as pd
 
 from src.llm import prompt_excel_extraction
-from src.utils import generate_schema_structure, get_excel_sheets
+from src.prompts.prompt_library import prompt_library
+from src.utils import estimate_page_count, generate_schema_structure, get_excel_sheets
 
 
-async def extract_data_from_sheet(params, sheet_name, sheet, response_schema):
+async def extract_data_from_sheet(
+    llm_client, sheet_name, sheet, response_schema, doc_type=None
+):
     logger.info(f"Processing sheet: {sheet_name}")
-    excel_content = pd.DataFrame(sheet.values)
+    excel_content = pd.DataFrame(sheet.values).dropna(how="all", axis=1)
+
     # Convert to Markdown format for the LLM model
     worksheet = (
         "This is from a excel. Pay attention to the cell position:\n"
@@ -29,9 +32,10 @@ async def extract_data_from_sheet(params, sheet_name, sheet, response_schema):
     prompt_docai = prompt_excel_extraction(worksheet)
 
     try:
-        result = await params["LlmClient"].get_unified_json_genai(
+        result = await llm_client.get_unified_json_genai(
             prompt_docai,
             response_schema=response_schema,
+            doc_type=doc_type,
         )
     except Exception as e:
         result = {}
@@ -45,6 +49,7 @@ async def extract_data_from_excel(
     input_doc_type,
     file_content,
     mime_type,
+    llm_client,
 ):
     """Extract data from the Excel file.
 
@@ -53,6 +58,7 @@ async def extract_data_from_excel(
         input_doc_type (str): The type of the document.
         file_content (bytes): The content of the Excel file to process.
         mime_type (str): The MIME type of the file.
+        llm_client: The LLM client to use for data extraction.
 
     Returns:
         formatted_data (list): A list of dictionaries containing the extracted data.
@@ -61,22 +67,43 @@ async def extract_data_from_excel(
 
     """
     # Generate the response structure
-    response_schema = generate_schema_structure(params, input_doc_type)
+    response_schema = (
+        prompt_library.library[input_doc_type]["other"]["placeholders"]
+        if input_doc_type
+        in [
+            "partnerInvoice",
+            "customsInvoice",
+            "bundeskasse",
+            "commercialInvoice",
+            "packingList",
+            "bookingConfirmation",
+        ]
+        else generate_schema_structure(params, input_doc_type)
+    )
 
     # Load the Excel file and get ONLY the "visible" sheet names
     sheets, workbook = get_excel_sheets(file_content, mime_type)
 
+    # Track the number of sheets in dd-trace
+    span = tracer.current_span()
+    if span:
+        estimated_page_counts = [
+            estimate_page_count(workbook[sheet]) for sheet in sheets
+        ]
+        est_page_count = sum(estimated_page_counts)
+        span.set_metric("est_page_count", est_page_count)
+
     # Excel files may contain multiple sheets. Extract data from each sheet
     sheet_extract_tasks = [
         extract_data_from_sheet(
-            params, sheet_name, workbook[sheet_name], response_schema
+            llm_client,
+            sheet_name,
+            workbook[sheet_name],
+            response_schema,
+            doc_type=input_doc_type,
         )
         for sheet_name in sheets
     ]
     extracted_data = {k: v for k, v in await asyncio.gather(*sheet_extract_tasks)}
 
-    # Convert LLM prediction dictionary to tuples of (value, page_number).
-    extracted_data = llm_prediction_to_tuples(extracted_data)
-    stored_data = json.dumps(extracted_data)
-
-    return extracted_data, stored_data, params["gemini_params"]["model_id"]
+    return extracted_data, extracted_data, llm_client.model_id

{data_science_document_ai-1.41.0 → data_science_document_ai-1.45.2}/src/io.py

@@ -12,13 +12,55 @@ from pathlib import Path
 from google.cloud import bigquery, storage
 
 
+def get_gcp_labels(**extra_labels):
+    """Generate standardized GCP labels for cost tracking.
+
+    Args:
+        **extra_labels: Additional custom labels
+
+    Returns:
+        dict: Labels dictionary with keys normalized (lowercase, hyphens, max 63 chars)
+    """
+    project_name = os.getenv("PROJECT_NAME")
+
+    # If not set, detect once and cache it
+    if not project_name:
+        # Try pyproject.toml first
+        try:
+            import toml
+
+            pyproject_path = Path(__file__).parent.parent / "pyproject.toml"
+            if pyproject_path.exists():
+                config = toml.load(pyproject_path)
+                project_name = config.get("tool", {}).get("poetry", {}).get("name")
+        except Exception:
+            pass
+
+        # Fallback to unknown
+        if not project_name:
+            project_name = "unknown"
+
+        # Cache it
+        os.environ["PROJECT_NAME"] = project_name
+
+    labels = {
+        "ds-project-name": project_name.lower(),
+        "ds-env": os.getenv("CLUSTER", "local").lower(),
+    }
+
+    # Add any extra labels
+    labels.update({k.lower(): str(v).lower() for k, v in extra_labels.items()})
+
+    return labels
+
+
 def get_bq_client(params):
     """Get Google BigQuery client."""
     bq_client = bigquery.Client(project=params["g_ai_project_name"])
     job_config = bigquery.QueryJobConfig(
         allow_large_results=True,
         # flatten_results=True,
-        labels={"project-name": params["project_name"]},
+        labels=get_gcp_labels(),
     )
     return bq_client, job_config
 
@@ -112,3 +154,29 @@ def download_dir_from_bucket(bucket, directory_cloud, directory_local) -> bool:
         Path(directory).mkdir(parents=True, exist_ok=True)
         blob.download_to_filename(directory_local / Path(blob.name))
     return result
+
+
+def bq_logs(data_to_insert, params):
+    """Insert logs into Google BigQuery.
+
+    Args:
+        data_to_insert (list): The data to insert into BigQuery.
+        params (dict): The parameters dictionary.
+    """
+    # Use the pre-initialized BigQuery client
+    bq_client = params["bq_client"]
+    # Get the table string
+    table_string = f"{params['g_ai_project_name']}.{params['g_ai_gbq_db_schema']}.{params['g_ai_gbq_db_table_out']}"
+
+    logger.info(f"Log table: {table_string}")
+    # Insert the rows into the table
+    insert_logs = bq_client.insert_rows_json(table_string, data_to_insert)
+
+    # Check if there were any errors inserting the rows
+    if not insert_logs:
+        logger.info("New rows have been added.")
+    else:
+        logger.info("Errors occurred while inserting rows: ", insert_logs)
+
+
+# type: ignore

{data_science_document_ai-1.41.0 → data_science_document_ai-1.45.2}/src/llm.py

@@ -15,6 +15,7 @@ from vertexai.generative_models import (
     Part,
 )
 
+from src.io import get_gcp_labels
 from src.utils import cache_on_disk
 
 
@@ -69,6 +70,7 @@ class LlmClient:
         document: str = None,
         response_schema: dict = None,
         response_mime_type: str = "application/json",
+        doc_type: str = None,
     ):
         """Ask the Gemini model a question.
 
@@ -76,6 +78,7 @@ class LlmClient:
             prompt (str): The prompt to send to the model.
             document (str, optional): An optional document to provide context.
             response_schema (dict, optional): Defines a specific response schema for the model.
+            doc_type (str, optional): Document type for cost tracking labels.
 
         Returns:
             str: The response from the model.
@@ -96,12 +99,13 @@ class LlmClient:
             # Prepare inputs for the model
             inputs = [document, prompt] if document else prompt
 
-            # Generate the response
+            # Generate the response with labels for cost tracking
             model_response = await cache_on_disk(
                 self.geminy_client.generate_content_async,
                 contents=inputs,
                 generation_config=config,
                 safety_settings=self.safety_config,
+                labels=get_gcp_labels(doc_type=doc_type),
             )
 
             response_text = model_response.text
@@ -113,7 +117,7 @@ class LlmClient:
             return "{}"
 
     async def get_unified_json_genai(
-        self, prompt, document=None, response_schema=None, model="gemini"
+        self, prompt, document=None, response_schema=None, model="gemini", doc_type=None
     ):
         """Send a prompt to a Google Cloud AI Platform model and returns the generated json.
 
@@ -122,6 +126,7 @@ class LlmClient:
             document: Content of the PDF document
             response_schema: The schema to use for the response
             model (str): The model to use for the response ["gemini" or "chatGPT"]. Default is "gemini".
+            doc_type (str, optional): Document type for cost tracking labels.
 
         Returns:
             dict: The generated json from the model.
@@ -131,7 +136,9 @@ class LlmClient:
             response = await self.ask_chatgpt(prompt, document, response_schema)
         else:
             # Default to Gemini
-            response = await self.ask_gemini(prompt, document, response_schema)
+            response = await self.ask_gemini(
+                prompt, document, response_schema, doc_type=doc_type
+            )
 
         try:
             return json.loads(response)