data-science-document-ai 1.13.0__py3-none-any.whl → 1.56.1__py3-none-any.whl

src/setup.py

@@ -1,11 +1,8 @@
 """Contains project setup parameters and initialization functions."""
-import argparse
-
-# import streamlit as st
+import json
 import os
 import random
 import time
-from pathlib import Path
 
 import toml
 import vertexai
@@ -18,7 +15,7 @@ from src.constants import project_parameters
 from src.constants_sandbox import project_parameters_sandbox
 
 # Parent repos are imported without .
-from src.io import download_dir_from_bucket, get_storage_client, logger
+from src.io import get_bq_client, get_storage_client, logger
 from src.llm import LlmClient
 
 
@@ -68,52 +65,16 @@ def get_docai_schema_client(params, async_=True):
     return client
 
 
-def parse_input():
-    """Manage input parameters."""
-    parser = argparse.ArgumentParser(description="", add_help=False)
-    parser.add_argument(
-        "--scope",
-        type=str,
-        dest="scope",
-        required=False,
-        help="Whether the function should 'upload' or 'download' documents",
-    )
-    parser.add_argument(
-        "--document_name",
-        type=str,
-        dest="document_name",
-        required=False,
-        help="Category of the document (e.g., 'commercialInvoice', 'packingList')",
-    )
-    parser.add_argument(
-        "--for_combinations",
-        type=bool,
-        default=False,
-        dest="for_combinations",
-        required=False,
-        help="A flag to download documents into a special subfolder",
-    )
-    parser.add_argument(
-        "--n_samples",
-        type=int,
-        default=50,
-        dest="n_samples",
-        required=False,
-        help="A number of samples to download",
-    )
-
-    # Remove declared missing arguments (e.g. model_type)
-    args = vars(parser.parse_args())
-    args_no_null = {
-        k: v.split(",") if isinstance(v, str) else v
-        for k, v in args.items()
-        if v is not None
-    }
-    return args_no_null
+def setup_params(args=None):
+    """
+    Set up the application parameters.
 
+    Args:
+        args: Command-line arguments.
 
-def setup_params(args=None):
-    """Manage setup parameters."""
+    Returns:
+        params: Dictionary containing application parameters.
+    """
     if args is None:
         args = {}
 
@@ -123,13 +84,22 @@ def setup_params(args=None):
     # Update parameters with constants
     params.update(project_parameters)
 
-    # Update the parameters with the sandbox parameters if the cluster is not production
-    if os.getenv("CLUSTER") != "production":
+    cluster = os.getenv("CLUSTER", "").lower()
+    # Update the parameters with the sandbox parameters if the cluster is not production and not ODE
+    if cluster not in ("production", "ode"):
         params.update(project_parameters_sandbox)
 
-    params["environment"] = (
-        "sandbox" if os.getenv("CLUSTER") != "production" else "production"
-    )
+    # Set up the bucket constants for ODE environment
+    if cluster == "ode":
+        ode_env_vars = {
+            "doc_ai_bucket_project_name": "PROJECT_ID",
+            "doc_ai_bucket_name": "BUCKET_NAME",
+            "doc_ai_bucket_batch_input": "INPUT_BUCKET_NAME",
+            "doc_ai_bucket_batch_output": "OUTPUT_BUCKET_NAME",
+        }
+        params.update(
+            {key: os.getenv(env_var) for key, env_var in ode_env_vars.items()}
+        )
 
     # print cluster info
     logger.info(f"Cluster: {os.getenv('CLUSTER')}")
@@ -145,6 +115,10 @@ def setup_params(args=None):
 
     params = setup_docai_client_and_path(params)
 
+    # Set up BigQuery client for logging
+    bq_client, _ = get_bq_client(params)
+    params["bq_client"] = bq_client
+
     # Set up Vertex AI for text embeddings
     setup_vertexai(params)
 
@@ -156,18 +130,16 @@ def setup_params(args=None):
         assert params.keys() & yaml_content.keys() == set()
         params.update(yaml_content)
 
-    # Get models meta data from cloud
-    client = get_storage_client(params)
-    bucket = client.bucket(params["doc_ai_bucket_name"])
-    downloaded_meta = download_dir_from_bucket(
-        bucket, params["g_model_data_folder"], Path(params["local_model_data_folder"])
-    )
-    if not downloaded_meta:
-        logger.info(f"Could not load models metadata from cloud.")
-
+    # Set up LLM clients
     params["LlmClient"] = LlmClient(
         openai_key=os.getenv("OPENAI_KEY"), parameters=params["gemini_params"]
     )
+    params["LlmClient_Flash"] = LlmClient(
+        openai_key=os.getenv("OPENAI_KEY"), parameters=params["gemini_flash_params"]
+    )
+
+    # Load lookup data from GCS bucket
+    setup_lookup_data(params)
 
     return params
 
@@ -198,3 +170,41 @@ def setup_vertexai(params):
         project=params["g_ai_project_name"],
         location=params["g_region"],
     )
+
+
+def setup_lookup_data(params):
+    """
+    Loads JSON mapping data from given GCP Bucket.
+    """
+    client = get_storage_client(params)
+    bucket = client.bucket(params["doc_ai_bucket_name"])
+
+    data = dict()
+
+    input_path_item_code = (
+        f'{params["g_model_fuzzy_lookup_folder"]}/{params["item_code_lookup"]}'
+    )
+    input_path_intermodal_partners = (
+        f'{params["g_model_fuzzy_lookup_folder"]}/{params["intermodal_partners"]}'
+    )
+    input_path_invoice_classification = f'{params["g_model_fuzzy_lookup_folder"]}/{params["invoice_classification_lookup"]}'  # noqa: E501
+    input_path_reverse_charge = f'{params["g_model_fuzzy_lookup_folder"]}/{params["reverse_charge_sentence_lookup"]}'
+
+    def download_json_from_bucket(path):
+        """Download JSON data from a specified path in a GCP bucket."""
+        blob = bucket.blob(path)
+        downloaded_data = blob.download_as_text(encoding="utf-8")
+        return json.loads(downloaded_data)
+
+    data["item_code"] = download_json_from_bucket(input_path_item_code)
+    data["intermodal_partners"] = download_json_from_bucket(
+        input_path_intermodal_partners
+    )
+    data["invoice_classification"] = download_json_from_bucket(
+        input_path_invoice_classification
+    )
+    data["reverse_charge_sentences"] = download_json_from_bucket(
+        input_path_reverse_charge
+    )
+
+    params["lookup_data"] = data

src/utils.py

@@ -6,36 +6,29 @@ import json
 import os
 import pickle
 from datetime import datetime
-from typing import Literal
+from typing import Any, Dict, List, Literal, Optional
 
+import httpx
+import numpy as np
 import openpyxl
 import pandas as pd
 from google.cloud import documentai_v1beta3 as docu_ai_beta
+from pypdf import PdfReader, PdfWriter
 
-from src.io import get_bq_client, get_storage_client, logger
+from src.io import bq_logs, get_storage_client, logger
 
 
-def bq_logs(data_to_insert, params):
-    """Insert logs into Google BigQuery.
+def get_pdf_page_count(pdf_bytes):
+    """Get the number of pages in a PDF document efficiently.
 
     Args:
-        data_to_insert (list): The data to insert into BigQuery.
-        params (dict): The parameters dictionary.
+        pdf_bytes (bytes): The PDF content as bytes.
+
+    Returns:
+        int: The number of pages in the PDF.
     """
-    # Get the BigQuery client
-    bq_client, config = get_bq_client(params)
-    # Get the table string
-    table_string = f"{params['g_ai_project_name']}.{params['g_ai_gbq_db_schema']}.{params['g_ai_gbq_db_table_out']}"
-
-    logger.info(f"Log table: {table_string}")
-    # Insert the rows into the table
-    insert_logs = bq_client.insert_rows_json(table_string, data_to_insert)
-
-    # Check if there were any errors inserting the rows
-    if not insert_logs:
-        logger.info("New rows have been added.")
-    else:
-        logger.info("Errors occurred while inserting rows: ", insert_logs)
+    reader = PdfReader(io.BytesIO(pdf_bytes))
+    return len(reader.pages)
 
 
 async def get_data_set_schema_from_docai(
@@ -137,7 +130,12 @@ def store_json_in_gcs(
         bucket = storage_client.bucket(params.get("doc_ai_bucket_name"))
         full_object_name = folder_path + document_id
         blob = bucket.blob(full_object_name)
-        blob.upload_from_string(json_data, content_type="application/json")
+
+        # Convert dict to JSON string if needed
+        json_string = (
+            json.dumps(json_data) if isinstance(json_data, dict) else json_data
+        )
+        blob.upload_from_string(json_string, content_type="application/json")
 
         logger.info(
             f"JSON object stored successfully in gs://{params.get('doc_ai_bucket_name')}/{full_object_name}"  # noqa
@@ -156,6 +154,8 @@ async def run_background_tasks(
     store_data,
     processor_version,
     mime_type,
+    elapsed_time=None,
+    page_count=None,
 ):
     """
     Run background tasks asynchronously.
@@ -168,6 +168,8 @@ async def run_background_tasks(
         store_data: The data to store in GCS.
         processor_version: The processor version used to extract the data.
         mime_type: The MIME type of the document.
+        elapsed_time: The time taken to process the document.
+        page_count (int, optional): The number of pages in the document.
 
     Returns:
         None
@@ -176,13 +178,8 @@ async def run_background_tasks(
 
     await loop.run_in_executor(None, store_json_in_gcs, params, doc_id, store_data)
 
-    # Keep the page count as 1 for Excel files.
-    page_count = 1
-    # calculate the number of pages processed for PDFs
-    try:
-        if mime_type == "application/pdf":
-            page_count = len(json.loads(store_data.encode("utf-8"))["pages"])
-    except AttributeError:
+    # Use the passed page_count or default to 0 if not provided
+    if page_count is None:
         page_count = 0
 
     # Log the request in BigQuery
@@ -200,6 +197,7 @@ async def run_background_tasks(
                 "processor_version": processor_version,
                 "page_count": page_count,
                 "mime_type": mime_type,
+                "elapsed_time": elapsed_time,
             }
         ],
         params,
@@ -240,7 +238,6 @@ def generate_schema_structure(params, input_doc_type):
     Args:
         params (dict): Parameters dictionary.
         input_doc_type (str): Document type to select the appropriate schema.
-        schema_client (documentai_v1beta3.DocumentServiceClient): Schema client.
 
     Returns:
         dict: The response schema structure.
@@ -254,7 +251,7 @@ def generate_schema_structure(params, input_doc_type):
             "type": "OBJECT",
             "properties": {
                 prop.name: {
-                    "type": prop.value_type,
+                    "type": "string",
                     "nullable": True,
                     "description": prop.description,
                 }
@@ -347,3 +344,183 @@ async def update_response_schema_from_docai(params, schema_client):
 
 def get_data_set_schema(params, processor_name):
     return params["docai_schema_dict"][processor_name]
+
+
+def extract_top_pages(pdf_bytes, num_pages=4):
+    """Extract the top pages from a PDF document."""
+    reader = PdfReader(io.BytesIO(pdf_bytes))
+    writer = PdfWriter()
+
+    for page_num in range(min(num_pages, len(reader.pages))):
+        writer.add_page(reader.pages[page_num])
+
+    output = io.BytesIO()
+    writer.write(output)
+
+    return output.getvalue()
+
+
+async def get_tms_mappings(
+    input_list: List[str], embedding_type: str, llm_ports: Optional[List[str]] = None
+) -> Dict[str, Any]:
+    """Get TMS mappings for the given values.
+
+    Args:
+        input_list (list[str]): List of strings to get embeddings for.
+        embedding_type (str): Type of embedding to use
+         (e.g., "container_types", "ports", "depots", "lineitems", "terminals").
+        llm_ports (list[str], optional): List of LLM ports to use. Defaults to None.
+
+    Returns:
+        dict or string: A dictionary or a string with the mapping results.
+    """
+    base_url = (
+        "http://0.0.0.0:8080/"
+        if os.getenv("CLUSTER") is None
+        else "http://tms-mappings.api.svc.cluster.local./"
+    )
+
+    # Ensure clean inputs
+    if not input_list:
+        return {}
+
+    # Ensure input_list is a list
+    if not isinstance(input_list, list):
+        input_list = [input_list]
+
+    # Always send a dict with named keys
+    payload = {embedding_type: input_list}
+
+    if llm_ports:
+        payload["llm_ports"] = llm_ports if isinstance(llm_ports, list) else [llm_ports]
+
+    # Make the POST request to the TMS mappings API
+    url = f"{base_url}{embedding_type}"
+
+    # Use a timeout so the code doesn't hang forever
+    timeout = httpx.Timeout(60.0, connect=10.0)
+
+    async with httpx.AsyncClient(timeout=timeout) as client:
+        try:
+            response = await client.post(url, json=payload)
+            response.raise_for_status()
+
+            # Structure expected: {"response": {"data": {"desc1": "code1", "desc2": "code2"}}}
+            return response.json().get("response", {}).get("data", {})
+
+        except httpx.HTTPStatusError as exc:
+            logger.error(
+                f"Error from TMS mappings API: {exc.response.status_code} - {exc.response.text}"
+            )
+            return {}
+
+
+async def batch_fetch_all_mappings(container_types, terminals, depots):
+    """Batch fetch all mappings for container types, terminals, and depots."""
+    # run batch calls concurrently
+    results = await asyncio.gather(
+        get_tms_mappings(list(container_types), "container_types"),
+        get_tms_mappings(list(terminals), "terminals"),
+        get_tms_mappings(list(depots), "depots"),
+    )
+
+    batch_container_map, batch_terminal_map, batch_depot_map = results
+
+    # Convert lists of tuples to dicts if necessary
+    return (
+        dict(batch_container_map or {}),
+        dict(batch_terminal_map or {}),
+        dict(batch_depot_map or {}),
+    )
+
+
+def transform_schema_strings(schema):
+    """
+    Recursively transforms a schema dictionary, replacing all "type": "STRING"
+    definitions with a new object containing "value" and "page_number" fields.
+    It preserves 'nullable' and 'description' fields by moving them to the
+    new 'value' property.
+
+    Args:
+        schema (dict): The input schema dictionary.
+
+    Returns:
+        dict: The transformed schema dictionary.
+    """
+    if not isinstance(schema, dict):
+        return schema
+
+    schema_type = schema.get("type")
+    if not schema_type:
+        return schema
+
+    # Base case: STRING → OBJECT (only if not already transformed)
+    if schema_type.upper() == "STRING":
+        return {
+            "type": "OBJECT",
+            "properties": {
+                "value": {
+                    "type": "STRING",
+                    "nullable": schema.get("nullable", False),
+                    "description": schema.get("description", ""),
+                },
+                "page_number": {
+                    "type": "STRING",
+                    "description": "Number of a page where the value was found in the document starting from 0.",
+                },
+            },
+            "required": [],
+        }
+
+    # Skip already transformed OBJECT (has both 'value' & 'page_number')
+    if (
+        schema_type.upper() == "OBJECT"
+        and "properties" in schema
+        and {"value", "page_number"}.issubset(schema["properties"].keys())
+    ):
+        return schema
+
+    # Recursive case for OBJECT
+    if schema_type.upper() == "OBJECT" and "properties" in schema:
+        new_schema = schema.copy()
+        new_schema["properties"] = {
+            k: transform_schema_strings(v) for k, v in schema["properties"].items()
+        }
+        return new_schema
+
+    # Recursive case for ARRAY
+    if schema_type.upper() == "ARRAY" and "items" in schema:
+        new_schema = schema.copy()
+        new_schema["items"] = transform_schema_strings(schema["items"])
+        return new_schema
+
+    return schema
+
+
+def estimate_page_count(sheet):
+    """Assuming a page is 10 columns x 50 rows."""
+    if hasattr(sheet, "shape"):
+        pg_cnt = sheet.shape[0] * sheet.shape[1]
+    elif hasattr(sheet, "max_row"):
+        pg_cnt = sheet.max_column * sheet.max_row
+    else:
+        return None
+    return np.ceil(pg_cnt / 500)
+
+
+def split_pdf_into_chunks(file_content: bytes, chunk_size: int = 1):
+    """Split PDF into smaller page chunks."""
+    pdf = PdfReader(io.BytesIO(file_content))
+    total_pages = len(pdf.pages)
+
+    # TODO: update the chunk_size based on doc length. However, it breaks the page number extraction logic.
+    for i in range(0, total_pages, chunk_size):
+        writer = PdfWriter()
+        for j in range(i, min(i + chunk_size, total_pages)):
+            writer.add_page(pdf.pages[j])
+
+        buffer = io.BytesIO()
+        writer.write(buffer)
+        buffer.seek(0)
+
+        yield buffer.getvalue()

data_science_document_ai-1.13.0.dist-info/RECORD

@@ -1,55 +0,0 @@
-src/constants.py,sha256=AP5ZfxMGU745IUcSRR0z7aTssbAuJuyqhnVNN9I0L1I,2524
-src/constants_sandbox.py,sha256=vdEOaFzeUmsKK-K66BDgfw3R-_MeQ8XNo3bIfF4EOmA,1241
-src/docai.py,sha256=1UxBRO0oC7WbFgscQAyxjmhsvh-Oc8g60m368WFZrOw,5234
-src/docai_processor_config.yaml,sha256=_XN0g7t9EGU9-vVmK9_t_IQ6OUkXOvawYlOjSSHOUtQ,2295
-src/excel_processing.py,sha256=HZGIinyYXFRRAekBj0yBcTaI0MhzdGuxsTTYpEnzRm8,2559
-src/io.py,sha256=8DxtfvsNrx7QCVPQwttGX21o0NthxHfH6zBR6X4COvg,3511
-src/llm.py,sha256=93naoL3wviBtrA5JaQxldW6hO_Cwpc61whNuz881fDQ,7828
-src/log_setup.py,sha256=RhHnpXqcl-ii4EJzRt47CF2R-Q3YPF68tepg_Kg7tkw,2895
-src/pdf_processing.py,sha256=cVLrd-gZmbr9p_Od2ihC2LXxfW7pPhMS6SzhdKS4snM,14962
-src/postprocessing/common.py,sha256=zmTs97KYJhOvmURP_U7RlJUxzqV--Aw62qY78XA-Tl8,33760
-src/postprocessing/postprocess_booking_confirmation.py,sha256=nK32eDiBNbauyQz0oCa9eraysku8aqzrcoRFoWVumDU,4827
-src/postprocessing/postprocess_commercial_invoice.py,sha256=3I8ijluTZcOs_sMnFZxfkAPle0UFQ239EMuvZfDZVPg,1028
-src/postprocessing/postprocess_partner_invoice.py,sha256=lwHr9pWRQ3LoclZbL1g4_3HCGCWk_C0C_UvbpxYIgKI,2374
-src/prompts/library/bookingConfirmation/evergreen/placeholders.json,sha256=Re2wBgZoaJ5yImUUAwZOZxFcKXHxi83TCZwTuqd2v2k,1405
-src/prompts/library/bookingConfirmation/evergreen/prompt.txt,sha256=qlBMFDHy-gwr2PVeuHrfMEg_8Ibdym243DnaCgINa7g,2614
-src/prompts/library/bookingConfirmation/hapag-lloyd/placeholders.json,sha256=Re2wBgZoaJ5yImUUAwZOZxFcKXHxi83TCZwTuqd2v2k,1405
-src/prompts/library/bookingConfirmation/hapag-lloyd/prompt.txt,sha256=sg11U3lIhhS36BsimX7IOzR7Pez_9gScdNmJna2pPuw,3355
-src/prompts/library/bookingConfirmation/maersk/placeholders.json,sha256=PKWXySGAls6A8tujbSjokYp4ldc3c0DmSP2ITKYiUF8,1405
-src/prompts/library/bookingConfirmation/maersk/prompt.txt,sha256=-00tzWzXtQnXX3EPtaCBM39leCoLa4FB52_t7Z3eoQk,3148
-src/prompts/library/bookingConfirmation/msc/placeholders.json,sha256=Re2wBgZoaJ5yImUUAwZOZxFcKXHxi83TCZwTuqd2v2k,1405
-src/prompts/library/bookingConfirmation/msc/prompt.txt,sha256=9wdbLofnp5s1acD19jCmQuw__HMcVq1yr4vIJNJlKVM,4630
-src/prompts/library/bookingConfirmation/oocl/placeholders.json,sha256=NnXjMiEsTCzTDWs2WY7BIMo2p4_98-DL3v1r7x-FL3A,1403
-src/prompts/library/bookingConfirmation/oocl/prompt.txt,sha256=aGowVvOgl4w6TjX5O2RtD4QOiWC1JnXiWgg0t0chThU,2060
-src/prompts/library/bookingConfirmation/other/placeholders.json,sha256=Re2wBgZoaJ5yImUUAwZOZxFcKXHxi83TCZwTuqd2v2k,1405
-src/prompts/library/bookingConfirmation/other/prompt.txt,sha256=XOrq5Ns0nl8lDI9VvoOEbIMbOQdv8mcM8HqP8-eIjc4,3304
-src/prompts/library/bookingConfirmation/yangming/placeholders.json,sha256=Re2wBgZoaJ5yImUUAwZOZxFcKXHxi83TCZwTuqd2v2k,1405
-src/prompts/library/bookingConfirmation/yangming/prompt.txt,sha256=BSFy-6zDlAmOH2uZjsp-zZkR_Uy3RS5sGtdv9wysiSI,3151
-src/prompts/library/commercialInvoice/other/prompt.txt,sha256=fYUF7btc48Uqv4mJH5BgJdY4JVwj9I1xKX_HRBIrN7M,2590
-src/prompts/library/customsAssessment/other/placeholders.json,sha256=5nSGsMbpfKrpKoYImcTto_RlOvPCHyld2RlwU0Zbbqw,361
-src/prompts/library/customsAssessment/other/prompt.txt,sha256=wgJ8PYM0PKXiIKSljhFXEFBQ23GRs2E2DE9lVwHDvBU,2116
-src/prompts/library/deliveryOrder/other/placeholders.json,sha256=6b_6OVsxT7bjFnV_v0OZkGEy-GN5K4AjL0ATzuoLdOU,1286
-src/prompts/library/deliveryOrder/other/prompt.txt,sha256=MVSS5AhkiWT17G9X4xk_AgKzYElagvWjLPCMr_ZhmOs,2393
-src/prompts/library/draftMbl/hapag-lloyd/prompt.txt,sha256=0k1xLW4zWaenCSNQJxXMXenIwI-eYmGgpxnAAcM3HOg,2251
-src/prompts/library/draftMbl/maersk/prompt.txt,sha256=GxaIYlksORvD2uAbodRx_9JFJXD4XbDaVFYtpN9uzxc,2050
-src/prompts/library/draftMbl/other/placeholders.json,sha256=wIN06_NWsESDyNEDfOLPi3F2Vq-XPa4O3U32A32s-_Q,1736
-src/prompts/library/draftMbl/other/prompt.txt,sha256=gqbPm1joXKDUss0wU6vMc-269sx-fYWh90gWuNKOBQc,2166
-src/prompts/library/finalMbL/hapag-lloyd/prompt.txt,sha256=0k1xLW4zWaenCSNQJxXMXenIwI-eYmGgpxnAAcM3HOg,2251
-src/prompts/library/finalMbL/maersk/prompt.txt,sha256=GxaIYlksORvD2uAbodRx_9JFJXD4XbDaVFYtpN9uzxc,2050
-src/prompts/library/finalMbL/other/placeholders.json,sha256=K_yJYhQo2DnZV_Rg6xXjo6sHkSGB-SMO4IQnY47V43w,1735
-src/prompts/library/finalMbL/other/prompt.txt,sha256=gqbPm1joXKDUss0wU6vMc-269sx-fYWh90gWuNKOBQc,2166
-src/prompts/library/packingList/other/prompt.txt,sha256=Qw16n7_48GGFYWz2vRepNowZCX1UPXKetEZ1UqFXPdY,2764
-src/prompts/library/partnerInvoice/other/placeholders.json,sha256=tXkr1VVeb1qqAR0SSWYrKu1Np3LXB9o4_2Ponsu0e4k,2352
-src/prompts/library/partnerInvoice/other/prompt.txt,sha256=Ih5VSfVBBYbo2_ufyYvp1DNYoXDYCScw8b1ylVbftsQ,2913
-src/prompts/library/postprocessing/port_code/placeholders.json,sha256=2TiXf3zSzrglOMPtDOlCntIa5RSvyZQAKG2-IgrCY5A,22
-src/prompts/library/postprocessing/port_code/prompt_port_code.txt,sha256=--1wunSqEr2ox958lEhjO-0JFBfOLzA3qfKYIzG_Iok,884
-src/prompts/library/preprocessing/carrier/placeholders.json,sha256=1UmrQNqBEsjLIpOO-a39Az6bQ_g1lxDGlwqZFU3IEt0,408
-src/prompts/library/preprocessing/carrier/prompt.txt,sha256=NLvRZQCZ6aWC1yTr7Q93jK5z7Vi_b4HBaiFYYnIsO-w,134
-src/prompts/library/shippingInstruction/other/prompt.txt,sha256=fyC24ig4FyRNnLuQM69s4ZVajsK-LHIl2dvaaEXr-6Q,1327
-src/prompts/prompt_library.py,sha256=VJWHeXN-s501C2GiidIIvQQuZdU6T1R27hE2dKBiI40,2555
-src/setup.py,sha256=MiFIP8ZOD0-WhzStEme18pJ52N8YpVYNZKsaueacQd8,6531
-src/tms.py,sha256=UXbIo1QE--hIX6NZi5Qyp2R_CP338syrY9pCTPrfgnE,1741
-src/utils.py,sha256=30EgwvPGwmtBGkX_EWI0B-PdB1wgxqRW58JKsp6Nl3I,11740
-data_science_document_ai-1.13.0.dist-info/METADATA,sha256=Pq4RD1k4Is-HrJ9mYZu0W7N3EzJSK5BPflg10_NBEeI,1951
-data_science_document_ai-1.13.0.dist-info/WHEEL,sha256=XbeZDeTWKc1w7CSIyre5aMDU_-PohRwTQceYnisIYYY,88
-data_science_document_ai-1.13.0.dist-info/RECORD,,

src/prompts/library/draftMbl/hapag-lloyd/prompt.txt

@@ -1,44 +0,0 @@
-You are a document entity extraction specialist. Given a document, the explained datapoint need to extract.
-
-blNumber: Bill of Lading number.
-voyage: The journey or route code taken by the vessel.
-portOfLoading: The port where cargo is loaded.
-portOfDischarge:  The port where cargo is unloaded.
-bookingNumber: A unique identifier for the booking.
-containers:
-    containerType: Type of the shipping container, usually related to it's size.
-    grossWeight: Total weight of the cargo, including the tare weight of the container.
-    measurements: Dimensions of the cargo (length, width, height) for freight calculations.
-    packageQuantity: package quantity.
-    packageType: Type of packaging used (e.g., cartons, pallets, barrels).
-    containerNumber: Unique ID for tracking the shipping container.
-    sealNumber: Number of the container's seal.
-vessel: The name of the vessel.
-
-
-Your task is to extract the text value of the following entities:
-
-Keywords for datapoints:
-- blNumber: Bill of Lading number, bill of landing no., swb-no., b/l no.
-- voyage: voyage, voy. no, voyage-no.
-- portOfLoading: port of loading, pol, from.]
-- portOfDischarge: port of discharge, pod, delivery, to
-- bookingNumber: Our reference, booking no., carrier reference
-- containers:
-    - containerType: x 40' container
-    - grossWeight: gross weight
-    - measurements: Dimensions of the cargo (length, width, height) for freight calculations
-    - packageQuantity: package quantity, number and kind of packages
-    - packageType: Type of packaging used (e.g., cartons, pallets, barrels), number and kind of packages, description of goods
-    - containerNumber: container number, cntr. nos., it is a combination of 4 letters and 7 digits separated by space right above 'SEAL'
-    - sealNumber: seal number, seal nos., shipper seal, seal.
-- vessel: vessel
-
-
-You must apply the following rules:
-- The JSON schema must be followed during the extraction.
-- The values must only include text found in the document
-- Do not normalize any entity value.
-- If 'sealNumber' is not found don't add it to the result.
-- Validate the JSON make sure it is a valid JSON ! No extra text, no missing comma!
-- Add an escape character (backwards slash) in from of all quotes in values

src/prompts/library/draftMbl/maersk/prompt.txt

@@ -1,17 +0,0 @@
-Extract the following information from the sea waybill document.
-
- **blNumber:** Find the value labeled as "B/L No.".
- **voyage:** Get the "Voyage No." value.
- **portOfLoading:** Find the value in the "Port of Loading" field.
- **portOfDischarge:** Extract the text from the "Port of Discharge" field.
- **bookingNumber:** Look for the value associated with "Booking No.".
- **containers:**
-    The document may contain multiple containers listed within the section "PARTICULARS FURNISHED BY SHIPPER" under the line starting with "Kind of Packages; Description of goods; Marks and Numbers; Container No./Seal No.".  Look for container information that starts with a line that includes "Container Said to Contain" and continues until the next instance of "Container Said to Contain" or the end of the section. For each container, extract the following:
-    * **containerType:** Extract the container type information. It is usually a combination of numbers, the word "DRY", and may include additional characters. It is found on the same line as the container number.
-    * **grossWeight:** Find the value corresponding to the "gross weight" of the container. It is usually represented in KGS and is found on the same line as the container number.
-    * **measurements:** Find the value corresponding to the "measurement" of the container. It is usually represented in CBM and is found on the same line as the container number.
-    * **packageQuantity:** Extract the "package quantity" information. It is usually a whole number and precedes the text "PACKAGE".  All container information will be on the same line as the "package quantity".
-    * **packageType:** Extract the value from the "Kind of Packages" field.
-    * **containerNumber:**  Find the container number. It starts with "MRKU" and is followed by a sequence of digits. It is found on the same line as the text "Container Said to Contain".
-    * **sealNumber:** Get the "Shipper Seal" value. It follows after the text "Shipper Seal :".
- **vessel:** Extract the text from the field "Vessel".

src/prompts/library/finalMbL/hapag-lloyd/prompt.txt

@@ -1,44 +0,0 @@
-You are a document entity extraction specialist. Given a document, the explained datapoint need to extract.
-
-blNumber: Bill of Lading number.
-voyage: The journey or route code taken by the vessel.
-portOfLoading: The port where cargo is loaded.
-portOfDischarge:  The port where cargo is unloaded.
-bookingNumber: A unique identifier for the booking.
-containers:
-    containerType: Type of the shipping container, usually related to it's size.
-    grossWeight: Total weight of the cargo, including the tare weight of the container.
-    measurements: Dimensions of the cargo (length, width, height) for freight calculations.
-    packageQuantity: package quantity.
-    packageType: Type of packaging used (e.g., cartons, pallets, barrels).
-    containerNumber: Unique ID for tracking the shipping container.
-    sealNumber: Number of the container's seal.
-vessel: The name of the vessel.
-
-
-Your task is to extract the text value of the following entities:
-
-Keywords for datapoints:
-- blNumber: Bill of Lading number, bill of landing no., swb-no., b/l no.
-- voyage: voyage, voy. no, voyage-no.
-- portOfLoading: port of loading, pol, from.]
-- portOfDischarge: port of discharge, pod, delivery, to
-- bookingNumber: Our reference, booking no., carrier reference
-- containers:
-    - containerType: x 40' container
-    - grossWeight: gross weight
-    - measurements: Dimensions of the cargo (length, width, height) for freight calculations
-    - packageQuantity: package quantity, number and kind of packages
-    - packageType: Type of packaging used (e.g., cartons, pallets, barrels), number and kind of packages, description of goods
-    - containerNumber: container number, cntr. nos., it is a combination of 4 letters and 7 digits separated by space right above 'SEAL'
-    - sealNumber: seal number, seal nos., shipper seal, seal.
-- vessel: vessel
-
-
-You must apply the following rules:
-- The JSON schema must be followed during the extraction.
-- The values must only include text found in the document
-- Do not normalize any entity value.
-- If 'sealNumber' is not found don't add it to the result.
-- Validate the JSON make sure it is a valid JSON ! No extra text, no missing comma!
-- Add an escape character (backwards slash) in from of all quotes in values