data-science-document-ai 1.42.1__py3-none-any.whl → 1.42.2__py3-none-any.whl

{data_science_document_ai-1.42.1.dist-info → data_science_document_ai-1.42.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: data-science-document-ai
-Version: 1.42.1
+Version: 1.42.2
 Summary: "Document AI repo for data science"
 Author: Naomi Nguyen
 Author-email: naomi.nguyen@forto.com

{data_science_document_ai-1.42.1.dist-info → data_science_document_ai-1.42.2.dist-info}/RECORD

@@ -2,7 +2,7 @@ src/constants.py,sha256=TF_UblovdXZnKIb1lnyJwUqQncJCbzBVihoelI6foSU,3579
 src/constants_sandbox.py,sha256=Iu6HdjCoNSmOX0AwoL9qUQkhq_ZnIN5U9e-Q2UfNuGc,547
 src/docai.py,sha256=dHuR0ehVjUi1CnoNvdp_yxJtpU_HFXqAZ61ywdz7BEo,5655
 src/docai_processor_config.yaml,sha256=qOMmCIORpLQ_D-ytvejXxFvER0e0uGYuzPVdZBGv4Pc,2105
-src/excel_processing.py,sha256=jBL6h5T3fJ4uM_rFiV8c0yWAy8Tt3V3RFtBBqb8ztfo,2744
+src/excel_processing.py,sha256=8toKsafUvwE5QN3TOQO3zfLo0Wv2sGxZHKPsL7n5LkA,2771
 src/io.py,sha256=tOJpMyI-mP1AaXKG4UFudH47MHWzjWBgVahFJUcjGfs,4749
 src/llm.py,sha256=OE4IEIqcM-hYK9U7e0x1rAfcqdpeo4iXPHBp64L5Qz0,8199
 src/log_setup.py,sha256=RhHnpXqcl-ii4EJzRt47CF2R-Q3YPF68tepg_Kg7tkw,2895
@@ -10,7 +10,7 @@ src/pdf_processing.py,sha256=dxsYvNnONAjzS-T7K5aSo89rz7QcdW3ZDfeuFyeCeII,16294
 src/postprocessing/common.py,sha256=lc95nGvy-KrFFQyX2X3ABMjrx1xVYDjuTBgeAXQTcuU,21570
 src/postprocessing/postprocess_booking_confirmation.py,sha256=nK32eDiBNbauyQz0oCa9eraysku8aqzrcoRFoWVumDU,4827
 src/postprocessing/postprocess_commercial_invoice.py,sha256=3I8ijluTZcOs_sMnFZxfkAPle0UFQ239EMuvZfDZVPg,1028
-src/postprocessing/postprocess_partner_invoice.py,sha256=cM4te4qjOI_bXyrF8Zhb6X7eNf5aMKoRaPCFfqFv-98,11538
+src/postprocessing/postprocess_partner_invoice.py,sha256=koGR7dN37FqJcepdzkrzNBHuBBUuCp_3CrteScASqyE,10590
 src/prompts/library/bookingConfirmation/evergreen/placeholders.json,sha256=IpM9nmSPdyroliZfXB1-NDCjiHZX_Ff5BH7-scNhGqE,1406
 src/prompts/library/bookingConfirmation/evergreen/prompt.txt,sha256=5ivskCG831M2scW3oqQaoltXIyHV-n6DYUygWycXxjw,2755
 src/prompts/library/bookingConfirmation/hapag-lloyd/placeholders.json,sha256=hMPNt9s3LuxR85AxYy7bPcCDleug6gSwVjefm3ismWY,1405
@@ -52,6 +52,6 @@ src/prompts/prompt_library.py,sha256=jPxybNPPGH7mzonqtAOqmw5WcT-RtbGP0pvMqqP22hg
 src/setup.py,sha256=M-p5c8M9ejKcSZ9N86VtmtPc4TYLxe1_4_dxf6jpfVc,7262
 src/tms.py,sha256=UXbIo1QE--hIX6NZi5Qyp2R_CP338syrY9pCTPrfgnE,1741
 src/utils.py,sha256=nU69zR3TB7IZmCc19DD8H27Riek8GJAldmhJjCSwNEE,16090
-data_science_document_ai-1.42.1.dist-info/METADATA,sha256=nsGhuml2YNlNF7s7aRUJPpY8psKss8wiLcIavpVInjs,2153
-data_science_document_ai-1.42.1.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
-data_science_document_ai-1.42.1.dist-info/RECORD,,
+data_science_document_ai-1.42.2.dist-info/METADATA,sha256=HdKZ83beNVAa6P-cWIunfRkjk4ry9FNaZd4-OSIKsCo,2153
+data_science_document_ai-1.42.2.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
+data_science_document_ai-1.42.2.dist-info/RECORD,,

src/excel_processing.py

@@ -20,7 +20,8 @@ async def extract_data_from_sheet(
     params, sheet_name, sheet, response_schema, doc_type=None
 ):
     logger.info(f"Processing sheet: {sheet_name}")
-    excel_content = pd.DataFrame(sheet.values)
+    excel_content = pd.DataFrame(sheet.values).dropna(how="all", axis=1)
+
     # Convert to Markdown format for the LLM model
     worksheet = (
         "This is from a excel. Pay attention to the cell position:\n"

src/postprocessing/postprocess_partner_invoice.py

@@ -138,36 +138,7 @@ def update_recipient_and_vendor(aggregated_data, is_recipient_forto):
 
 def process_partner_invoice(params, aggregated_data, document_type_code):
     """Process the partner invoice data."""
-    # Post process containerNumber.
-    # TODO: Remove this block of code after migrating to LLM completely and update the placeholder in the prompt library
-    if "containerNumber" in aggregated_data and isinstance(
-        aggregated_data["containerNumber"], dict
-    ):
-        container_number = aggregated_data.get("containerNumber", {}).get(
-            "formattedValue", None
-        )
-        if container_number:
-            aggregated_data["containerNumber"] = (
-                [
-                    {
-                        "documentValue": aggregated_data.get("containerNumber", {}).get(
-                            "documentValue", ""
-                        ),
-                        "formattedValue": ctr_number,
-                    }
-                    for ctr_number in container_number
-                ]
-                if isinstance(container_number, list)
-                else [
-                    {
-                        "documentValue": aggregated_data.get("containerNumber", {}).get(
-                            "documentValue", ""
-                        ),
-                        "formattedValue": container_number,
-                    }
-                ]
-            )
-
+    # Post process bundeskasse invoices
     if document_type_code == "bundeskasse":
         post_process_bundeskasse(aggregated_data)
         return
@@ -197,9 +168,13 @@ def process_partner_invoice(params, aggregated_data, document_type_code):
                 params,
             )
 
+            # Add page number for the consistency
+            line_item["itemCode"]["page"] = line_item["lineItemDescription"]["page"]
+
         if reverse_charge:
             # Distribute reverseChargeSentence to all line items
             line_item["reverseChargeSentence"] = reverse_charge
+            line_item["reverseChargeSentence"]["page"] = reverse_charge["page"]
 
 
 def compute_score(args):