idvpackage 3.0.11__py3-none-any.whl → 3.0.13__py3-none-any.whl

idvpackage/lebanon_passport_extraction.py

@@ -1,161 +0,0 @@
-import json
-import time
-import openai
-import re
-import logging
-import json
-import time
-import datetime
-import openai
-from langchain.tools import tool
-from langchain.tools.render import format_tool_to_openai_function
-from langchain.prompts import ChatPromptTemplate
-from langchain.chat_models import ChatOpenAI
-from pydantic import BaseModel, Field, validator
-from langchain.utils.openai_functions import convert_pydantic_to_openai_function
-from typing import Optional, Literal
-from langchain.output_parsers.openai_functions import JsonOutputFunctionsParser
-from langchain.agents.output_parsers import OpenAIFunctionsAgentOutputParser
-import idvpackage.genai_utils as genai_utils
-import idvpackage.genai_utils as sanity_utils
-from idvpackage.genai_utils import convert_pydantic_to_openai_function2
-from datetime import datetime, timedelta
-from dateutil.relativedelta import relativedelta
-from pydantic import ValidationError
-import logging
-from langchain.schema.agent import AgentFinish
-
-import openai
-import json
-
-import openai
-import json
-
-class Verify_LBN_Passport(BaseModel):
-    """Validates whether a given OCR text represents a valid Lebanese Passport"""
-    is_valid_id: Literal["True", "False", ""] = Field(..., description="Return True if document is a valid Lebanese Passport" 
-            "It should contain English Text: Republic of Lebanon"
-            "It should contain things a passport typically has like two-line mrz, date of issue, expiry, name, etc."
-            "Return False otherwise.")
-
-@tool(args_schema=Verify_LBN_Passport)
-def verify_lbn_passport(is_valid_id):
-    return is_valid_id
-
-def route(result):
-    if isinstance(result, AgentFinish):
-        return result.return_values['output']
-    else:
-        tools = {
-            "verify_lbn_passport": verify_lbn_passport
-        }
-        return tools[result.tool].run(result.tool_input)
-
-def verify_lbn_pss_chain(ocr_text, openai_key):
-    gpt_model = 'gpt-4o'
-    print("WE ARE IN EXTRACTION CHAIN")
-
-    prompt = ChatPromptTemplate.from_messages([
-        ("system", "You are an expert at identifying Lebanese Passports."
-         ),
-        ("user", "{ocr_text}")
-    ])
-
-    model = ChatOpenAI(model=gpt_model, temperature=0,
-                       openai_api_key=openai_key)
-    functions = [convert_pydantic_to_openai_function(Verify_LBN_Passport)]
-    verification_model = model.bind(functions=functions)
-    verification_chain = prompt | verification_model | JsonOutputFunctionsParser()
-
-    result = verification_chain.invoke({"ocr_text":ocr_text})
-    is_valid_id = result.get("is_valid_id","")
-    return is_valid_id
-
-def make_api_request_with_retries(prompt: str, max_retries: int = 3, delay_seconds: float = 2):
-    """
-    Helper function to make API requests with retry logic using OpenAI
-    """
-    for attempt in range(max_retries):
-        try:
-            response = openai.ChatCompletion.create(
-                model="gpt-4o",
-                temperature=0.4,
-                max_tokens=2000,
-                messages=[
-                    {
-                        "role": "user",
-                        "content": prompt
-                    }
-                ]
-            )
-            logging.info(f"OpenAI successfully prompted.{response.choices[0].message.content} ")
-            result = response.choices[0].message.content
-
-            try:
-                return json.loads(result)
-            except json.JSONDecodeError:
-                try:
-                    json_match = re.search(r'```(json|python|plaintext)?\s*(.*?)\s*```|\s*({.*?})', result, re.DOTALL)
-                    if json_match:
-                        json_str = json_match.group(2) or json_match.group(3)
-                        try:
-                            return json.loads(json_str)
-                        except:
-                            return eval(json_str.replace("'", '"'))
-                except:
-                    pass
-
-            return json.loads(result)
-
-        except Exception as e:
-            print(f"Error during API request (attempt {attempt + 1} of {max_retries}): {str(e)}")
-            if attempt < max_retries - 1:
-                time.sleep(delay_seconds)
-            else:
-                raise Exception(f"Max retries exceeded. Last error: {str(e)}")
-
-
-def lebanon_passport_extraction(passport_data):
-    try:
-        prompt = f"From the attached text, please extract the data in a structured format, the response should be a dictionary, having first_name(only English), mother_name(mother english name if available, else empty string ''), father_name(only English), name which is last_name(only English), passport_number, dob(dd/mm/yyyy), place_of_birth, nationality(ISO 3166-1 alpha-3 country code), issue_date(dd/mm/yyyy), expiry_date(dd/mm/yyyy), gender(FEMALE, MALE), mrz1, mrz2, registry_place_and_issue_number(if not available then empty string ''). Note that the passport_number should always be  2 letters and 7 digits, if the length is less than 7 then append 0 in the start for passport_number_en and same way for passport_number_ar(numbers in passport arabic as well). Also note that the names should be extracted correctly, don't pick any random words for names, especially for first and last_name, it can be verified from the mrz1 string. So please make sure the names are correctly extracted.  The structure of the response should be 'first_name', 'father_name', 'last_name', 'mother_name', 'id_number', 'dob', 'expiry_date', 'issue_date', 'place_of_birth', nationality, registry_place_and_issue_number, etc.. Make sure that the response should only contain a dictionary, and nothing else. Here's the text for your task: {passport_data}"
-
-        back_data = make_api_request_with_retries(prompt)
-
-        if back_data:
-
-            if back_data.get('registry_place_and_issue_number', ''):
-                back_data['registry_place_and_number'] = back_data.pop('registry_place_and_issue_number', '')
-
-            if back_data.get('passport_number', ''):
-                back_data['id_number'] = back_data.pop('passport_number', '')
-
-            if back_data.get('mrz1', '') and back_data.get('mrz2', ''):
-                back_data['mrz'] = back_data.get('mrz1', '') + back_data.get('mrz2', '')
-
-            back_data['issuing_country'] = 'LBN'
-
-            if not back_data.get('nationality', ''):
-                back_data['nationality'] = 'LBN'
-
-            if "gender" in back_data:
-                gender = back_data["gender"].strip().upper()
-                if gender == "F":
-                    back_data["gender"] = "FEMALE"
-                elif gender == "M":
-                    back_data["gender"] = "MALE"
-
-            if 'gender' in back_data:
-                back_data["gender"] = back_data["gender"].strip().upper()
-
-
-    except Exception as e:
-        print(f"Error in processing the extracted data: {e}")
-        back_data = {
-            'first_name': '',
-            'last_name': '',
-            'dob': '',
-            'place_of_birth': '',
-            'expiry_date': ''
-        }
-
-    return back_data

idvpackage/sau_id_extraction.py

@@ -1,248 +0,0 @@
-import pandas as pd
-import re
-from datetime import datetime
-from hijri_converter import convert
-# from googletrans import Translator
-import pycountry
-import os
-
-os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "streamlit-connection-b1a38b694505 (1).json"
-
-def get_country_code(name_in_foreign_language):
-    # translator = Translator()
-    # translated_country_name = translator.translate(name_in_foreign_language, src='auto', dest='en').text
-    
-    # Manual mapping for known challenges or discrepancies in translation
-    manual_mapping = {
-        'ميانمار': 'MMR',  # Myanmar
-        'البحرين': 'BHR',  # Bahrain
-        'اليمن الجنوبي': 'YEM',  # Yemen
-        'فلسطين': 'PSE',  # Palestine
-        'الفلبين': 'PHL',  # Philippines
-        'أفغانستان': 'AFG',  # Afghanistan
-        'سوريا': 'SYR',  # Syria
-        'سري لنكا': 'LKA'  # Sri Lanka
-    }
-    
-    if name_in_foreign_language in manual_mapping:
-        return manual_mapping[name_in_foreign_language]
-    
-    # try:
-    #     country = pycountry.countries.get(name=translated_country_name)
-    #     return country.alpha_3
-    # except AttributeError:
-    #     return "Country Not Found"
-    
-pattern = r'(\d{4}/\d{1,2}/\d{1,2}|[۰-۹]{4}/[۰-۹]{1,2}/[۰-۹]{1,2})'
-def eastern_arabic_to_english(eastern_numeral):
-    arabic_to_english_map = {
-        '٠': '0', '۰': '0',
-        '١': '1', '۱': '1',
-        '٢': '2', '۲': '2',
-        '٣': '3', '۳': '3',
-        '٤': '4', '۴': '4',
-        '٥': '5', '۵': '5',
-        '٦': '6', '۶': '6',
-        '٧': '7', '۷': '7',
-        '٨': '8', '۸': '8',
-        '٩': '9', '۹': '9',
-        '/': '/'
-    }
-
-    # If the character is an Eastern Arabic numeral, convert it to English; otherwise, keep it unchanged.
-    english_numeral = ''.join([arabic_to_english_map[char] if char in arabic_to_english_map else char for char in eastern_numeral])
-    
-    return english_numeral
-
-def distinguish_dates(date_list):
-    today = datetime.now().date()
-    
-    # Calculate the difference between each date and today's date
-    differences = [(abs((today - datetime.strptime(date, '%Y/%m/%d').date()).days), date) for date in date_list]
-    
-    # Sort by difference
-    differences.sort(key=lambda x: x[0])
-    
-    # The date with the smallest difference is considered Gregorian, and the one with the largest difference is considered Hijri
-    gregorian_date = differences[0][1]
-    hijri_date = differences[-1][1]
-
-    return hijri_date, gregorian_date
-
-def hijri_to_gregorian(hijri_date):
-    try:
-        # Split the hijri date
-        year, month, day = map(int, hijri_date.split('/'))
-
-        # Convert the hijri date to Gregorian
-        gregorian_date = convert.Hijri(year, month, day).to_gregorian()
-    
-    # Format the result as a string
-        return f"{gregorian_date.year}/{gregorian_date.month:02}/{gregorian_date.day:02}"
-    except:
-        return hijri_date
-    
-def extract_dates(input_list):
-    # Regex pattern to match YYYY/MM/DD, YYYY/MM/DD in Arabic numerals, 
-    # and some other variations found in the list
-    pattern = r"(\d{4}/\d{2}/\d{2}|[۰۱۲۳۴۵۶۷۸۹]{4}/[۰۱۲۳۴۵۶۷۸۹]{2}/[۰۱۲۳۴۵۶۷۸۹]{2})"
-    
-    extracted_dates = []
-    for item in input_list:
-        match = re.search(pattern, item)
-        if match:
-            extracted_dates.append(match.group(0))
-        else:
-            extracted_dates.append('')
-    return extracted_dates
-
-
-def detect_script(word):
-    arabic_chars = range(0x0600, 0x06FF)  # Arabic Unicode Block
-    english_chars = range(0x0041, 0x007A)  # English uppercase Unicode Block
-    english_chars_lower = range(0x0061, 0x007A)  # English lowercase Unicode Block
-    
-    has_arabic = any(ord(char) in arabic_chars for char in word)
-    has_english = any(ord(char) in english_chars or ord(char) in english_chars_lower for char in word)
-    
-    if has_arabic and has_english:
-        return "Mixed"
-    elif has_arabic:
-        return "Arabic"
-    elif has_english:
-        return "English"
-    else:
-        return "Other"
-    
-def extract_english_strings(data):
-    english_strings = []
-    for string in data:
-        if not re.search("[\u0600-\u06FF\d]", string):  # Filters out strings containing Arabic letters or digits
-            english_strings.append(string)
-    return english_strings
-
-def extract_arabic_strings(data):
-    arabic_strings = []
-    for string in data:
-        if re.search(r"[\u0600-\u06FF\d]", string):  # Filters out strings containing Arabic letters or digits
-            arabic_strings.append(string)
-    return arabic_strings
-
-def clean_special_chars(data):
-    cleaned_data = []
-    for string in data:
-        cleaned_string = re.sub(r'[^A-Za-z\s]', '', string)  # Retains only alphabets and spaces
-        cleaned_data.append(cleaned_string.strip())  # .strip() removes any leading or trailing spaces
-    return cleaned_data
-    
-def extract_id_details(result):
-    # result = detect_text(uploaded_id)
-    df = pd.DataFrame({'res':[result]})
-    pattern = r'(\d{4}/\d{1,2}/\d{1,2}|[۰-۹]{4}/[۰-۹]{1,2}/[۰-۹]{1,2})'
-    i = 0
-    df['Extracted_data']=''
-    try:
-        nationality=[ele for ele in [ele for ele in df['res'].iloc[i] if 'الجنسية' in ele ][0].split('الجنسية') if ele!=''][0].strip()
-
-        nationality=get_country_code(nationality)
-    except:
-        nationality=''
-    try:
-        ## employer
-        employer_ar=[ele for ele in [ele for ele in df['res'].iloc[i] if 'صاحب العمل' in ele ]][0]
-        employer=[ele for ele in employer_ar.split('صاحب العمل') if ele!=''][0].strip()
-
-    except:
-        employer=''
-    try:
-        ### issuing place
-        issuing_place_ar=[ele for ele in df['res'].iloc[i] if 'مكان الإصدار' in ele][0]
-        issuing_place=issuing_place_ar.split('مكان الإصدار')[-1].strip()
-    except:
-        issuing_place=''
-    try:
-        comon_pattern=[ele for ele in [ele for ele in df['res'].iloc[i] if (('الإصدار' in ele ) and('مكان' not in ele))][0].split('الإصدار') if ele!=''][0].strip()
-        matches = re.findall(pattern, comon_pattern)
-
-        matches=[eastern_arabic_to_english(ele) for ele in matches]
-
-        issuing_date, dob=matches[0],matches[1]
-
-        #issuing_date = hijri_to_gregorian(issuing_date)
-        
-    except:
-        
-        try: 
-            dob=[ele for ele in [ele for ele in df['res'].iloc[i] if 'الميلاد' in ele ][0].split('الميلاد') if ele!=''][0].strip()
-            issuing_date= [ele for ele in [ele for ele in df['res'].iloc[i] if( 'الإصدار' in ele) and ('مكان' not in ele ) ][0].split('الإصدار') if ele!=''][0].strip()
-            #issuing_date=hijri_to_gregorian(issuing_date)
-        except:
-            try:
-                dob=[ele for ele in [ele for ele in df['res'].iloc[i] if 'الميلاد' in ele ][0].split('الميلاد') if ele!=''][-1].strip()
-                issuing_date=[ele for ele in [ele for ele in df['res'].iloc[i] if 'الميلاد' in ele ][0].split('الميلاد') if ele!=''][0].strip('الانتهاء').strip()
-                #issuing_date=hijri_to_gregorian(issuing_date)
-            except:
-                issuing_date,dob='',''
-    
-    try:
-
-        #issuing_date_ar,dob_ar=re.findall(pattern, comon_pattern)
-
-        ### Id Number 
-        id_number=[item for item in df['res'].iloc[i] if re.fullmatch(r'\d{10}', item)][0]
-        id_number=eastern_arabic_to_english(id_number)
-    
-    except:
-        
-        try:
-            id_number = [ele for ele in eastern_arabic_to_english([ele for ele in [ele for ele in df['res'].iloc[i] if 'الرقم' in ele ][0].split('الرقم') if ele!=''][0].strip()).split(' ') if len(ele)==10][0]
-            id_number=eastern_arabic_to_english(id_number)
-        except:
-            id_number=''
-    
-    try:
-        profession_Ar=[ele for ele in [ele for ele in df['res'].iloc[i] if 'المهنة' in ele ]][0]
-
-        profession=[ele for ele in profession_Ar.split('المهنة') if ele!=''][-1]
-
-    except:
-        profession=''
-    try:
-        Name_Index=[extract_arabic_strings(df['res'].iloc[i]).index(ele) for ele in extract_arabic_strings(df['res'].iloc[i]) if 'وزارة' in ele][0]
-        Name_1=extract_arabic_strings(df['res'].iloc[i])[Name_Index+1]
-        Name_length=len(Name_1.split(' '))
-        Name_en=max([ele for ele in clean_special_chars(extract_english_strings(df['res'].iloc[i])) if ele not in ['KINGDOM OF SAUDI ARABIA','MINISTRY OF INTERIOR']], key=lambda x: x.count(' '))
-        Name_ar=[ele for ele in [Name_1,Name_en] if ele!=Name_en][0]  
-        
-    except:
-        
-        Name_en,Name_ar='',''
-
-    df['Extracted_data'].iloc[i]=[nationality,employer,issuing_date, dob,id_number,profession,Name_en,Name_ar]
-    
-    cols = ['nationality', 'employer', 'issuing_date', 'dob', 'id_number', 'profession', 'Name_en', 'Name_ar']
-
-    for index, col_name in enumerate(cols):
-        df[col_name] = df['Extracted_data'].apply(lambda x: x[index])
-
-    df['dob']=extract_dates(df['dob'].tolist())
-
-    df['dob']=df['dob'].apply(lambda x: eastern_arabic_to_english(x))
-
-    df['issuing_date']=df['issuing_date'].apply(lambda x: eastern_arabic_to_english(x))
-
-    df['issuing_date']=df['issuing_date'].apply(lambda x: hijri_to_gregorian(x))
-
-    dob = df['dob'].iloc[0]
-
-    if dob:
-        parsed_date = datetime.strptime(dob, "%Y/%m/%d")
-        dob = parsed_date.strftime("%d/%m/%Y")
-
-    # df['gender']= ''
-    # df['expiry_data']= ' '
-
-    # print(df)
-    # TODO: gender, expiry_data
-    return {'id_number': df['id_number'].iloc[0], 'nationality': df['nationality'].iloc[0], 'gender': '', 'dob': dob, 'expiry_date': '', 'name': df['Name_en'].iloc[0], 'occupation': df['profession'].iloc[0], 'employer': df['employer'].iloc[0]}
-