idvpackage 3.0.10__py3-none-any.whl → 3.0.12__py3-none-any.whl

idvpackage/ocr_utils.py

@@ -6,11 +6,11 @@ from google.cloud import vision_v1
 from idvpackage.constants import *
 import pkg_resources
 import base64
-from googletrans import Translator
 from rapidfuzz import fuzz
 import logging
+from datetime import datetime
+
 logging.basicConfig(level=logging.INFO)
-translator = Translator()
 
 def create_final_result(dictionary):
     result = ""
@@ -53,7 +53,6 @@ def create_sub_result(document_report):
 
     return sub_result
 
-
 def mark_clear(data, keys):
     if isinstance(data, dict):
         for key, value in list(data.items()):
@@ -67,7 +66,6 @@ def mark_clear(data, keys):
     else:
         return data
 
-
 def mark_nested_result(data, value):
     if isinstance(data, dict):
         if 'result' in data:
@@ -81,20 +79,18 @@ def mark_nested_result(data, value):
     else:
         return data
     
-from datetime import datetime
-import logging
 
 def parse_date(date_str: str, fmt: str = "%d/%m/%Y"):
     """Safely parse a date string and return a datetime object or None."""
     if not date_str:
         return None
+    date_str = date_str.replace(".", "/").replace("-", "/")  # unify separators
     try:
         return datetime.strptime(date_str, fmt)
     except Exception as e:
         logging.exception(f"Failed to parse date: {date_str}, Error: {e}")
         return None
 
-
 def validation_checks_passport(data: dict,  id_number_type: str, country: str):
     """
     Perform validation checks for passport-related fields such as:
@@ -103,18 +99,28 @@ def validation_checks_passport(data: dict,  id_number_type: str, country: str):
     - Expiry and Issue date validations
     - Gender consistency
     """
+    # Initialize validation_range with default value
+    validation_range = range(1823, 1829)  # Default: 6 years +/- 2 days
+
     if country == 'SDN':
         validation_range = range(3648, 3654)  # 10 years +/- 2 days for Sudanese passports
 
-    if country == 'LBN':
+    elif country == 'LBN':
         validation_range = list(range(1823, 1828)) + list(range(3648, 3654))  # 5 years +/- 2 days for Lebanon passports
 
-    if country == 'JOR':
+    elif country == 'JOR':
         validation_range = range(1823, 1828)  # 5 years (1825) +/- 2 days for Jordan passports
 
+    elif country == 'IRQ': #range is 8 years , in days
+        validation_range = range(2920, 2928)  # 8 years  Iraq passports
+
+    elif country == 'PSE':
+        validation_range = range(1823, 1828)  # 5 years +/- 2 days for Palestinian passports
+    
+    elif country == 'SYR':
+        validation_range = [days for years in [2, 3, 6] for days in range(years * 365 - 2, years * 365 + 3)]
 
     logging.info(f"Performing validation checks for country: {country} and validation_range: {validation_range}")
-    # --- Normalize dates ---
     
     dob = normalize_date_generic(data.get("dob"))
     dob_mrz = normalize_date_generic(data.get("dob_mrz"))
@@ -131,7 +137,11 @@ def validation_checks_passport(data: dict,  id_number_type: str, country: str):
     gender = data.get("gender", "")
     gender_mrz = data.get("gender_mrz", "")
 
-    logging.info(f"Extracted Data for Validation - DOB: {dob}, DOB MRZ: {dob_mrz}, Expiry Date: {expiry_date}, Expiry Date MRZ: {expiry_date_mrz}, Issue Date: {issue_date}, ID Number: {id_number}, ID Number MRZ: {id_number_mrz}, Gender: {gender}, Gender MRZ: {gender_mrz}")   
+    if gender:
+        gender = gender[0].upper()
+    if gender_mrz:
+        gender_mrz = gender_mrz[0].upper()
+    logging.info(f"Extracted Data for Validation - DOB: {dob}, DOB MRZ: {dob_mrz}, Expiry Date: {expiry_date}, Expiry Date MRZ: {expiry_date_mrz}, Issue Date: {issue_date}, ID Number: {id_number}, ID Number MRZ: {id_number_mrz}, Gender: {gender}, Gender MRZ: {gender_mrz}")
     # --- Initialize Flags ---
     is_id_number_same_mrz = False
     is_expiry_date_same_mrz = False
@@ -140,13 +150,14 @@ def validation_checks_passport(data: dict,  id_number_type: str, country: str):
     is_gender_mrz_match = False
 
     # --- ID Number Validation ---
+    logging.info(f"Comparing ID Number: data field {id_number} vs MRZ {id_number_mrz}")
+
     if id_number and id_number_mrz:
-        logging.info(f"Comparing ID Number: data field {id_number} vs MRZ {id_number_mrz}")
         is_id_number_same_mrz = (id_number == id_number_mrz)
 
     # --- Expiry Date Match (OCR vs MRZ) ---
+    logging.info(f"Comparing Expiry Date: data field {expiry_date} vs MRZ {expiry_date_mrz}")
     if expiry_date and expiry_date_mrz:
-        logging.info(f"Comparing Expiry Date: data field {expiry_date} vs MRZ {expiry_date_mrz}")
         is_expiry_date_same_mrz = (expiry_date == expiry_date_mrz)
 
     # --- Issue vs Expiry Duration (MRZ) ---
@@ -187,72 +198,6 @@ def validation_checks_passport(data: dict,  id_number_type: str, country: str):
         "is_gender_mrz_match": is_gender_mrz_match
     }
 
-# def validation_checks_passport(data, country):
-
-#     dob_back_str = data.get("dob", "")
-#     dob_back_mrz_str = data.get("date_of_birth_mrz", "")
-    
-#     dob_back = normalize_date_generic(dob_back_str)
-#     dob_back_mrz = normalize_date_generic(dob_back_mrz_str)
-
-#     expiry_date = data.get("expiry_date", "")
-#     issue_date = data.get("issue_date", "")
-#     expiry_date_mrz = data.get("expiry_date_mrz", "")
-
-#     id_number = data.get("id_number", "")
-#     id_number_mrz = data.get("id_number_mrz", "")
-
-#     if id_number and id_number_mrz:
-#         if id_number == id_number_mrz:
-#             is_id_number_same_mrz= True
-#         else:
-#             is_id_number_same_mrz = False
-
-#     if data.get('expiry_date','') and data.get('expiry_date_mrz',''):
-
-#         try:
-#             #check if both dates are same
-#             expiry_date_obj = datetime.strptime(data.get("expiry_date", ""), "%d/%m/%Y")
-#             expiry_date_mrz_obj = datetime.strptime(data.get("expiry_date_mrz", ""), "%d/%m/%Y")
-            
-#             logging.info(f"expiry_date_obj: {expiry_date_obj}, expiry_date_mrz_obj: {expiry_date_mrz_obj}")
-        
-#             if expiry_date_obj == expiry_date_mrz_obj:
-#                 is_expiry_date_same_mrz = True
-#             else:
-#                 is_expiry_date_same_mrz = False
-#         except:
-
-#             is_expiry_date_same_mrz = False
-#             logging.info("Error in comparing expiry dates for SDN ID")
-#             pass
-
-#         if data.get('issue_date', '') and data.get("expiry_date_mrz",''):
-#             try:
-#                 issue_date_obj = datetime.strptime(data.get("issue_date", ""), "%d/%m/%Y")
-#                 logging.info(f"difference_in_days issue_date_mrz_obj: {issue_date_obj}, expiry_date_mrz_obj: {expiry_date_mrz_obj} differece is : {(expiry_date_mrz_obj - issue_date_obj).days}")
-#                 difference_in_days_mrz_obj = (expiry_date_mrz_obj - issue_date_obj).days
-
-#                 valid_id_duration_mrz = difference_in_days_mrz_obj in [3649,3650, 3651, 3652, 3653]
-#             except:
-#                 logging.info("Error in calculating date difference between issue and expiry dates for SDN ID")
-#                 pass
-
-#         if data.get("issue_date",'') and data.get("expiry_date",''):
-#             try:
-                
-#                 logging.info(f"difference_in_days issue_date_obj: {issue_date_obj}, expiry_date_obj: {expiry_date_obj} differece is : {(expiry_date_obj - issue_date_obj).days}")
-#                 difference_in_days_obj = (expiry_date_obj - issue_date_obj).days
-
-#                 valid_id_duration = difference_in_days_obj in [ 3649,3650, 3651, 3652, 3653]
-
-#             except:
-#                 logging.info("Error in calculating date difference between issue and expiry dates from MRZ for SDN ID")
-#                 pass
-
-          
-                
-
 def age_validation(dob, age_threshold=18):
     age_val = {
         "breakdown": {
@@ -319,46 +264,43 @@ def is_valid_and_not_expired(expiry_date, country):
         except:
             return 'consider'
 
-def identify_document_type(text, country, origial_text=None):
-    text = text.upper()
-    emirates_id_pattern = r'\b(ILARE\w*|IDARE\w*|RESIDENT IDENTITY)\b'
-    iqama_id_pattern = r'KINGDOM OF SAUDI ARABIA|RESIDENT IDENTITY|MINISTRY OF INTERIOR'
-    iraq_id_pattern = r'Register|Signature|Family Number|The Directorate of Nationality|IDIRQ|The Republic of Iraq|Ministry of|National Card|Iraq'
-    lbn_id_pattern = r'Marital status|Family|Lebanon Republic|Republic of Lebanon'
-    qtr_id_pattern = r'State of Qatar|Residency Permit|Director General of the General Department|Directorate of Passports|Passport number'
-    passport_pattern = r'\b(PASSPORT|PPT)\b'
-    driver_license_pattern = r'\b(DRIVER|LICENSE|DL)\b'
-    
-    if re.search(emirates_id_pattern, text, re.IGNORECASE):
-        return "EID"
+def normalize_gender(gender: str):
+    if not gender:
+        return None
 
-    if country == 'IRQ' and (re.search(iraq_id_pattern, text, re.IGNORECASE) or re.search(iraq_id_pattern, origial_text, re.IGNORECASE)):
-        return "INC"
-    
-    if country == 'LBN' and re.search(lbn_id_pattern, text, re.IGNORECASE):
-        return "LBN"
-    
-    if country == 'QAT' and re.search(qtr_id_pattern, text, re.IGNORECASE):
-        return "QAT"
-        
-    if re.search(passport_pattern, text, re.IGNORECASE):
-        return "PASSPORT"
+    gender = gender.strip().lower()
 
-    if re.search(driver_license_pattern, text, re.IGNORECASE):
-        return "DL"
+    if gender in {"m", "male"}:
+        return "Male"
 
-    return "Unknown"
+    if gender in {"f", "female"}:
+        return "Female"
 
-def identify_front_id(text):
-    front_id_keywords = ['Resident Identity', 'United arab emirates', 'federal authority for identity', 'ID Number', 'Kingdom of saudi arabia', 'ministry of']
-    pattern = '|'.join(map(re.escape, front_id_keywords))
-    
-    if re.search(pattern, text, re.IGNORECASE):
-        return True
-    else:
-        return False
+    return None
+
+def normalize_mrz_date(mrz_date, out_format="%Y-%m-%d"):
+    """
+    MRZ date format: YYMMDD
+    """
+    if not mrz_date or len(mrz_date) != 6:
+        return None
+
+    try:
+        yy = int(mrz_date[0:2])
+        mm = int(mrz_date[2:4])
+        dd = int(mrz_date[4:6])
+
+        # Century inference
+        
+        year = 2000 + yy
+
+        dt = datetime(year, mm, dd)
+        return dt.strftime(out_format)
+
+    except Exception as e:
+        logging.info(f"Failed to parse MRZ date {mrz_date}: {e}")
+        return None
     
-from datetime import datetime
 def normalize_date_generic(date_str, out_format="%Y-%m-%d"):
     date_str = date_str.replace(".", "/").replace("-", "/")  # unify separators
     logging.info(f"Normalizing date: {date_str}")
@@ -381,74 +323,6 @@ def normalize_date_generic(date_str, out_format="%Y-%m-%d"):
     logging.info(f"Unable to parse date: {date_str}")
     return None
 
-def identify_back_id(text):
-    back_id_keywords = ['ILARE', 'IDARE', 'Signature']
-    pattern = '|'.join(map(re.escape, back_id_keywords))
-    
-    if re.search(pattern, text, re.IGNORECASE):
-        return True
-    else:
-        return False
-
-def identify_front_id_iraq(text):
-    front_id_keywords = ['The Republic of Iraq', 'Ministry of', 'National Card', 'Passports and Residence', 'Republic', 'Ministry', 'Iraq']
-    pattern = '|'.join(map(re.escape, front_id_keywords))
-    
-    if re.search(pattern, text, re.IGNORECASE):
-        return True
-    else:
-        return False
-
-def identify_back_id_iraq(back_id_text, text):
-    back_id_keywords = ['IDIRQ', 'Signature', 'Register', 'Family Number', 'The Directorate of Nationality', 'IDIR']
-    pattern = '|'.join(map(re.escape, back_id_keywords))
-    
-    if re.search(pattern, text, re.IGNORECASE):
-        return True
-    else:
-        if re.search(pattern, back_id_text, re.IGNORECASE):
-            return True
-        else:
-            return False
-
-def identify_front_id_lebanon(text):
-    front_id_keywords = ['Lebanon Republic', 'Ministry of', 'Republic of Lebanon']
-    pattern = '|'.join(map(re.escape, front_id_keywords))
-    
-    if re.search(pattern, text, re.IGNORECASE):
-        return True
-    else:
-        return False
-
-def identify_back_id_lebanon(text):
-    logging.info(f'\n\nIdentifying back ID Lebanon with text: {text}\n')
-    back_id_keywords = ['Marital status', 'Family']
-    pattern = '|'.join(map(re.escape, back_id_keywords))
-    
-    if re.search(pattern, text, re.IGNORECASE):
-        return True
-    else:
-        return False
-
-def identify_front_id_qatar(text):
-    front_id_keywords = ['State of Qatar', 'Residency Permit']
-    pattern = '|'.join(map(re.escape, front_id_keywords))
-    
-    if re.search(pattern, text, re.IGNORECASE):
-        return True
-    else:
-        return False
-
-def identify_back_id_qatar(text):
-    back_id_keywords = ['Director General of the General Department', 'Directorate of Passports', 'Passport number']
-    pattern = '|'.join(map(re.escape, back_id_keywords))
-    
-    logging.info(f'{re.search(pattern, text, re.IGNORECASE)}')
-    if re.search(pattern, text, re.IGNORECASE):
-        return True
-    else:
-        return False
-
 def document_on_printed_paper(image, block_size=11, c_value=2, texture_threshold=0.4, contour_area_threshold=5000):
     image_gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
     image_thresh = cv2.adaptiveThreshold(image_gray, 255, cv2.ADAPTIVE_THRESH_MEAN_C, cv2.THRESH_BINARY, block_size, c_value)
@@ -466,142 +340,6 @@ def document_on_printed_paper(image, block_size=11, c_value=2, texture_threshold
     else:
         return 'clear'
 
-
-def check_logo_existence_iraq(logo_content, id_card_content, threshold=3, good_mathces_filter=0.6):
-    nparr_logo = np.frombuffer(logo_content, np.uint8)
-    logo = cv2.imdecode(nparr_logo, cv2.IMREAD_GRAYSCALE)
-
-    nparr_id_card = np.frombuffer(id_card_content, np.uint8)
-    id_card = cv2.imdecode(nparr_id_card, cv2.IMREAD_COLOR)
-    id_card_gray = cv2.cvtColor(id_card, cv2.COLOR_BGR2GRAY)
-
-    sift = cv2.SIFT_create()
-
-    kp1, des1 = sift.detectAndCompute(logo, None)
-    kp2, des2 = sift.detectAndCompute(id_card_gray, None)
-
-    FLANN_INDEX_KDTREE = 1
-    index_params = dict(algorithm=FLANN_INDEX_KDTREE, trees=5)
-    search_params = dict(checks=50)
-    flann = cv2.FlannBasedMatcher(index_params, search_params)
-
-    matches = flann.knnMatch(des1, des2, k=2)
-
-    good_matches = []
-    for m, n in matches:
-        if m.distance < good_mathces_filter * n.distance:
-            good_matches.append(m)
-
-    # 40
-    min_good_matches = threshold
-    # min_good_matches = 3
-
-    if len(good_matches) >= min_good_matches:
-        return True
-    else:
-        return False
-
-
-def detect_logo(client, input_image_content, country, compare_type=None, side=None):
-    if country == 'UAE':
-        reference_logo_path = pkg_resources.resource_filename('idvpackage', 'template_images/emirates_id_logo.jpeg')
-
-        with open(reference_logo_path, 'rb') as logo_file:
-            reference_image_content = logo_file.read()
-
-        reference_image = vision_v1.types.Image(content=reference_image_content)
-        input_image = vision_v1.types.Image(content=input_image_content)
-
-        reference_response = client.logo_detection(image=reference_image)
-        input_response = client.logo_detection(image=input_image)
-
-        reference_logos = reference_response.logo_annotations
-        input_logos = input_response.logo_annotations
-
-        for reference_logo in reference_logos:
-            for input_logo in input_logos:
-                if reference_logo.description.lower() == input_logo.description.lower():
-                    return 'clear'
-
-        return 'consider'
-
-    if country in ['IRQ', 'LBN', 'QAT'] and compare_type != 'template':
-        if country == 'IRQ':
-            logo_path = 'template_images/iraq_id_logo.png'
-            good_mathces_filter = 0.6
-        elif country == 'QAT':
-            logo_path = 'template_images/qatar_id_logo.jpeg'
-            good_mathces_filter = 0.7
-
-        reference_logo_path = pkg_resources.resource_filename('idvpackage', logo_path)
-        with open(reference_logo_path, 'rb') as logo_file:
-            reference_image_content = logo_file.read()
-        result = check_logo_existence_iraq(reference_image_content, input_image_content, 3, good_mathces_filter)
-
-        if result:
-            return 'clear'
-        else:
-            return 'consider'
-    
-    if country in ['IRQ', 'LBN', 'QAT'] and compare_type == 'template':
-        if side == 'front':
-            if country == 'QAT':
-                threshold = 300
-                front_template_path = 'template_images/qatar_front_standard_template.jpeg'
-            elif country == 'IRQ':
-                threshold = 40
-                front_template_path = 'template_images/iraq_standard_template.png'
-            else:  # LBN
-                threshold = 60
-                front_template_path = 'template_images/lbn_front_standard_template.png'
-            reference_logo_path = pkg_resources.resource_filename('idvpackage', front_template_path)
-        if side == 'back':
-            if country == 'QAT':
-                threshold = 235
-                back_template_path = 'template_images/qatar_back_standard_template.jpeg'
-            elif country == 'IRQ':
-                threshold = 140
-                back_template_path = 'template_images/iraq_back_standard_template.jpg'
-            else:  # LBN
-                threshold = 120
-                back_template_path = 'template_images/lbn_back_standard_template.png'
-            reference_logo_path = pkg_resources.resource_filename('idvpackage', back_template_path)
-
-        with open(reference_logo_path, 'rb') as logo_file:
-            reference_image_content = logo_file.read()
-
-        result = check_logo_existence_iraq(reference_image_content, input_image_content, threshold)
-
-        if result:
-            return 'clear'
-        else:
-            return 'consider'
-
-
-
-def detect_logo_saudi(client, input_image):
-    reference_logo_path = pkg_resources.resource_filename('idvpackage', 'template_images/sau_id_logo.png')
-    
-    with open(reference_logo_path, 'rb') as logo_file:
-        reference_image_content = logo_file.read()
-
-    reference_image = vision_v1.types.Image(content=reference_image_content)
-    input_image = vision_v1.types.Image(content=input_image)
-
-    reference_response = client.logo_detection(image=reference_image)
-    input_response = client.logo_detection(image=input_image)
-
-    reference_logos = reference_response.logo_annotations
-    input_logos = input_response.logo_annotations
-
-    for reference_logo in reference_logos:
-        for input_logo in input_logos:
-            if reference_logo.description.lower() == input_logo.description.lower():
-                return 'clear'
-
-    return 'consider'
-
-
 def perform_feature_matching(image, template_image_path):
     template_image = cv2.imread(template_image_path, cv2.IMREAD_COLOR)
 
@@ -625,8 +363,6 @@ def perform_feature_matching(image, template_image_path):
 
     return similarity_score
 
-
-# part of detecting screenshot - checks for time values seen on phone
 def extract_time_values(text_annotations):
     time_values = []
     for annotation in text_annotations:
@@ -635,7 +371,6 @@ def extract_time_values(text_annotations):
         time_values.extend(time_matches)
     return time_values
 
-
 def detect_screenshot(client, image):
     icons = 0
     battery_value_and_time = 0
@@ -667,26 +402,20 @@ def detect_screenshot(client, image):
     else:
         return 'clear'
 
-
 def detect_photo_on_screen(client, image):
     flag = 'clear'
 
     image_data = vision_v1.Image(content=base64.b64decode(image))
-    
     image_response = client.label_detection(image=image_data)
     image_labels = image_response.label_annotations
 
     confidence_threshold = 0.58  
-
     # 'display device'
-
     keywords = ['mobile phone', 'mobile device', 'portable communications device', 'communication device', 'smartphone', 'cell phone', 'touchscreen', 'laptop', 'notebook', 'computer', 'screen', 'gadget']
 
     for label in image_labels:
         description = label.description.lower()
         confidence = label.score
-        # print(f"Description: {description}")
-        # print(f"Confidence: {confidence}")
         if confidence >= confidence_threshold:
             match = any(fuzz.ratio(description, keyword.lower()) >= 90 for keyword in keywords)
             if match:
@@ -699,28 +428,8 @@ def fuzzy_match_fields(field1, field2, threshold=55):
     # print(f"similarity: {similarity}")
     return similarity >= threshold
 
-def standardize_date(input_date):
-        input_formats = [
-            "%Y-%m-%d", "%m-%d-%Y", "%Y%m%d",
-            "%Y/%m/%d", "%m/%d/%Y", "%d/%m/%Y",
-            "%Y.%m.%d", "%d.%m.%Y", "%m.%d.%Y",
-            "%Y %m %d", "%d %m %Y", "%m %d %Y",
-        ]
-
-        for format in input_formats:
-            try:
-                parsed_date = datetime.strptime(input_date, format)
-                standardized_date = parsed_date.strftime("%d/%m/%Y")
-                print(f"\n\n---------------Standardized date: { standardized_date}")
-                return standardized_date
-            except ValueError:
-                pass
-
-        return None
-
 def compare_dates(date_str1, date_str2):
     date_format = "%d/%m/%Y"
-
     date1 = datetime.strptime(date_str1, date_format)
     date2 = datetime.strptime(date_str2, date_format)
 
@@ -758,9 +467,6 @@ def data_comparison_check(data, country):
                 data_comparison['breakdown']['gender']['result'] = 'clear'
             else:
                 data_comparison['breakdown']['gender']['result'] = 'consider'
-
-        # if not fuzzy_match_fields(data.get('gender', '').lower(),user_data.get('gender', '').lower()):
-        #     data_comparison['breakdown']['gender']['result'] = 'consider'
     
         if country == 'IRQ':
             if data.get('doc_type') == 'national_identity_card':
@@ -796,18 +502,12 @@ def data_comparison_check(data, country):
             last_name = data.get('last_name', '')
 
         if user_data.get('first_name', ''):
-            # print(f'\n\n--------------------------- Extracted first name: {first_name}')
-            # print(f"--------------------------- User first name: {user_data.get('first_name', '')}")
-
             if fuzzy_match_fields(first_name.lower(), user_data.get('first_name', '').lower()):
                 data_comparison['breakdown']['first_name']['result'] = 'clear'
             else:
                 data_comparison['breakdown']['first_name']['result'] = 'consider'
 
         if user_data.get('last_name', ''):
-            # print(f"\n--------------------------- Extracted last name: {last_name}")
-            # print(f"--------------------------- User last name: {user_data.get('last_name', '')}")
-
             if fuzzy_match_fields(last_name.lower().replace('-', '').replace('_', ''), user_data.get('last_name', '').lower().replace('-', '').replace('_', '')):
                 data_comparison['breakdown']['last_name']['result'] = 'clear'
             else:
@@ -829,15 +529,7 @@ def data_comparison_check(data, country):
 
     return data_comparison
 
-
-
-
-
-
 def data_consistency_check(data, front_id_text, back_id_text, country, back_img):
-    print(f"Country here: {country}")
-    from deep_translator import GoogleTranslator
-
     data_consistency = DATA_CONSISTENCY
     data_consistency = mark_clear(data_consistency, ['breakdown', 'result'])
 
@@ -858,42 +550,21 @@ def data_consistency_check(data, front_id_text, back_id_text, country, back_img)
         if not fuzzy_match_fields(data.get('last_name').lower(),passport_data.get('passport_surname').lower()):
             data_consistency['breakdown']['last_name']['result'] = 'consider'
 
-    #### For data consistency compare data from different sources, like id and passport. 
-    #### so the dob from id should match with dob extracted from passport
-
     if country == 'UAE':
-        doc_type1 = identify_document_type(front_id_text, country)
-        doc_type2 = identify_document_type(back_id_text, country)
-        if doc_type1 == 'EID' or doc_type2=='EID':
-            data_consistency['breakdown']['document_type']['result'] = 'clear'
-        else:
-            data_consistency['breakdown']['document_type']['result'] = 'consider'
-
-        if data.get('id_number_front', '') and data.get('id_number', ''):
-            if data.get('id_number_front', '') != data.get('id_number', ''):
+        
+        data_consistency['breakdown']['document_type']['result'] = 'clear'
+        if data.get('id_number', '') and data.get('id_number_mrz', ''):
+            if data.get('id_number', '') != data.get('id_number_mrz', ''):
                 data_consistency['breakdown']['multiple_data_sources_present']['result'] = 'consider'
-
-
-
-    if country == 'SAU':
-        doc_type1 = identify_front_id(front_id_text)
-        if doc_type1:
-            data_consistency['breakdown']['document_type']['result'] = 'clear'
-        else:
-            data_consistency['breakdown']['document_type']['result'] = 'consider'
     
     if country == 'IRQ':
-        if data.get('doc_type') == 'passport':
-            if 'passport' and ('iraq' or 'republic of iraq') in data.get('passport_data').lower():
-                data_consistency['breakdown']['document_type']['result'] = 'clear'
-            else:
-                data_consistency['breakdown']['document_type']['result'] = 'clear'
-        else:
-            
-            data_consistency['breakdown']['document_type']['result'] = 'clear'
+        data_consistency['breakdown']['document_type']['result'] = 'clear'
 
-        front_id_number = data.get('id_number_front', '')
-        back_id_number = data.get('id_number', '')
+        front_id_number = data.get('id_number', '')
+        back_id_number = data.get('id_number_mrz', '')
+        
+        logging.info(f'Front ID Number: {front_id_number}')
+        logging.info(f'Back ID Number: {back_id_number}')
 
         if front_id_number and back_id_number:
             if front_id_number != back_id_number:
@@ -956,42 +627,38 @@ def data_consistency_check(data, front_id_text, back_id_text, country, back_img)
     if country in ['LBN', 'SDN', 'SYR', 'JOR', 'PSE']:
         data_consistency['breakdown']['document_type']['result'] = 'clear'
 
-    ## set default to avoid any problems of cache
     data_consistency['result'] = 'clear'
-
     result = create_final_result(data_consistency)    
     data_consistency['result'] = result
 
     if country == 'QAT' and data_consistency['breakdown']['multiple_data_sources_present']['result'] != 'consider':
         data_consistency['result'] = 'clear'
-
-    logging.info(f"\n\n---------------------------- DATA CONSISTENCY: {data_consistency}")
+    
+    logging.info(f"---------------------------- DATA CONSISTENCY: {data_consistency}")
 
     return data_consistency
 
-
-
+def normalize_name_parts(name_parts):
+    cleaned = []
+    for part in name_parts:
+        # split on any non-letter characters #use . as well
+        tokens = re.split(r'[^A-Za-z]+', part)
+        cleaned.extend(token for token in tokens if token)
+    return cleaned
 
 def get_name_match_mrz(data, doc_type):
-
-    if data['nationality'] == 'SDN' and doc_type != 'passport':
+    if (data['nationality'] == 'SDN' or data.get('country') == 'UAE') and doc_type != 'passport':
         name_mrz = []
         try:
             for word in data.get("mrz3", "").split("<"):
                 if word and word.isalpha():
                     name_mrz.append(word)
-
-            # data['name_mrz'] = " ".join(name_mrz)
-            name = data.get("full_name_generic", "")
-            name = name.split(" ")
-           
         except Exception as e:
-            name = []
+            name_mrz = []
             logging.info(f"Error in extracting name from MRZ for SDN ID: {e}")
             pass
 
-    elif doc_type == 'passport' and data['nationality'] in ['SDN', 'LBN', 'JOR']:
-
+    elif doc_type == 'passport' and data['nationality'] in ['SDN', 'LBN', 'JOR', 'IRQ', 'PSE', 'SYR']:
         name_mrz = []
         mrz1 = data.get('mrz1', '')
         logging.info(f"MRZ1 extracted: {mrz1}")
@@ -1005,10 +672,8 @@ def get_name_match_mrz(data, doc_type):
 
             except Exception as e:
                 logging.info(f"Error in extracting name from MRZ for passport: {e}")
-                name_mrz = []
                 pass
                 
-
     if data['nationality'] == 'SDN':
         name = data.get("full_name_generic", "")
 
@@ -1018,24 +683,43 @@ def get_name_match_mrz(data, doc_type):
     elif data['nationality'] == 'JOR':
         name = data.get("name", "")
 
+    elif data['nationality'] == 'IRQ':
+        name = data.get("name", "")
+        logging.info(f"Name extracted for IRQ: {name}")
+    
+    elif data['nationality'] == 'PSE':
+        name = data.get("full_name_openai", "")
+        
+    elif data['nationality'] == 'SYR':
+        name = data.get("full_name", "")
+
+    elif data['country'] == 'UAE':
+        name = data.get("name", "")
+        
     name = name.split(" ")
+    logging.info(f"Name extracted from ID: {name}")
     try:
 
         logging.info(f"name on card: {name}, name from mrz: {name_mrz}")
 
         # sort the name parts to ensure order does not affect comparison
+        name = normalize_name_parts(name)
+        name_mrz = normalize_name_parts(name_mrz)
+        logging.info(f"Name parts after removing special chars - Card: {name}, MRZ: {name_mrz}")
         name = sorted(name)
-        name_mrz = sorted(name_mrz)
-        logging.info(f"Sorted name on card: {name}, Sorted name from mrz: {name_mrz}")
-        min_length = min(len(name), len(name_mrz))
+        sorted_name_mrz = sorted(name_mrz)
+        logging.info(f"Sorted name on card: {name}, Sorted name from mrz: {sorted_name_mrz}")
+        min_length = min(len(name), len(sorted_name_mrz))
+        logging.info(f"Name parts after removing special chars - Card: {name}, MRZ: {sorted_name_mrz}")
+        min_length = min(len(name), len(sorted_name_mrz))
 
         from rapidfuzz import fuzz
         flag = True
 
         count = 0
         for i in range(min_length):
-            ratio = fuzz.ratio(name[i].lower(), name_mrz[i].lower())
-            logging.info(f"Comparing name parts: {name[i]} and {name_mrz[i]}, ratio: {ratio}")
+            ratio = fuzz.ratio(name[i].lower(), sorted_name_mrz[i].lower())
+            logging.info(f"Comparing name parts: {name[i]} and {sorted_name_mrz[i]}, ratio: {ratio}")
             if ratio < 70:
                 count = count +1
 
@@ -1069,7 +753,6 @@ def is_age_18_above(dob_str):
     logging.warning(f"Could not parse date: {dob_str}")
     return False
 
-
 def is_expired_id(expiry_date):
     """
     Checks if an ID is expired.
@@ -1120,7 +803,6 @@ def data_validation_check(data, country):
 
     try:
         dob = data.get('dob')
-        # print(f"DOB: {dob}")
         dob = get_dates_to_generic_format(dob)
         parsed_date = datetime.strptime(dob, "%d/%m/%Y")
         data_validation["breakdown"]['date_of_birth']["result"] = 'clear'
@@ -1216,7 +898,7 @@ def data_validation_check(data, country):
         data_validation["breakdown"]['expiry_date']["result"] = 'clear'
 
     if country == 'IRQ' and data.get('doc_type') == 'national_identity_card' and not data.get('card_number'):
-        doc_no = data.get('card_number_front', '')
+        doc_no = data.get('card_number', '')
         if len(doc_no)==9:
             data_validation["breakdown"]['document_numbers']["result"] = 'clear'
         else:
@@ -1246,7 +928,6 @@ def data_validation_check(data, country):
         else:
             data_validation['breakdown']['expiry_date']['result'] = 'consider'
 
-
     result = create_final_result(data_validation)
     data_validation['result'] = result
 
@@ -1255,7 +936,6 @@ def data_validation_check(data, country):
 
 ## pending
 def image_integrity_check(data, front_id_text, back_id_text, coloured, blurred, glare, missing_fields, country):
-    from deep_translator import GoogleTranslator
     image_integrity = IMAGE_INTEGRITY
     image_integrity = mark_clear(image_integrity, ['breakdown', 'result'])
 
@@ -1275,14 +955,13 @@ def image_integrity_check(data, front_id_text, back_id_text, coloured, blurred,
     image_quality_result = create_final_result(image_integrity['breakdown']['image_quality'])
     image_integrity['breakdown']['image_quality']['result'] = image_quality_result
 
-    f_result = 'clear'
-    if data.get('doc_type') == 'national_identity_card':
-        f_result = identify_front_id(front_id_text)
-    
+    f_result = True
     front_doc_on_pp = data.get('front_doc_on_pp')
 
     if country == 'UAE' and not data.get('uae_pass_data', ''):
-        b_result = identify_back_id(back_id_text)
+        logging.info(f"UAE Image integrity check for ID card or passport {back_id_text}")
+        b_result = True
+        logging.info(f"UAE back id result: {b_result}")
 
         if back_id_text and b_result:
             image_integrity['breakdown']['conclusive_document_quality']['properties']['missing_back'] = 'clear'
@@ -1293,21 +972,11 @@ def image_integrity_check(data, front_id_text, back_id_text, coloured, blurred,
             image_integrity['breakdown']['supported_document']['result'] = 'clear'
         else:
             image_integrity['breakdown']['supported_document']['result'] = 'consider'
-    
-    if country == 'SAU':
-        if f_result:
-            image_integrity['breakdown']['supported_document']['result'] = 'clear'
-        else:
-            image_integrity['breakdown']['supported_document']['result'] = 'consider'
-        
-        image_integrity['breakdown']['conclusive_document_quality']['properties']['digital_document'] = front_doc_on_pp
 
     if country == 'IRQ' and data.get('doc_type') == 'national_identity_card':
         image_integrity['breakdown']['conclusive_document_quality']['properties']['missing_back'] = 'clear'
         image_integrity['breakdown']['supported_document']['result'] = 'clear'
-
-       
-
+        
     elif country == 'IRQ' and data.get('doc_type') == 'passport':
         image_integrity['breakdown']['conclusive_document_quality']['properties']['missing_back'] = 'clear'
         if 'passport' and ('iraq' or 'republic of iraq') in data.get('passport_data').lower():
@@ -1383,20 +1052,18 @@ def image_integrity_check(data, front_id_text, back_id_text, coloured, blurred,
     if country=='IRQ':
         image_integrity['result']='clear'
 
-    
-    logging.info(f"\n\n---------------------------- IMAGE INTEGRITY: {image_integrity}")
+    logging.info(f"---------------------------- IMAGE INTEGRITY: {image_integrity}")
 
     return image_integrity
 
 def visual_authenticity_check(data, front_id_text, back_id_text, selfie, facial_similarity, face_match_threshold, country):
-    from deep_translator import GoogleTranslator
     visual_authenticity = VISUAL_AUTHENTICITY
     visual_authenticity = mark_clear(visual_authenticity, ['breakdown', 'result'])
 
     if np.any(selfie):
         ## if facial similarity is matching the threshold or even if facial similarity comes above 40 then we say face was detected - approach can be changed
         logging.info(f'Facial similarity: {facial_similarity}, Threshold: {face_match_threshold}\n')
-        if facial_similarity > face_match_threshold:
+        if facial_similarity >= face_match_threshold:
             visual_authenticity['breakdown']['face_detection'] = 'clear'
             visual_authenticity['breakdown']['security_features'] = 'clear'
         else:
@@ -1406,9 +1073,6 @@ def visual_authenticity_check(data, front_id_text, back_id_text, selfie, facial_
         visual_authenticity['breakdown']['face_detection'] = ''
         visual_authenticity['breakdown']['security_features'] = ''
     
-    if data.get('doc_type') == 'national_identity_card':
-        doc_type1 = identify_document_type(front_id_text, country, front_id_text)
-
     front_doc_on_pp = data.get('front_doc_on_pp')
     front_screenshot = data.get('front_screenshot_result')
     front_photo_on_screen_result = data.get('front_photo_on_screen_result')
@@ -1423,35 +1087,18 @@ def visual_authenticity_check(data, front_id_text, back_id_text, selfie, facial_
             visual_authenticity['breakdown']['original_document_present']['properties']['photo_of_screen'] = 'consider'
 
     if country == 'UAE' and not data.get('uae_pass_data', ''):
-        doc_type2 = identify_document_type(back_id_text, country)
-        if doc_type1 == 'EID' and doc_type2 == 'EID':
-            visual_authenticity['breakdown']['original_document_present']['properties']['scan'] = 'clear'
-        else:
-            visual_authenticity['breakdown']['original_document_present']['properties']['scan'] = 'consider'
-
-    if country == 'SAU':
-        if identify_front_id(front_id_text):
-            visual_authenticity['breakdown']['original_document_present']['properties']['scan'] = 'clear'
-        else:
-            visual_authenticity['breakdown']['original_document_present']['properties']['scan'] = 'consider'
-    
-        visual_authenticity['breakdown']['original_document_present']['properties']['document_on_printed_paper'] = front_doc_on_pp
-        visual_authenticity['breakdown']['original_document_present']['properties']['screenshot'] = front_screenshot
-        visual_authenticity['breakdown']['original_document_present']['properties']['photo_of_screen'] = front_photo_on_screen_result
-
+        visual_authenticity['breakdown']['original_document_present']['properties']['scan'] = 'clear'
+        visual_authenticity['breakdown']['face_detection'] = 'clear'
+        visual_authenticity['breakdown']['security_features'] = 'clear'
+       
     if country == 'IRQ':
         if data.get('doc_type') == 'passport':
-            if 'passport' and ('iraq' or 'republic of iraq') in data.get('passport_data').lower():
-                visual_authenticity['breakdown']['original_document_present']['properties']['scan'] = 'clear'
-            else:
-                visual_authenticity['breakdown']['original_document_present']['properties']['scan'] = 'clear'
+            visual_authenticity['breakdown']['original_document_present']['properties']['scan'] = 'clear'
         else:
-           
             visual_authenticity['breakdown']['original_document_present']['properties']['scan'] = 'clear'
             visual_authenticity['breakdown']['template']['result'] = data.get('front_template_result')
             visual_authenticity['breakdown']['digital_tampering']['result'] = data.get('tampering_result')
-
-
+            
     if country == 'LBN':
         visual_authenticity['breakdown']['original_document_present']['properties']['scan'] = 'clear'
         visual_authenticity['breakdown']['original_document_present']['properties']['screenshot'] = 'clear'
@@ -1466,7 +1113,6 @@ def visual_authenticity_check(data, front_id_text, back_id_text, selfie, facial_
         visual_authenticity['breakdown']['digital_tampering']['result'] = 'clear'
         visual_authenticity['breakdown']['template']['result'] = 'clear'
 
-    
     if country in ['SDN', 'SYR', 'JOR', 'PSE']:
         visual_authenticity['breakdown']['original_document_present']['properties']['scan'] = 'clear'
         visual_authenticity['breakdown']['original_document_present']['properties']['screenshot'] = 'clear'
@@ -1478,12 +1124,9 @@ def visual_authenticity_check(data, front_id_text, back_id_text, selfie, facial_
 
     ## set default to avoid any problems of cache
     visual_authenticity['result'] = 'clear'
-
     final_result = create_final_result(visual_authenticity)
     visual_authenticity['result'] = final_result
 
-    # print(f"\n\n------------------------- VISUAL AUTHENTICITY DOC: {visual_authenticity}")
-
     back_doc_on_pp = data.get('doc_on_pp')
     if front_doc_on_pp == 'consider' or back_doc_on_pp == 'consider':
         visual_authenticity['breakdown']['original_document_present']['properties']['document_on_printed_paper'] = 'consider'
@@ -1493,7 +1136,7 @@ def visual_authenticity_check(data, front_id_text, back_id_text, selfie, facial_
         visual_authenticity['breakdown']['original_document_present']['properties']['document_on_printed_paper'] = 'clear'
         visual_authenticity['result'] = 'clear'
 
-    logging.info(f"\n\n---------------------------- VISUAL AUTHENTICITY: {visual_authenticity}")
+    logging.info(f"---------------------------- VISUAL AUTHENTICITY: {visual_authenticity}")
     return visual_authenticity
 
 def main_details(data, country):
@@ -1689,15 +1332,30 @@ def main_details(data, country):
         for key in keys:
             main_properties[key] = data.get(key, '')
         
-        if country in ['IRQ', 'LBN', 'QAT', 'SDN', 'SYR', 'JOR', 'PSE']:
+        if country in ['IRQ', 'LBN', 'QAT', 'SDN', 'SYR', 'JOR', 'PSE', 'UAE']:
+
+            if country == 'UAE':
+                main_prop_data = {}
+                if data.get('doc_type') == 'national_identity_card':
+                    keys = ['dob_mrz', 'expiry_date_mrz', 'id_number_mrz', 'name_mrz', 'card_number_mrz','is_name_match_mrz',
+                            'id_number_front_back_mrz_match','card_number_back_mrz_match','dob_mrz_match','expiry_date_mrz_match','gender_mrz_match','name_mrz_match'
+                            ]
+                    keys_to_remove = []
             if country == 'IRQ':
                 main_prop_data = {}
                 if data.get('doc_type') == 'passport':
+                    logging.info(f"Extracting main details for IRQ passport")
                     keys_to_remove = ['issuing_place', 'occupation', 'employer', 'mrz_line3', 'family_sponsor', 'issuing_authority_en', 'place_of_birth_en', 'gender_ar', 'name_en', 'name', 'first_name']
-                    keys = ['full_name', 'id_number', 'mrz', 'issue_date','issuing_authority', 'mother_name', 'place_of_birth', 'mother_first_name', 'mother_first_name_en', 'mother_last_name', 'mother_last_name_en']
+                    keys = ['full_name', 'id_number', 'mrz', 'issue_date','issuing_authority', 'mother_name', 'place_of_birth', 'mother_first_name', 'mother_first_name_en', 'mother_last_name',
+                             'mother_last_name_en','expiry_date_mrz','dob_mrz', 'gender_mrz','id_number_mrz','name_mrz','is_name_match_mrz',
+                            'is_dob_match_mrz', 'is_id_number_match_mrz','is_expiry_date_match_mrz','is_valid_id_duration','is_valid_id_duration_mrz','is_gender_mrz_match']
                 else:
                     keys_to_remove = ['occupation', 'employer', 'family_sponsor', 'issuing_place']
-                    keys = ['name_en', 'name', 'first_name', 'father_name', 'third_name', 'mother_first_name', 'mother_last_name', 'last_name', 'first_name_en', 'father_name_en', 'last_name_en', 'third_name_en', 'mother_first_name_en', 'mother_last_name_en', 'issuing_authority', 'nationality', 'gender', 'issuing_country', 'gender_ar', 'mrz', 'place_of_birth', 'issuing_authority_en', 'place_of_birth_en', "issue_date", "id_number", "card_number", 'family_number', 'family_number_en']
+                    keys = ['name_en', 'name', 'first_name', 'father_name', 'third_name', 'mother_first_name', 'mother_last_name', 
+                            'last_name', 'first_name_en', 'father_name_en', 'last_name_en', 'third_name_en', 'mother_first_name_en',
+                            'mother_last_name_en', 'issuing_authority', 'nationality', 'gender', 'issuing_country', 'gender_ar', 'mrz',
+                            'place_of_birth', 'issuing_authority_en', 'place_of_birth_en', "issue_date", "id_number", "card_number", 'family_number', 'family_number_en',
+                            'card_number_mrz', 'id_number_mrz', 'valid_id_duration_mrz', 'dob_back_and_mrz_match', 'exp_date_back_and_mrz_match', 'gender_front_and_back_match', 'id_number_front_back_match']
 
             if country == 'LBN':
                 main_prop_data = {}
@@ -1749,18 +1407,25 @@ def main_details(data, country):
                 main_prop_data = {}
                 keys_to_remove = ['name', 'issuing_place', 'occupation', 'employer', 'mrz_line3', 'family_sponsor', 'issuing_country']
                 if data.get('doc_type') == 'passport':
-                    keys = ['full_name', 'first_name', 'last_name', 'father_name', 'mother_name', 'place_of_birth', 'place_of_issue', 'dob', 'nationality', 'gender', 'passport_number', 'issue_number', 'issuing_date', 'national_number', 'mrz1', 'mrz2', 'mrz', 'issuing_country']
+                    keys = ['full_name', 'first_name', 'last_name', 'father_name', 'mother_name', 'place_of_birth', 'place_of_issue', 'dob', 'nationality', 'gender', 'passport_number', 'issue_number', 
+                            'issuing_date', 'national_number', 'mrz1', 'mrz2', 'mrz', 'issuing_country',
+                             'passport_number_mrz','name_mrz', 'dob_mrz', 'expiry_date_mrz','gender_mrz',
+                            'is_valid_id_duration','is_valid_id_duration_mrz','is_passport_number_match_mrz','is_dob_match_mrz','is_gender_mrz_match','is_name_match_mrz']
 
 
             if country == 'PSE':
                 main_prop_data = {}
                 keys_to_remove = ['name', 'issuing_place', 'occupation', 'employer', 'mrz_line3', 'family_sponsor']
                 if data.get('doc_type') == 'passport':
-                    keys = ['full_name', 'first_name', 'last_name', 'mother_name', 'place_of_birth', 'dob', 'issuing_date', 'nationality', 'gender', 'place_of_issue', 'passport_number', 'id_number', 'mrz1', 'mrz2', 'mrz', 'issuing_country']
+                    keys = ['full_name', 'first_name', 'last_name', 'mother_name', 'place_of_birth', 'dob', 'issuing_date', 'nationality', 'gender',
+                            'place_of_issue', 'passport_number', 'id_number', 'mrz1', 'mrz2', 'mrz', 'issuing_country',
+                             'passport_number_mrz','name_mrz', 'dob_mrz', 'expiry_date_mrz','gender_mrz',
+                            'is_valid_id_duration','is_valid_id_duration_mrz','is_passport_number_match_mrz','is_dob_match_mrz','is_gender_mrz_match','is_name_match_mrz']
 
             for key in keys:
                 main_prop_data[key] =  data.get(key, '')
 
+            logging.info(f"Main properties before removal: {main_prop_data.keys()}")
             if keys_to_remove:
                 for key in keys_to_remove:
                     if key in main_properties:
@@ -1771,7 +1436,6 @@ def main_details(data, country):
             if main_properties.get('issue_date', ''):
                 main_properties['issue_date'] = onfido_date_format(get_dates_to_generic_format(main_properties['issue_date']))
 
-            
 
             if not data.get('card_number') and data.get('card_number_front'):
                 card_data_t = {
@@ -1823,7 +1487,6 @@ def form_final_data_document_report(data, front_id_text, back_id_text, country,
             dob = data.get('dob', '')
 
         document_report = {
-            ## pending - to be filled by dev
             "_id": "",
             "breakdown": {
                 "age_validation": age_validation(dob),
@@ -1847,20 +1510,16 @@ def form_final_data_document_report(data, front_id_text, back_id_text, country,
                 "police_record": {},
                 "visual_authenticity": visual_authenticity_check(data, front_id_text, back_id_text, selfie, facial_similarity, face_match_threshold, country)
             },
-            ## pending - to be filled by dev
             "check_id": "", 
             "created_at": created_at(),
             "documents": [
                 {
-                ## pending - id value in table stored in db for front id - to be filled by dev
                 "id": ""
                 },
                 {
-                ## pending - id value in table stored in db for front id - to be filled by dev
                 "id": ""
                 }
             ],
-            # to be filled by dev
             "href": "",
             "name": "document",
             "properties": main_details(data, country),
@@ -1960,7 +1619,6 @@ def form_final_facial_similarity_report(data, selfie, facial_similarity, livenes
             facial_report['breakdown']['visual_authenticity']['breakdown']['spoofing_detection']['properties']['score'] = ''
             facial_report['breakdown']['visual_authenticity']['breakdown']['spoofing_detection']['result'] = ''
 
-        # if np.any(selfie) and liveness_result:
         visual_authenticity_final_result = create_final_result(facial_report['breakdown']['visual_authenticity'])
         facial_report['breakdown']['visual_authenticity']['result'] = visual_authenticity_final_result