idvpackage 3.0.11__py3-none-any.whl → 3.0.12__py3-none-any.whl

idvpackage/ocr.py

@@ -1,39 +1,30 @@
-from google.oauth2.service_account import Credentials
-from google.cloud import vision_v1
-import json
-import pycountry
-from googletrans import Translator
-from PIL import Image, ImageEnhance, ImageOps, ImageFilter
 import base64
 import io
-import numpy as np
-import cv2
-from datetime import datetime, timedelta
-import tempfile
-from rapidfuzz import fuzz
-import face_recognition
-import re
+import json
+import logging
 import os
-from idvpackage.constants import BRIGHTNESS_THRESHOLD, BLUR_THRESHOLD
-from io import BytesIO
+import tempfile
 import time
-import logging
-
-# import anthropic
-import openai
-from idvpackage.blur_detection import is_image_blur
-# from idvpackage.common import (
-#     # load_and_process_image_deepface,
-#     load_and_process_image_deepface_all_orientations
-# )
+from datetime import datetime, timedelta
+from io import BytesIO
 
+import cv2
+import face_recognition
+import numpy as np
+import pycountry
+from google.cloud import vision_v1
+from google.oauth2.service_account import Credentials
+from PIL import Image, ImageEnhance
 
-import openai
-from concurrent.futures import ThreadPoolExecutor, as_completed
-# from idvpackage.blur_detection import is_image_blur
+from idvpackage.constants import BLUR_THRESHOLD, BRIGHTNESS_THRESHOLD
 
 
-logging.basicConfig(level=logging.INFO)
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s - %(levelname)s - %(filename)s:%(lineno)d - %(message)s",
+    datefmt="%Y-%m-%d %H:%M:%S",
+    force=True,
+)
 
 google_client_dict = {}
 
@@ -44,54 +35,24 @@ class IdentityVerification:
         This is the initialization function of a class that imports a spoof model and loads an OCR
         reader.
         """
-
-    # logs all keys
-    # logging.info(f"API Key: {api_key[0:8]}, GenAI Key: {genai_key}")
         try:
-        # TODO: Make gemni and openai keys work for stg and dev environment. It's different argument placement. One code to work for both stg and dev.
             st = time.time()
             credentials_dict = json.loads(credentials_string)
             credentials = Credentials.from_service_account_info(credentials_dict)
-
             self.client = google_client_dict.get(credentials)
+
             if not self.client:
                 self.client = vision_v1.ImageAnnotatorClient(credentials=credentials)
                 google_client_dict[credentials] = self.client
 
-
-            # self.genai_client = anthropic.Anthropic(
-            #         api_key=genai_key,
-            #     )
-
             self.openai_key = genai_key
 
-            self.translator = Translator()
             self.iso_nationalities = [country.alpha_3 for country in pycountry.countries]
             logging.info(f"\nInitialization time inside IDV Package: {time.time() - st}")
         
         except Exception as e:
             logging.error(f"Error during initialization: {e}")
 
-
-    def preprocess_image(
-        self, image, sharpness=1.0, contrast=2.0, radius=2, percent=150, threshold=3
-    ):
-        """Preprocess the image by sharpening and enhancing contrast."""
-
-        # Apply sharpening
-        enhancer = ImageEnhance.Sharpness(image)
-        image = enhancer.enhance(sharpness)  # Sharpen the image (increase sharpness)
-
-        # Enhance the contrast
-        enhancer = ImageEnhance.Contrast(image)
-        image = enhancer.enhance(contrast)  # Increase contrast
-
-        image = image.filter(
-            ImageFilter.UnsharpMask(radius=radius, percent=percent, threshold=threshold)
-        )
-
-        return image
-
     def image_conversion(self, image):
         """
         This function decodes a base64 string data and returns an image object.
@@ -113,7 +74,7 @@ class IdentityVerification:
                 img = img.copy()
             return img
 
-    def rgb2yuv(self, img):
+   
         """
         Convert an RGB image to YUV format.
         """
@@ -123,50 +84,11 @@ class IdentityVerification:
         except Exception as e:
             raise Exception(f"Error: {e}")
 
-    def find_bright_areas(self, image, brightness_threshold):
-        gray_image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
-        thresh_image = cv2.threshold(
-            gray_image, brightness_threshold, 255, cv2.THRESH_BINARY
-        )[1]
-        contours, hierarchy = cv2.findContours(
-            thresh_image, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE
-        )
-
-        bright_areas = []
-
-        for contour in contours:
-            bounding_box = cv2.boundingRect(contour)
-
-            area = bounding_box[2] * bounding_box[3]
-
-            if area > 800:
-                bright_areas.append(bounding_box)
-
-        return len(bright_areas)
-
     def is_blurry(self, image):
         gray_image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
-
         laplacian_variance = cv2.Laplacian(gray_image, cv2.CV_64F).var()
-
         return laplacian_variance
-
-    def identify_input_type(self, data):
-        if isinstance(data, bytes):
-            return "video_bytes"
-        else:
-            pass
-
-        try:
-            decoded_data = base64.b64decode(data)
-
-            if decoded_data:
-                return "base_64"
-        except Exception:
-            pass
-
-        return "unknown"
-
+    
     def sharpen_image(self, image):
         kernel = np.array([[-1, -1, -1], [-1, 9, -1], [-1, -1, -1]])
         return cv2.filter2D(image, -1, kernel)
@@ -183,16 +105,8 @@ class IdentityVerification:
         enhanced_image = enhancer.enhance(factor)
         return np.array(enhanced_image)
 
-    def enhance_quality(self, image):
-        sharpened_image = self.sharpen_image(image)
-        enhanced_image = self.adjust_brightness(sharpened_image, 1.2)
-        enhanced_contrast = self.adjust_contrast(enhanced_image, 1.2)
-        # grayscale_image = cv2.cvtColor(enhanced_contrast, cv2.COLOR_BGR2GRAY)
-
-        return enhanced_contrast
-
-
     def check_document_quality(self, data):
+        
         video_quality = {"error": ""}
         temp_video_file = tempfile.NamedTemporaryFile(delete=False)
         temp_video_file_path = temp_video_file.name
@@ -234,14 +148,12 @@ class IdentityVerification:
             return video_quality
         
         finally:
-              
             if video_capture is not None:
                 try:
                     video_capture.release()
                 except Exception:
                     pass
                 del video_capture
-
             # Remove temp file
             if temp_video_file_path and os.path.exists(temp_video_file_path):
                 try:
@@ -252,13 +164,10 @@ class IdentityVerification:
             # Force cleanup of OpenCV / numpy memory
             import gc
             gc.collect()
-        
 
     def extract_selfie_from_video(self, video_capture):
         """Extract the best quality selfie from video with speed optimizations for frontal faces."""
         video_dict = {'error': ''}
-        
-
         try:
             # Get rotation metadata from video
             try:
@@ -287,7 +196,6 @@ class IdentityVerification:
 
             best_face = None
             best_score = -1
-            best_frame_position = None
             best_frame = None
 
             logging.info(f"Analyzing video with {total_frames} frames")
@@ -321,10 +229,6 @@ class IdentityVerification:
                     encode_params = [cv2.IMWRITE_JPEG_QUALITY, 90]
                     _, buffer = cv2.imencode(".jpg", small_frame, encode_params)
 
-                    # image = vision_v1.Image(content=buffer.tobytes())
-                    # response = self.client.face_detection(image=image, max_results=2)
-                    # faces = response.face_annotations
-
                     image_bytes = buffer.tobytes()
                     del buffer
 
@@ -442,14 +346,6 @@ class IdentityVerification:
                             }
 
                     if frame_best_face is not None:
-
-                        
-                        # frame_results.append({
-                        #     'frame': target_frame,
-                        #     'face': frame_best_face,
-                        #     'score': frame_best_score,
-                        #     'frame_data': frame.copy()
-                        # })
                         logging.info(f"Frame {target_frame}: Best face score {frame_best_score:.2f} "
                                      f"(Frontal: {frame_best_face['frontal_score']:.2f}, "
                                      f"Center: {frame_best_face['center_score']:.2f}, "
@@ -460,29 +356,11 @@ class IdentityVerification:
                             best_score = frame_best_score
                             best_face = frame_best_face
                             best_frame = frame.copy()
-                            best_frame_position = target_frame
 
                 except Exception as e:
                     logging.info(f"Error processing frame {target_frame}: {e}")
                     continue
-
-            # # Process results
-            # if len(frame_results) > 0:
-            #     # Sort faces by score
-            #     frame_results.sort(key=lambda x: x['score'], reverse=True)
-
-                # for i, result in enumerate(frame_results[:min(3, len(frame_results))]):
-                #     face_info = result['face']
-                #     print(f"Rank {i+1}: Frame {face_info['frame']}, "
-                #           f"Score: {result['score']:.2f}, "
-                #           f"Frontal: {face_info['frontal_score']:.2f}, "
-                #           f"Center: {face_info['center_score']:.2f}")
-
-                # Use the best frame
-                # best_frame = frame_best_face['frame']
-
-                # print(f"Selected frame {best_face['frame']} as best selfie")
-
+                
             if best_face and best_frame is not None:
                 try:
                     left = best_face['left']
@@ -499,7 +377,7 @@ class IdentityVerification:
                         video_dict['error'] = 'invalid_cropped_face'
                         return video_dict
 
-                    print(f"Face shape: {cropped_face.shape}")
+                    logging.info(f"Face shape: {cropped_face.shape}")
                     return cropped_face
 
                 except Exception as e:
@@ -514,8 +392,6 @@ class IdentityVerification:
             video_dict['error'] = 'video_processing_error'
             return video_dict
 
-        
-
     def is_colored(self, base64_image):
         img = self.image_conversion(base64_image)
         img = np.array(img)
@@ -541,46 +417,7 @@ class IdentityVerification:
 
         return blurred, glare
 
-    def standardize_date(self, input_date):
-        input_formats = [
-            "%Y/%m/%d",
-            "%m/%d/%Y",
-            "%m-%d-%Y",
-            "%Y-%m-%d",
-            "%d-%m-%Y",
-            "%d/%m/%Y",
-            "%Y%m%d",
-            "%m%d%Y",
-            "%d%m%Y",
-            "%Y.%m.%d",
-            "%d.%m.%Y",
-            "%m.%d.%Y",
-            "%Y %m %d",
-            "%d %m %Y",
-            "%m %d %Y",
-        ]
-
-        for format in input_formats:
-            try:
-                parsed_date = datetime.strptime(input_date, format)
-                standardized_date = parsed_date.strftime("%d/%m/%Y")
-                return standardized_date
-            except ValueError:
-                pass
-
-        return None
-
-    def compare_dates(self, date_str1, date_str2):
-        date_format = "%d/%m/%Y"
-
-        date1 = datetime.strptime(date_str1, date_format)
-        date2 = datetime.strptime(date_str2, date_format)
-
-        if date1 == date2:
-            return True
-        else:
-            return False
-
+    
     def check_nationality_in_iso_list(self, nationality):
         try:
             if len(nationality) > 3:
@@ -589,185 +426,22 @@ class IdentityVerification:
                     nationality = country.alpha_3
                 except:
                     return "consider"
-
             ## Handling case for OMN as it comes as MN, due to O being considered as 0
             if nationality.upper() == "MN":
                 nationality = "OMN"
-
             if nationality.upper() in self.iso_nationalities:
                 return "clear"
             else:
                 return "consider"
-
         except:
             return "consider"
 
-    def get_face_orientation(self, face_landmarks):
-        left_eye = np.array(face_landmarks["left_eye"]).mean(axis=0)
-        right_eye = np.array(face_landmarks["right_eye"]).mean(axis=0)
-
-        eye_slope = (right_eye[1] - left_eye[1]) / (right_eye[0] - left_eye[0])
-        angle = np.degrees(np.arctan(eye_slope))
-
-        return angle
-
-    def rotate_image(self, img):
-        from skimage.transform import radon
-
-        img_array = np.array(img)
-
-        if len(img_array.shape) == 2:
-            gray = img_array
-        else:
-            gray = cv2.cvtColor(img_array, cv2.COLOR_RGB2GRAY)
-
-        h, w = gray.shape
-        if w > 640:
-            gray = cv2.resize(gray, (640, int((h / w) * 640)))
-        gray = gray - np.mean(gray)
-        sinogram = radon(gray)
-        r = np.array(
-            [np.sqrt(np.mean(np.abs(line) ** 2)) for line in sinogram.transpose()]
-        )
-        rotation = np.argmax(r)
-        angle = round(abs(90 - rotation) + 0.5)
-
-        if abs(angle) > 5:
-            rotated_img = img.rotate(angle, expand=True)
-            return rotated_img
-
-        return img
-
-    def load_and_process_image_fr(self, base64_image, arr=False):
-        try:
-            if not arr:
-                img = self.image_conversion(base64_image)
-                img = np.array(img)
-                image = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
-            else:
-                if base64_image.dtype != np.uint8:
-                    base64_image = base64_image.astype(np.uint8)
-
-                image = cv2.cvtColor(base64_image, cv2.COLOR_BGR2RGB)
-
-            # base64_image = base64_image.split(',')[-1]
-            # image_data = base64.b64decode(base64_image)
-            # image_file = io.BytesIO(image_data)
-
-            # image = face_recognition.load_image_file(image_file)
-
-            face_locations = []
-            face_locations = face_recognition.face_locations(image)
-
-            if not face_locations:
-                return [], []
-
-            face_encodings = []
-            face_encodings = face_recognition.face_encodings(image, face_locations)
-
-            return face_locations, face_encodings
-        except:
-            return [], []
-
     def calculate_similarity(self, face_encoding1, face_encoding2):
         similarity_score = (
             1 - face_recognition.face_distance([face_encoding1], face_encoding2)[0]
         )
         return round(similarity_score + 0.25, 2)
 
-    def try_detect_face_at_angles(self, image_array):
-        """Try detecting faces at different angles and return the first successful detection"""
-        angles = [0, 90, 180, 270]
-        best_result = ([], [])
-
-        for angle in angles:
-            if angle == 0:
-                rotated = image_array
-            else:
-                rotated = np.array(
-                    Image.fromarray(image_array).rotate(angle, expand=True)
-                )
-
-            face_locs, face_encs = self.load_and_process_image_fr(rotated, arr=True)
-            if face_locs:
-                return face_locs, face_encs, angle
-
-        return [], [], None
-
-    def try_detect_face_parallel(self, image_array):
-        """Try detecting faces at different angles in parallel"""
-
-        def check_angle(angle):
-            try:
-                # Create a copy of the image array to prevent memory issues
-                image_copy = np.copy(image_array)
-
-                if angle == 0:
-                    rotated = image_copy
-                else:
-                    # Convert to PIL, rotate, and convert back to numpy
-                    pil_image = Image.fromarray(image_copy)
-                    rotated_pil = pil_image.rotate(angle, expand=True)
-                    rotated = np.array(rotated_pil)
-
-                # Ensure the array is contiguous
-                if not rotated.flags["C_CONTIGUOUS"]:
-                    rotated = np.ascontiguousarray(rotated)
-
-                face_locs, face_encs = self.load_and_process_image_fr(rotated, arr=True)
-                return (face_locs, face_encs, angle)
-            except Exception as e:
-                print(f"Error processing angle {angle}: {e}")
-                return ([], [], None)
-
-        angles = [0, 90, 180, 270]
-        results = []
-
-        try:
-            with ThreadPoolExecutor(max_workers=4) as executor:
-                futures = [executor.submit(check_angle, angle) for angle in angles]
-
-                for future in as_completed(futures):
-                    try:
-                        face_locs, face_encs, angle = future.result()
-                        if face_locs:
-                            # Cancel remaining futures
-                            for f in futures:
-                                f.cancel()
-                            return face_locs, face_encs, angle
-                        results.append((face_locs, face_encs, angle))
-                    except Exception as e:
-                        print(f"Error getting future result: {e}")
-                        continue
-        except Exception as e:
-            print(f"Error in parallel processing: {e}")
-
-        return [], [], None
-
-    def try_detect_face_deepface(self, image_array):
-        """Try detecting faces using DeepFace"""
-        try:
-            from idvpackage.common import load_and_process_image_deepface
-
-            # Ensure image is in correct format
-            if image_array is None:
-                print("Image array is None")
-                return [], [], None
-
-            # Ensure we're working with RGB
-            if len(image_array.shape) == 3 and image_array.shape[2] == 3:
-                # If image is BGR, convert to RGB
-                if image_array.dtype != np.uint8:
-                    image_array = image_array.astype(np.uint8)
-                # No need to convert color space as the image should already be in RGB
-                face_locs, face_encs = load_and_process_image_deepface(image_array)
-                return face_locs, face_encs, None
-            else:
-                print(f"Unexpected image shape: {image_array.shape}")
-                return [], [], None
-        except Exception as e:
-            print(f"Error in try_detect_face_deepface: {e}")
-            return [], [], None
 
     def extract_face_and_compute_similarity(
         self, selfie, front_face_locations, front_face_encodings
@@ -776,7 +450,7 @@ class IdentityVerification:
 
         try:
             if selfie is None:
-                print("Error: Selfie image is None")
+                logging.info("Error: Selfie image is None")
                 return 0
 
             # Ensure the input array is contiguous and in the correct format
@@ -790,103 +464,31 @@ class IdentityVerification:
                 else:
                     selfie = selfie.astype(np.uint8)
 
-            # Try DeepFace first as it's generally more reliable
-            # start_time = time.time()
             face_locations1, face_encodings1 = load_and_process_image_deepface(selfie)
-            # end_time = time.time()
-
+            
             if not face_locations1 or not face_encodings1:
-                print("No face detected in Selfie Video by DeepFace")
+                logging.info("No face detected in Selfie Video by DeepFace")
                 return 0
-
-            # print(f"Face detection took {end_time - start_time:.3f} seconds")
-
             face_locations2, face_encodings2 = (
                 front_face_locations,
                 front_face_encodings,
             )
-
+            
             if not face_encodings2.any():
-                print("No face detected in front ID")
+                logging.info("No face detected in front ID")
                 return 0
 
-            largest_face_index1 = face_locations1.index(
-                max(
-                    face_locations1,
-                    key=lambda loc: (loc[2] - loc[0]) * (loc[3] - loc[1]),
-                )
-            )
-            largest_face_index2 = face_locations2.index(
-                max(
-                    face_locations2,
-                    key=lambda loc: (loc[2] - loc[0]) * (loc[3] - loc[1]),
-                )
-            )
-
-            face_encoding1 = face_encodings1[largest_face_index1]
-            face_encoding2 = face_encodings2[largest_face_index2]
-
-            similarity_score = self.calculate_similarity(face_encoding1, face_encoding2)
-            # print(f"Calculated similarity score: {similarity_score}")
-
+            similarity_score = self.calculate_similarity(face_encodings1[0], face_encodings2[0])
+            logging.info(f"Similarity score between selfie and front ID: {similarity_score}")
             return min(1, similarity_score)
 
         except Exception as e:
-            print(f"Error in extract_face_and_compute_similarity: {e}")
-            import traceback
-
-            traceback.print_exc()
+            logging.info(f"Error in extract_face_and_compute_similarity: {e}")
+            
             return 0
 
-    def calculate_landmarks_movement(self, current_landmarks, previous_landmarks):
-        return sum(
-            abs(cur_point.position.x - prev_point.position.x)
-            + abs(cur_point.position.y - prev_point.position.y)
-            for cur_point, prev_point in zip(current_landmarks, previous_landmarks)
-        )
-
-    def calculate_face_movement(self, current_face, previous_face):
-        return abs(current_face[0].x - previous_face[0].x) + abs(
-            current_face[0].y - previous_face[0].y
-        )
-
-    def calculate_liveness_result(
-        self, eyebrow_movement, nose_movement, lip_movement, face_movement
-    ):
-        eyebrow_movement_threshold = 15.0
-        nose_movement_threshold = 15.0
-        lip_movement_threshold = 15.0
-        face_movement_threshold = 10.0
-
-        if (
-            eyebrow_movement > eyebrow_movement_threshold
-            or nose_movement > nose_movement_threshold
-            or lip_movement > lip_movement_threshold
-            or face_movement > face_movement_threshold
-        ):
-            return True
-        else:
-            return False
-
-    def detect_image_format(self, base64_image):
-        import imghdr
 
-        decoded_image = base64.b64decode(base64_image)
-        format = imghdr.what(None, decoded_image)
-
-        return format
-
-    def frame_count_and_save(self, cap):
-        frames = []
-        status, frame = cap.read()
-        while status:
-            frames.append(frame)
-            status, frame = cap.read()
-
-        cap.release()
-        return frames
-
-    def check_for_liveness(self, similarity, video_bytes, face_match_threshold=0.59):
+    def check_for_liveness(self, video_bytes):
         st = time.time()
         from idvpackage.liveness_spoofing_v2 import test
 
@@ -905,109 +507,20 @@ class IdentityVerification:
                 if result:
                     return result
             except Exception as e:
-                print(f"\nError in Liveness: {e}")
+                logging.info(f"\nError in Liveness: {e}")
                 return None
 
         finally:
             # Ensure the temporary file is deleted
             if os.path.exists(temp_video_file_path):
                 os.remove(temp_video_file_path)
-                # print(f"Temporary file {temp_video_file_path} has been deleted.")
+                # logging.info(f"Temporary file {temp_video_file_path} has been deleted.")
             logging.info(
                 f"--------------Time taken for Liveness and Spoofing in IDV package: {time.time() - st} seconds\n"
             )
 
-    def reverse_date(self, date_str: str) -> str:
-        """
-        Convert a date from yyyy/mm/dd to dd/mm/yyyy, or vice versa.
-        Returns empty string if invalid.
-
-        TODO: vice versa conversion not implemented yet.
-        """
-        try:
-            dt = datetime.strptime(date_str.strip(), "%Y/%m/%d")
-            return dt.strftime("%d/%m/%Y")
-        except ValueError:
-            return ""
-
-    # Example
-
-    def convert_dob(self, input_date):
-        day = input_date[4:6]
-        month = input_date[2:4]
-        year = input_date[0:2]
-
-        current_year = datetime.now().year
-        current_century = current_year // 100
-        current_year_last_two_digits = current_year % 100
-
-        century = current_century
-        # If the given year is greater than the last two digits of the current year, assume last century
-        if int(year) > current_year_last_two_digits:
-            century = current_century - 1
-
-        final_date = f"{day}/{month}/{century}{year}"
-
-        return final_date
-
-    def convert_expiry_date(self, input_date):
-        day = input_date[4:6]
-        month = input_date[2:4]
-        year = input_date[0:2]
-
-        current_year = datetime.now().year
-        current_century = current_year // 100
-        current_year_last_two_digits = current_year % 100
-        century = current_century
-
-        if int(year) <= current_year_last_two_digits:
-            century = current_century
-        else:
-            century = current_century
-        final_date = f"{day}/{month}/{century}{year}"
-
-        return final_date
-
-    def clean_string(self, input_string):
-        cleaned_string = re.sub(r"[^\w\s]", " ", input_string)
-        return cleaned_string.strip()
-
-    def find_and_slice_number(input_number, digits):
-        from itertools import permutations
-
-        # Generate all possible permutations of the digits
-        perms = ["".join(p) for p in permutations(digits)]
-
-        # Initialize variables to keep track of the found pattern and its index
-        found_pattern = None
-        found_index = -1
-
-        # Search for any permutation of the digits in the input_number
-        for perm in perms:
-            found_index = input_number.find(perm)
-            if found_index != -1:
-                found_pattern = perm
-                break
-
-        # If a pattern is found, slice the number accordingly
-        if found_pattern:
-            if found_index > len(input_number) - found_index - len(found_pattern):
-                # Slice to the left
-                sliced_number = input_number[: found_index + len(found_pattern)]
-            else:
-                # Slice to the right
-                sliced_number = input_number[found_index:]
-
-            return sliced_number
-        else:
-            return ""
-
     def get_ocr_results(self, processed_image, country=None, side=None):
-        # with io.BytesIO() as output:
-        #     processed_image.save(output, format="PNG")
-        #     image_data = output.getvalue()
-
-        # image = vision_v1.types.Image(content=image_data)
+        
         logging.info(f"Getting OCR results for country: {country}, side: {side}")
 
         if country == "QAT" or country == "LBN" or country == "IRQ" or country == "SDN":
@@ -1031,36 +544,36 @@ class IdentityVerification:
     ):
         st = time.time()
         document_data = {}
-      
+        
         logging.info(f"Starting extraction for document_type: {document_type}, country: {country}, side: {side}, nationality: {nationality}, \n step data: {step_data}")
 
         if country == "IRQ":
             document_data = self.agent_extraction(
-                image, country, nationality, side, step_data
+                image, country,document_type, nationality, side, step_data
             )
             logging.info(
-                f"--------------Time taken for Front ID Extraction in IDV package: {time.time() - st} seconds\n"
+                f"--------------Time taken for {side} ID Extraction in IDV package: {time.time() - st} seconds\n"
             )
             return document_data
 
         if document_type == "national_id" and side == "front":
             document_data = self.extract_front_id_info(image, country, nationality)
             logging.info(
-                f"--------------Time taken for Front ID Extraction in IDV package: {time.time() - st} seconds\n"
+                f"--------------Time taken for {side} ID Extraction in IDV package: {time.time() - st} seconds\n"
             )
             return document_data
 
         if document_type == "national_id" and side == "back":
             document_data = self.extract_back_id_info(image, country, nationality, step_data)
             logging.info(
-                f"--------------Time taken for Back ID Extraction in IDV package: {time.time() - st} seconds\n"
+                f"--------------Time taken for {side} ID Extraction in IDV package: {time.time() - st} seconds\n"
             )
 
         if document_type == "passport" and (
             side == "first" or side == "page1" or side == ""
         ):
             logging.info(f"Starting passport extraction for side: {side} country: {country} nationality: {nationality}")
-            document_data = self.exract_passport_info(
+            document_data = self.extract_passport_info(
                 image, country, nationality, step_data
             )
             logging.info(
@@ -1078,326 +591,123 @@ class IdentityVerification:
 
         return document_data
 
-    
-    def agent_extraction(self, front_id, country, nationality,side, step_data=None):
-        from idvpackage.ocr_utils import detect_photo_on_screen, detect_screenshot, document_on_printed_paper
-        from idvpackage.common import get_facial_encodings_deepface_irq, load_and_process_image_deepface, cosine_similarity
-        from idvpackage.iraq_id_extraction_withopenai import extraction_chain
-        import idvpackage.genai_utils as sanity_utils
-        result = {'error': '', "error_details": ''}
-
-        logging.info(f'Starting agent_extraction for country: {country}, nationality: {nationality}, side: {side}')
-        try:
-            st = time.time()
-            processed_front_id = self.image_conversion(front_id)
-            logging.info(f'----------------Time taken for image conversion: {time.time() - st} seconds\n')
-            compressed_image = BytesIO()
-            processed_front_id.save(compressed_image, format="JPEG", quality=85, optimize=True)
-            compressed_image_data = compressed_image.getvalue()
-
-            st = time.time()
-            front_id_text = self.get_ocr_results(compressed_image_data, country=country)
-            front_id_text_desc = front_id_text[0].description
-            logging.info(f'----------------Time taken for Google Vision API call: {time.time() - st} seconds\n')
+    def agent_extraction(self, image, country, document_type, nationality=None, side=None, step_data=None):
 
-            #To allow for non-iraqi passports
 
-            #when uploading passports later, after one has 'skipped passport' during onboading, we get nationality=None
+          #Handling non-iraqi passports
+        if nationality is not None and nationality != "IRQ" and (side == "page1" or side == "first"):
+            logging.info(f"Starting non-iraqi passport extraction for side: {side} country: {country} nationality: {nationality}")
+            data = self.extract_passport_info(image, country, nationality, step_data)
+            return data
 
+        elif nationality is not None and nationality != "IRQ" and (side == "page2" or side == "last"):
+            logging.info(f"Starting non-iraqi passport extraction for side: {side} country: {country} nationality: {nationality}")
+            data = self.extract_passport_info_back(image, country, nationality, step_data)
+            return data
+        
 
-            if side in ['page1', 'first'] and nationality != 'IRQ' and nationality is not None:
-                front_data = self.exract_passport_info(front_id, country, nationality)
+        # Handling Iraqi National ID Front
+        elif  document_type == "national_id" and side == "front":
+            data = self.extract_front_id_info(image, country, nationality)
+            return data
+        
+        #Handling Iraqi National ID Back
+        elif document_type == "national_id" and side == "back":
+            data = self.extract_back_id_info(image, country, nationality, step_data)
+            return data
+        
+        #handling Iraqi passport 
+        elif document_type == "passport" and (side == "first" or side == "page1"):
+            data = self.extract_passport_info(image, country, nationality, step_data)
+            return data
 
-                if front_data.get("name", ''):
-                    front_data['full_name'] = front_data.get('name', '')
-                elif front_data.get('first_name') and front_data.get("last_name"):
-                    front_data['full_name'] = front_data.get("first_name", '') + ' ' + front_data.get("last_name", "")
+    def image_conversion_and_compression(self, image):
+        """
+        This function decodes a base64 string data and returns an image object.
+        If the image is in RGBA mode, it is converted to RGB mode.
+        :return: an Image object that has been created from a base64 encoded string.
+        """
+    
+        img_data = io.BytesIO(base64.decodebytes(bytes(image, "utf-8")))
+        with Image.open(img_data) as img:
+            if img.mode == "RGBA":
+                # Create a blank background image
+                background = Image.new("RGB", img.size, (255, 255, 255))
+                # Paste the image on the background.
+                background.paste(img, mask=img.split()[3])  # 3 is the alpha channel
+                img = background
+            else:
+                img = img.copy()
 
-                # handle expired passports
-                expiry_date = (
-                        front_data.get("expiry_date")
-                        or front_data.get("date_of_expiry")
-                )
-                if expiry_date:
-                    is_doc_expired = sanity_utils.is_expired_id(expiry_date)
+        compressed_image = BytesIO()
+        img.save(
+            compressed_image, format="JPEG", quality=75, optimize=True
+        )
+        compressed_image_data = compressed_image.getvalue()
 
-                    if is_doc_expired:
-                        return {"error": "expired_id", "error_details": "expired ID"}
+        return img,compressed_image_data
 
-                return front_data
+    def extract_front_id_info(self, front_id, country, nationality=None,):
 
-            if side in ['page2','last'] and nationality!='IRQ':
-                front_data = self.exract_passport_info_back(front_id,country,nationality)
+        if country == "UAE":
+            try:
 
-                if front_data.get("name",''):
-                    front_data['full_name'] = front_data.get('name','')
-                elif front_data.get('first_name') and front_data.get("last_name"):
-                    front_data['full_name'] = front_data.get("first_name",'')+ ' ' + front_data.get("last_name","")
+                from idvpackage.common import load_and_process_image_deepface
 
-                #handle expired passports
-                expiry_date = (
-                        front_data.get("expiry_date")
-                        or front_data.get("date_of_expiry")
-                )
-                if expiry_date:
-                    is_doc_expired = sanity_utils.is_expired_id(expiry_date)
+                from idvpackage.uae_id_extraction import get_response_from_openai_uae
 
-                    if is_doc_expired:
-                        return {"error": "expired_id", "error_details": "expired ID"}
+                front_data = {"error": "", "doc_type": "national_identity_card"}
 
-                return front_data
+                start_time = time.time()
+                logging.info(f"Starting extraction for UAE front ID")
+                output = get_response_from_openai_uae(front_id, "front", country, self.openai_key)
+                logging.info(f"Time taken for UAE front ID extraction: {time.time() - start_time} seconds")
+                
+                logging.info(f"UAE front ID extraction output: {json.dumps(output, ensure_ascii=False, indent=2)}")
+                front_data.update(output)
+                
+                if not output.get("header_verified", False):
+                    front_data["error"] = "not_front_id"
+                    return front_data
 
-            #the extra side here is for testing. So that when we test, we can pass in side='auto', instead of passing front and back seperately.
-            st = time.time()
-            result_extraction, side = extraction_chain(ocr_text=front_id_text_desc, openai_key=self.openai_key, side=side)
-            logging.info(f'----------------Time taken for Extraction Chain (openAI + langchain call): {time.time() - st} seconds\n')
+                dob = front_data.get('dob', '')
+                try:
+                    if  dob:
+                        logging.info(f"Extracted DOB for age verification: {dob}")
+                        from idvpackage.ocr_utils import is_age_18_above
+                        is_legal_age = is_age_18_above(dob)
+                        logging.info(f"Is legal age (18+): {is_legal_age}")
+                        if not is_legal_age:
+                            front_data['error'] = 'under_age'
+                            return front_data
 
-            if result_extraction['error']:
-                return result_extraction
-            result.update(result_extraction)
+                except Exception as e:
+                    logging.info(f"Error in age calculation: {e}")
+                    
+                    
+                expiry_date = front_data.get('expiry_date', '')
+                try:
+                    if expiry_date:
+                        logging.info(f"Extracted Expiry Date for expiry verification: {expiry_date}")
+                        from idvpackage.ocr_utils import is_expired_id
+                        if is_expired_id(expiry_date):
+                            front_data['error'] = 'expired_id'
+                            logging.info(f"ID is expired with expiry date: {expiry_date}")
+                            return front_data
+                except Exception as e:
+                    logging.info(f"Error in expiry date calculation: {e}")
 
+                        
 
-            st = time.time()
-            image = np.array(processed_front_id)
-            doc_on_pp_result = 'clear'
-            template_result = 'clear'
-            logo_result = 'clear'
+                front_face_locations, front_face_encodings = (
+                    load_and_process_image_deepface(front_id)
+                )
 
-            #uses google vision api
-            st = time.time()
-            # screenshot_result = detect_screenshot(self.client, front_id)
-            # #
-            # # #uses google vision api
-            # photo_on_screen_result = detect_photo_on_screen(self.client, front_id)
-            # valid_nationality_result = self.check_nationality_in_iso_list(result.get('nationality',''))
-            # front_blurred, front_glare = self.get_blurred_and_glared_for_doc(image)
-            #logging.info(f'----------------Time taken for fraud detection attributes: {time.time() - st} seconds\n')
-
-
-            logging.info(f'----------------Time taken for gvision api calls for: detect screensot, detect_photo_on_screen, get_blurred_and_glared_for_doc: {time.time() - st} seconds\n')
-
-            # if side=='front' or side=='page1':
-            #     st = time.time()
-            #     front_face_locations, front_face_encodings = load_and_process_image_deepface(front_id)
-            #     logging.info(f'----------------Time taken for face extraction: {time.time() - st} seconds\n')
-            #
-            #     front_face_locations_str = json.dumps([tuple(face_loc) for face_loc in front_face_locations])
-            #     front_face_encodings_str = json.dumps([face_enc.tolist() for face_enc in front_face_encodings])
-            #
-            #
-            #     if step_data:
-            #         try:
-            #             print(f"Matching face from Passport with National ID")
-            #             national_id_front_face_locations = step_data.get("front_face_locations")
-            #             national_id_front_face_encodings = json.loads(step_data.get("front_face_encodings"))
-            #             st = time.time()
-            #             national_id_front_face_encodings = np.array(national_id_front_face_encodings[0], dtype=np.float32)
-            #             similarity = self.calculate_similarity(national_id_front_face_encodings, front_face_encodings[0])
-            #             logging.info(f'----------------Time taken for face extraction for matching passport with front id: {time.time() - st} seconds\n')
-            #             result["similarity_score"] = similarity
-            #             print(f"Front ID and Passport similarity score: {similarity}")
-            #             if similarity <= 0.65:
-            #                 result["error"] = 'face_mismatch'
-            #                 return {"error":"face_mismatch", "error_details":"Front ID and Passport Face dont match."}
-            #         except Exception as e:
-            #             result["error"] = 'National ID Image Not Found'
-            #             result["error_details"] = e
-            #             return result
-
-
-            if side=='front' or side=='page1':
+                if front_face_encodings is None or len(front_face_encodings) == 0:
+                    front_data['error'] = 'face_not_detected'
+                    logging.info("No face detected in front image")
+                    return front_data
 
-                st = time.time()
-                front_face_locations, front_face_encodings = load_and_process_image_deepface(front_id)
-                logging.info(f'----------------Time taken for face extraction: {time.time() - st} seconds\n')
-
-                front_face_locations_str = json.dumps([tuple(face_loc) for face_loc in front_face_locations])
-                front_face_encodings_str = json.dumps([face_enc.tolist() for face_enc in front_face_encodings])
-
-
-                if step_data:
-                    try:
-                        print(f"Matching face from Passport with National ID")
-                        national_id_front_face_locations = json.loads(step_data.get("front_face_locations"))
-                        national_id_front_face_encodings = json.loads(step_data.get("front_face_encodings"))
-                        st = time.time()
-
-                        largest_face_index1 = national_id_front_face_locations.index(
-                            max(national_id_front_face_locations, key=lambda loc: (loc[2] - loc[0]) * (loc[3] - loc[1])))
-                        largest_face_index2 = front_face_locations.index(
-                            max(front_face_locations, key=lambda loc: (loc[2] - loc[0]) * (loc[3] - loc[1])))
-
-                        face_encoding1 = national_id_front_face_encodings[largest_face_index1]
-                        face_encoding2 = front_face_encodings[largest_face_index2]
-
-                        similarity = self.calculate_similarity(face_encoding1, face_encoding2)
-
-                        logging.info(
-                            f'----------------Time taken for face extraction for matching passport with front id: {time.time() - st} seconds\n')
-                        result["similarity_score"] = similarity
-                        print(f"Front ID and Passport similarity score: {similarity}")
-                        if similarity <= 0.65:
-                            result["error"] = 'face_mismatch'
-                            return {"error": "face_mismatch",
-                                    "error_details": "Front ID and Passport Face dont match."}
-
-                    except Exception as e:
-                        result["error"] = 'covered_photo'
-                        result["error_details"] = e
-                        return result
-
-                if side=='front':
-                    data_temp = {
-                        'front_extracted_data': front_id_text_desc,
-                        f'translated_{side}_id_text':'',
-                        'front_coloured': True,
-                        'back_coloured':True,
-                        'front_doc_on_pp': doc_on_pp_result,
-                        'front_logo_result': logo_result,
-                        'front_template_result': template_result,
-                        'front_screenshot_result': '',
-                        'front_photo_on_screen_result': '',
-                        'front_blurred': '',
-                        'front_glare': '',
-                        'front_face_locations': front_face_locations_str,
-                        'front_face_encodings': front_face_encodings_str,
-                        'front_tampered_result': 'clear',
-                        'issuing_country':'IRQ',
-                        'valid_nationality': 'valid_nationality_result'
-                    }
-
-                elif side=='page1':
-                    data_temp = {
-                        # 'back_tampered_result': tampered_result_back,
-                        'passport_data': front_id_text_desc,
-                        'front_coloured': True,
-                        'back_coloured': True,
-                        'front_logo_result': 'clear',
-                        'front_doc_on_pp': doc_on_pp_result,
-                        'front_screenshot_result': 'screenshot_result',
-                        'front_photo_on_screen_result': 'photo_on_screen_result',
-                        'doc_on_pp': doc_on_pp_result,
-                        'screenshot_result': 'screenshot_result',
-                        'photo_on_screen_result': 'photo_on_screen_result',
-                        'front_blurred': 'front_blurred',
-                        'front_glare': 'front_glare',
-                        'back_blurred': 'front_blurred',
-                        'back_glare': 'front_glare',
-                        'front_face_locations': front_face_locations_str,
-                        'front_face_encodings': front_face_encodings_str,
-                        'valid_nationality': 'valid_nationality_result',
-                        'issuing_country': 'IRQ',
-                        'nationality_received': nationality
-                    }
-                else:
-                    data_temp={}
-
-                result.update(data_temp)
-                required_keys = ["front_face_locations", "front_face_encodings"]
-                empty_string_keys = [key for key, value in result.items() if key in required_keys and value == '']
-                empty_string_keys = [key for key, value in result.items() if key in required_keys and value == '[]']
-
-
-
-                if empty_string_keys:
-                    result['error'] = 'covered_photo'
-                    result['error_details'] = f"Missing fields: {empty_string_keys}"
-            if side=='page1':
-                result['passport_data'] = front_id_text_desc
-            if side=='back':
-                valid_nationality_result = self.check_nationality_in_iso_list(result.get('nationality'))
-                back_data_update = {
-                    # 'back_tampered_result': tampered_result_back,
-                    'valid_nationality': valid_nationality_result,
-                    'back_extracted_data': front_id_text_desc,
-                    'translated_back_id_text': '',
-                    'back_coloured': True,
-                    'occupation': '',
-                    'employer': '',
-                    'doc_on_pp': doc_on_pp_result,
-                    'screenshot_result': 'screenshot_result',
-                    'photo_on_screen_result': 'photo_on_screen_result',
-                    'back_blurred': 'front_blurred',
-                    'back_glare': 'front_glare',
-                    'back_tampered_result':'clear'
-                }
-
-                result.update(back_data_update)
-
-
-        except Exception as e:
-            result['error'] = 'bad_image'
-            result['error_details'] = e
-
-        logging.info(f'result from agent extraction: {result}')
-        return result
-
-
-
-    def extract_front_id_info(self, front_id, country, nationality=None):
-        if country == "UAE":
-            print("working on UAE")
-            from idvpackage.ocr_utils import (
-                detect_logo,
-                detect_photo_on_screen,
-                detect_screenshot,
-                document_on_printed_paper,
-            )
-            from idvpackage.common import (
-                detect_id_card_uae,
-                load_and_process_image_deepface,
-            )
-            # from idvpackage.uae_id_extraction import extract_uae_front_id
-            from idvpackage.uae_id_extraction import get_response_from_openai_uae
-
-            front_data = {"error": "", "doc_type": "national_identity_card"}
-
-            try:
-                # output = extract_uae_front_id(front_id)
-                processed_front_id = self.image_conversion(front_id)
-                output = get_response_from_openai_uae(processed_front_id, "front", country, self.openai_key)
-
-                logging.info(f"UAE Front ID extraction output: {output}")
-                # if isinstance(output, dict):
-                #     if output.get("error", "") == "covered_photo":
-                #         return {
-                #             "error": "covered_photo",
-                #             "error_details": "Issue in extracting id number from ID card",
-                #         }
-
-                # processed_front_id = self.image_conversion(front_id)
-                # front_id_text = self.get_ocr_results(processed_front_id)
-                # front_id_text_desc = front_id_text[0].description
-                # combined_pattern = r'(Resident Identity|Identity Card|Golden Card|FEDERAL AUTHORITY FOR IDENTITY)'
-                # match = re.search(combined_pattern, front_id_text_desc, re.IGNORECASE)
-
-                if not output.get("header_verified", False):
-                    front_data["error"] = "not_front_id"
-                    return front_data
-
-                # img = processed_front_id
-                # image = np.array(img)
-                # pil_image = Image.fromarray(image)
-                #
-                # doc_on_pp_result = document_on_printed_paper(image)
-
-                # with io.BytesIO() as output:
-                #     pil_image.save(output, format="PNG")
-                #     image_data = output.getvalue()
-
-                # logo_result = detect_logo(self.client, image_data, country)
-                logo_result = "clear"
-                #
-                # screenshot_result = detect_screenshot(self.client, front_id)
-                # photo_on_screen_result = detect_photo_on_screen(self.client, front_id)
-                #
-                # front_blurred, front_glare = self.get_blurred_and_glared_for_doc(image)
-                # print(f"blurred, glare: {front_blurred, front_glare}")
-
-                front_face_locations, front_face_encodings = (
-                    load_and_process_image_deepface(front_id)
-                )
-                # front_face_locations, front_face_encodings = self.load_and_process_image_fr(front_id)
 
                 front_face_locations_str = json.dumps(
                     [tuple(face_loc) for face_loc in front_face_locations]
@@ -1406,43 +716,13 @@ class IdentityVerification:
                     [face_enc.tolist() for face_enc in front_face_encodings]
                 )
 
-                # tampered_result, part_text = detect_id_card_uae(self.client, image_data, front_id_text)
-
-                # dob, expiry = '', ''
-                # date_matches = re.findall(r'\d{2}/\d{2}/\d{4}', front_id_text_desc)
-                # sorted_dates = sorted(date_matches)
-
-                # if len(sorted_dates) > 1:
-                #     dob = sorted_dates[0]
-                #     expiry = sorted_dates[-1]
-
-                # front_data = {
-                #     'front_extracted_data': front_id_text_desc,
-                #     'front_coloured': True,
-                #     'front_doc_on_pp': doc_on_pp_result,
-                #     'front_logo_result': logo_result,
-                #     'front_screenshot_result': screenshot_result,
-                #     'front_photo_on_screen_result': photo_on_screen_result,
-                #     'front_blurred': front_blurred,
-                #     'front_glare': front_glare,
-                #     'front_face_locations': front_face_locations_str,
-                #     'front_face_encodings': front_face_encodings_str,
-                #     'front_tampered_result': tampered_result
-                # }
-
-                # front_id_text_desc = (
-                #     str(output.id_number)
-                #     + " "
-                #     + "is header verified:"
-                #     + str(output.is_header_verified)
-                # )
-                # id_number_front = str(outputid_number).replace("-", "")
-                front_data = {
-                    "id_number_front": output.get("id_number",""),
-                    "front_extracted_data": "",
+
+                front_data_update = {
+                    "id_number_front": output.get("id_number", ""),
+                    "front_extracted_data": "front_extracted_data",
                     "front_coloured": True,
                     "front_doc_on_pp": "clear",
-                    "front_logo_result": logo_result,
+                    "front_logo_result": 'clear',
                     "front_screenshot_result": "clear",
                     "front_photo_on_screen_result": "clear",
                     "front_blurred": "clear",
@@ -1452,6 +732,8 @@ class IdentityVerification:
                     "front_tampered_result": "clear",
                 }
 
+                front_data.update(front_data_update)
+
                 non_optional_keys = [
                     "front_face_locations",
                     "front_face_encodings",
@@ -1464,275 +746,174 @@ class IdentityVerification:
                 ]
 
                 if empty_string_keys:
-                    front_data["error"] = "covered_photo"
+                    front_data["error"] = "missing_key_fields"
+                    logging.info(f"Empty string fields found in front_data: {empty_string_keys}")
+                    return front_data
 
+                
             except Exception as e:
                 front_data["error"] = "bad_image"
+                logging.info(f"Exception in UAE front ID extraction: {e}")
                 front_data["error_details"] = e
 
             return front_data
 
         if country == "IRQ":
             logging.info("-------------Working on IRQ \n")
+            from idvpackage.common import load_and_process_image_deepface
+            from idvpackage.iraq_id_extraction_withopenai import (
+                get_response_from_openai_irq,
+            )
             from idvpackage.ocr_utils import (
                 detect_photo_on_screen,
                 detect_screenshot,
                 document_on_printed_paper,
             )
-            from idvpackage.iraq_id_extraction import (
-                iraq_front_id_extraction,
-                extract_mother_surname,
-                extract_mother_name,
-                extract_paternal_grandfather_name,
-            )
-            from idvpackage.common import load_and_process_image_deepface
-            from deep_translator import GoogleTranslator
 
-            front_data = {"error": "", "doc_type": "national_identity_card"}
+            result = {"error": "", "doc_type": "national_identity_card"}
 
             try:
                 st = time.time()
-                processed_front_id = self.image_conversion(front_id)
-                logging.info(
-                    f"----------------Time taken for image conversion: {time.time() - st} seconds\n"
-                )
-
+                processed_front_id, compressed_image_data = self.image_conversion_and_compression(front_id)
+                logging.info(f'----------------Time taken for image conversion: {time.time() - st} seconds\n')
+                
+                
+                logging.info(f"starting the extraction using openai for IRQ front_id")
                 st = time.time()
-                compressed_image = BytesIO()
-                processed_front_id.save(
-                    compressed_image, format="JPEG", quality=75, optimize=True
-                )
-                compressed_image_data = compressed_image.getvalue()
-
-                front_id_text = self.get_ocr_results(
-                    compressed_image_data, country="IRQ", side="front"
-                )
-                front_id_text_desc = front_id_text[0].description
-                logging.info(
-                    f"----------------Time taken for vision: {time.time() - st} seconds\n"
-                )
-
-                try:
-                    translated_id_text = self.translator.translate(
-                        front_id_text_desc, src="ar", dest="en"
-                    ).text
-                except Exception as e:
-                    logging.info(
-                        f"--------------Fallback for translation keyword matching\n"
-                    )
-                    translated_id_text = GoogleTranslator("ar", "en").translate(
-                        front_id_text_desc
-                    )
 
-                # logging.info(f'\n----------------Time taken for translation: {time.time() - st} seconds')
+                front_data = get_response_from_openai_irq(compressed_image_data, openai_key=self.openai_key,side="front")
+                
 
-                combined_pattern = r"(Ministry of Interior|Republic of Iraq|National Card|Passports and Residence|Republic|Ministry|Iraq)"
-                match = re.search(combined_pattern, translated_id_text, re.IGNORECASE)
+                logging.info(f'----------------Time taken for OpenAI and  final extraction front_id: {time.time() - st} seconds\n')
 
-                if not match:
-                    front_data["error"] = "not_front_id"
-                    return front_data
+                logging.info(f"front_data: {json.dumps(front_data, ensure_ascii=False, indent=2)}")
+                
 
-                st = time.time()
-                image = np.array(processed_front_id)
-                # pil_image = Image.fromarray(image)
+                if not front_data.get('header_verified', False):
+                    result["error"] = "not_front_id"
+                    return result
 
-                ## TODO: uncomment this later with more sophisticated approach for doc_on_pp
-                # doc_on_pp_result = document_on_printed_paper(image)
-                doc_on_pp_result = "clear"
+                if not front_data.get("id_number", '').isdigit() and len(front_data.get("id_number", '')) == 12:
+                    result["error"] = "invalid_national_number"
+                    logging.info(f" invalid_national_number number found in front_data: {front_data.get('id_number')}")
+                    return result
 
-                # with io.BytesIO() as output:
-                #     pil_image.save(output, format="PNG")
-                #     image_data = output.getvalue()
+                if len(front_data.get("card_number", '')) != 9:
+                    result["error"] = "invalid_document_number"
+                    logging.info(f" invalid_document_number number found in front_data: {front_data.get('card_number')}")
+                    return result
 
-                template_result = "clear"
-                logo_result = "clear"
+                if front_data.get("gender", '').lower() not in ['male', 'female']:
+                    logging.info(f" invalid_gender found in front_data: {front_data.get('gender')}")
+                    front_data["gender"] = ""
 
-                screenshot_result = detect_screenshot(self.client, front_id)
-                photo_on_screen_result = detect_photo_on_screen(self.client, front_id)
+                image= np.array(processed_front_id)
 
-                front_blurred, front_glare = self.get_blurred_and_glared_for_doc(image)
-                logging.info(
-                    f"----------------Time taken for fraud detection attributes: {time.time() - st} seconds\n"
-                )
+                if not front_data['last_name'] or not front_data['last_name_en']:
 
-                st = time.time()
-                front_face_locations, front_face_encodings = (
-                    load_and_process_image_deepface(front_id)
-                )
-                logging.info(
-                    f"----------------Time taken for face extraction: {time.time() - st} seconds\n"
-                )
-                # front_face_locations, front_face_encodings = self.load_and_process_image_fr(front_id)
+                    front_data['name'] = front_data.get('first_name', '') + " " + front_data.get('father_name', '')
+                    front_data['name_en']= front_data.get('first_name_en', '') + " " + front_data.get('father_name_en', '')
 
-                front_face_locations_str = json.dumps(
-                    [tuple(face_loc) for face_loc in front_face_locations]
-                )
-                front_face_encodings_str = json.dumps(
-                    [face_enc.tolist() for face_enc in front_face_encodings]
-                )
+                else:
+                    front_data['name'] = front_data.get('first_name', '') + " " + front_data.get('father_name','') + " " + front_data.get('last_name', '')
+                    front_data['name_en'] = front_data.get('first_name_en', '') + " " + front_data.get('father_name_en', '') + " " + front_data.get("last_name_en", '')
 
-                # image_format = 'jpg'
                 st = time.time()
-                image_format = self.detect_image_format(front_id)
-
-                front_data_fields = iraq_front_id_extraction(
-                    self.client,
-                    compressed_image_data,
-                    front_id_text,
-                    front_id_text_desc,
-                    translated_id_text,
-                    image_format,
-                )
-                logging.info(
-                    f"----------------Time taken for iraq data formatting and final extraction: {time.time() - st} seconds\n"
-                )
-
-                front_data_temp = {
-                    "front_extracted_data": front_id_text_desc,
-                    "translated_front_id_text": translated_id_text,
-                    "front_coloured": True,
-                    "front_doc_on_pp": doc_on_pp_result,
-                    "front_logo_result": logo_result,
-                    "front_template_result": template_result,
-                    # 'front_tampered_result': tampered_result_front,
-                    "front_screenshot_result": screenshot_result,
-                    "front_photo_on_screen_result": photo_on_screen_result,
-                    "front_blurred": front_blurred,
-                    "front_glare": front_glare,
-                    "front_face_locations": front_face_locations_str,
-                    "front_face_encodings": front_face_encodings_str,
-                }
+                from idvpackage.common import load_and_process_image_deepface
+                front_face_locations, front_face_encodings = load_and_process_image_deepface(front_id)
+                logging.info(f'----------------Time taken for face extraction: {time.time() - st} seconds\n')
 
-                front_data.update(front_data_fields)
 
-                front_data.update(front_data_temp)
+                if front_face_encodings is None or len(front_face_encodings) == 0:
+                    result['error'] = 'face_not_detected'
+                    logging.info("No face detected in front image")
+                    return result
 
-                required_keys = [
-                    "front_face_locations",
-                    "front_face_encodings",
-                    "id_number",
-                    "name",
-                ]
-                empty_string_keys = [
-                    key
-                    for key, value in front_data.items()
-                    if key in required_keys and value == ""
-                ]
+                front_face_locations_str = json.dumps([tuple(face_loc) for face_loc in front_face_locations])
+                front_face_encodings_str = json.dumps([face_enc.tolist() for face_enc in front_face_encodings])
+                
+                # optional_fields = ('last_name', 'last_name_en','serial_number','blood_type','gender','gender_ar')
+                non_optional_keys = ['id_number_front',
+                                    'card_number_front',
+                                    'first_name',
+                                    'first_name_en',
+                                    'father_name',
+                                    'father_name_en',
+                                    'third_name',
+                                    'third_name_en',
+                                    'mother_first_name',
+                                    'mother_first_name_en',
+                                    'mother_last_name',
+                                    'mother_last_name_en',
+                                    'doc_type',
+                                    'nationality',
+                                    'nationality_en']
+                #check if any non optional key has empty string value
+                empty_string_keys = [ key for key, value in front_data.items() if key in non_optional_keys and (value == "" or value == [] or value == "[]")]
+                logging.info(f"Empty string keys in back_data: {empty_string_keys}")
 
                 if empty_string_keys:
-                    front_data["error"] = "covered_photo"
-
-            except Exception as e:
-                front_data["error"] = "bad_image"
-                front_data["error_details"] = e
-
-            try:
-                dict_mother_surname = extract_mother_surname(front_id_text_desc)
-                dict_mother_surname["mother_last_name_en"] = (
-                    GoogleTranslator("ar", "en")
-                    .translate(f"Name: {dict_mother_surname['mother_last_name']}")
-                    .upper()
-                )
-                # Check if "NAME: " is in the translated string, and remove it
-                if "NAME: " in dict_mother_surname["mother_last_name_en"]:
-                    dict_mother_surname["mother_last_name_en"] = dict_mother_surname[
-                        "mother_last_name_en"
-                    ].replace("NAME: ", "")
-
-            except Exception as e:
-                dict_mother_surname = {
-                    "mother_last_name": "",
-                    "mother_last_name_en": "",
-                }
-
-            front_data.update(dict_mother_surname)
-
-            try:
-                dict_third_name = extract_paternal_grandfather_name(front_id_text_desc)
-                dict_third_name["third_name_en"] = (
-                    GoogleTranslator("ar", "en")
-                    .translate(dict_third_name["third_name"])
-                    .upper()
-                )
-            except Exception as e:
-                dict_third_name = {"third_name_en": "", "third_name": ""}
+                    result['error'] = 'missing_key_fields'
+                    logging.info(f"Empty string fields found in front_data: {empty_string_keys}")
+                    return result
 
-            front_data.update(dict_third_name)
+                data_temp = {
+                        'front_extracted_data': '',
+                        'translated_front_id_text':'',
+                        'front_coloured': True,
+                        'back_coloured':True,
+                        'nationality': 'IRQ',
+                        'nationality_en': 'IRQ',
+                        "doc_on_pp_result": "clear",
+                        "template_result": "clear",
+                        "logo_result": "clear",
+                        'front_doc_on_pp': "clear",
+                        'front_logo_result': "clear",
+                        'front_template_result': "clear",
+                        'front_screenshot_result': "clear",
+                        'front_photo_on_screen_result': "clear",
+                        'front_blurred': "clear",
+                        'front_glare': "clear",
+                        'front_face_locations': front_face_locations_str,
+                        'front_face_encodings': front_face_encodings_str,
+                        'front_tampered_result': 'clear',
+                        'issuing_country':'IRQ',
+                        'valid_nationality': "clear"
+                    }
+                front_data.update(data_temp)
+                result.update(front_data)
+                return result
 
-            try:
-                dict_mother_name = extract_mother_name(front_id_text_desc)
-                dict_mother_name["mother_first_name_en"] = (
-                    GoogleTranslator("ar", "en")
-                    .translate(f"Name: {dict_mother_name['mother_first_name']}")
-                    .upper()
-                )
-                # Check if "NAME: " is in the string, and remove it
-                if "NAME: " in dict_mother_name["mother_first_name_en"]:
-                    dict_mother_name["mother_first_name_en"] = dict_mother_name[
-                        "mother_first_name_en"
-                    ].replace("NAME: ", "")
 
             except Exception as e:
-                dict_mother_name = {"mother_first_name": "", "mother_first_name_en": ""}
-
-            front_data.update(dict_mother_name)
-
-            if front_data.get("last_name", "") and front_data.get(
-                "mother_first_name", ""
-            ):
-                if front_data.get("last_name") == front_data.get("mother_last_name"):
-                    front_data["last_name"], front_data["last_name_en"] = "", ""
-
-            return front_data
+                result["error"] = "bad_image"
+                result["error_details"] = e
+                return result
 
         if country == "QAT":
-            print("working on QAT with compression")
-            # from idvpackage.qatar_id_extraction import qatar_front_id_extraction
-            from idvpackage.qatar_id_extraction import get_response_from_openai_qat
+            logging.info(f"Working on QAT NID ....")
             from idvpackage.common import load_and_process_image_deepface
             from idvpackage.ocr_utils import document_on_printed_paper
+            from idvpackage.qatar_id_extraction import get_response_from_openai_qat
 
             front_data = {"error": "", "doc_type": "national_identity_card"}
 
             try:
+                processed_front_id ,compressed_image_data= self.image_conversion_and_compression(front_id)
                 st = time.time()
-                processed_front_id = self.image_conversion(front_id)
-                logging.info(
-                    f"----------------Time taken for image conversion front: {time.time() - st} seconds\n"
-                )
-
-                st = time.time()
-                compressed_image = BytesIO()
-                processed_front_id.save(
-                    compressed_image, format="JPEG", quality=70, optimize=True
-                )
-                compressed_image_data = compressed_image.getvalue()
-
-                st = time.time()
-                # front_data_fields = qatar_front_id_extraction(
-                #     self.client,
-                #     compressed_image_data,
-                #     front_id_text,
-                #     front_id_text_desc,
-                #     openai.api_key,
-                # )
                 front_data_fields = get_response_from_openai_qat(
                     compressed_image_data, "front", country, self.openai_key
                 )
+                logging.info(f"----------------Time taken for vision front: {time.time() - st} seconds\n")
+                logging.info(f"front_data_fields: {json.dumps(front_data_fields, indent=2, ensure_ascii=False)}")
 
                 front_data_fields['issuing_country'] = 'QAT'
 
                 if not front_data_fields["header_verified"]:
                     front_data["error"] = "not_front_id"
                     return front_data
-
-
-                logging.info(f"front_data_fields: {json.dumps(front_data_fields, indent=2, ensure_ascii=False)}")
                 
-                logging.info(
-                    f"----------------Time taken for qatar data formatting and final extraction front: {time.time() - st} seconds\n"
-                )
+                logging.info(  f"----------------Time taken for qatar OpenAI and  final extraction front: {time.time() - st} seconds\n")
 
                 expiry_date = front_data_fields.get("expiry_date", "")
                 if expiry_date:
@@ -1744,53 +925,28 @@ class IdentityVerification:
                     except:
                         pass
 
-                # front_id_text = self.get_ocr_results(
-                #     compressed_image_data, country="QAT", side="front"
-                # )
-                # front_id_text_desc = front_id_text[0].description
-                # logging.info(
-                #     f"----------------Time taken for vision front: {time.time() - st} seconds\n"
-                # )
-                # combined_pattern = r"(State of Qatar|Residency Permit)"
-                # match = re.search(combined_pattern, front_id_text_desc, re.IGNORECASE)
-
-                # if not match:
-                #     front_data["error"] = "not_front_id"
-                #     return front_data
-
                 image = np.array(processed_front_id)
                 doc_on_pp_result = document_on_printed_paper(image)
 
                 st = time.time()
-                # qatar's original face encodings code
                 front_face_locations, front_face_encodings = (
                     load_and_process_image_deepface(front_id, country="QAT")
                 )
 
-                if front_face_locations == 0:
-                    return {
-                        "error": "face_not_found",
-                        "error_details": "Face not found at angle 0.",
-                    }
-
-                # front_face_locations, front_face_encodings = load_and_process_image_deepface_optimized(front_id)
-                logging.info(
-                    f"----------------Time taken for face extraction front: {time.time() - st} seconds\n"
-                )
-                # front_face_locations, front_face_encodings = self.load_and_process_image_fr(front_id)
+                
+                if front_face_encodings is None or len(front_face_encodings) == 0:
+                    front_data['error'] = 'face_not_detected'
+                    logging.info("No face detected in front image")
+                    return front_data
+                
+                logging.info(f"----------------Time taken for face extraction front: {time.time() - st} seconds\n")
 
                 front_face_locations_str = json.dumps(
                     [tuple(face_loc) for face_loc in front_face_locations]
                 )
                 front_face_encodings_str = json.dumps(
                     [face_enc.tolist() for face_enc in front_face_encodings]
-                )
-
-               
-
-                # print(f"TAMPERING: {tampered_result_front}")
-                # valid_nationality_result = self.check_nationality_in_iso_list(front_data_fields.get('nationality'))
-
+                )            
                 front_data_temp = {
                     "front_extracted_data": "",
                     "valid_nationality": "clear",
@@ -1798,7 +954,6 @@ class IdentityVerification:
                     "front_doc_on_pp": "clear",
                     "front_logo_result": "clear",
                     "front_template_result": "clear",
-                    # 'front_tampered_result': tampered_result_front,
                     "front_screenshot_result": "clear",
                     "front_photo_on_screen_result": "clear",
                     "front_blurred": "clear",
@@ -1809,9 +964,8 @@ class IdentityVerification:
 
                 front_data_fields.update(front_data_temp)
                 front_data.update(front_data_fields)
-
                 
-                required_keys = ["expiry", "name", "id_number"]
+                required_keys = ["expiry_date", "name", "id_number"]
                 empty_string_keys = [
                     key
                     for key, value in front_data.items()
@@ -1819,14 +973,14 @@ class IdentityVerification:
                 ]
 
                 if empty_string_keys:
-                    front_data["error"] = "covered_photo"
+                    front_data["error"] = "missing_key_fields"
 
                 if front_data.get("error"):
                     return front_data
 
 
             except Exception as e:
-                print(e)
+                logging.info(f"exception in QAT front ID extraction: {e}")
                 front_data["error"] = "bad_image"
                 front_data["error_details"] = e
 
@@ -1834,43 +988,28 @@ class IdentityVerification:
 
         if country == "LBN":
             logging.info("----------------Working on LBN\n")
-            from idvpackage.ocr_utils import (
-                detect_logo,
-                detect_photo_on_screen,
+            from idvpackage.blur_detection import is_image_blur
+            from idvpackage.common import load_and_process_image_deepface
+            from idvpackage.lebanon_id_extraction import lebanon_id_extraction_from_text
+            from idvpackage.ocr_utils import (                
                 detect_screenshot,
                 document_on_printed_paper,
             )
-            # from idvpackage.lebanon_id_extraction import lebanon_front_id_extraction
-            from idvpackage.lebanon_id_extraction import lebanon_id_extraction_from_text
-            from idvpackage.common import load_and_process_image_deepface
-            from idvpackage.blur_detection import is_image_blur
 
             front_data = {"error": "", "doc_type": "national_identity_card"}
 
             try:
                 st = time.time()
-                processed_front_id = self.image_conversion(front_id)
-                logging.info(
-                    f"----------------Time taken for image conversion front: {time.time() - st} seconds\n"
-                )
-
-                st = time.time()
-                compressed_image = BytesIO()
-                processed_front_id.save(
-                    compressed_image, format="JPEG", quality=100, optimize=True
-                )
-                compressed_image_data = compressed_image.getvalue()
+                processed_front_id ,compressed_image_data= self.image_conversion_and_compression(front_id)
+                logging.info(f"----------------Time taken for image conversion front: {time.time() - st} seconds\n")
 
+                st = time.time()    
                 front_id_text = self.get_ocr_results(
                     compressed_image_data, country="LBN"
                 )
                 front_id_text_desc = front_id_text[0].description
-
                 logging.info(f"Extracted LBN front OCR text: {front_id_text_desc}")
-
-                logging.info(
-                    f"----------------Time taken for vision front: {time.time() - st} seconds\n"
-                )
+                logging.info(f"----------------Time taken for vision front: {time.time() - st} seconds\n")
 
                 # Check for blur using the new comprehensive method
                 image = np.array(processed_front_id)
@@ -1881,36 +1020,35 @@ class IdentityVerification:
                     fft_threshold=120,
                     bright_reflection_min_area=1.0,
                 ):
-                    print(
+                    logging.info(
                         f"Blur/Brightness issue detected in front image, marking as covered photo"
                     )
                     front_data["error"] = "blur_photo"
                     return front_data
 
-                # Extract data and check for necessary keys
                 st = time.time()
                 front_data_fields = lebanon_id_extraction_from_text(front_id_text_desc, compressed_image_data, 'front', self.openai_key)
 
-                logging.info(
-                    f"----------------Time taken for data formatting and final extraction front: {time.time() - st} seconds\n"
-                )
-
+                logging.info(f"----------------Time taken for OpenAI and  final extraction front: {time.time() - st} seconds\n")
                 logging.info(f"Extracted LBN front data fields: {json.dumps(front_data_fields, indent=2, ensure_ascii=False)}")
 
-                if front_data_fields.get("header_verified", False) is False:
+                if front_data_fields.get('header_verified') is not True:
                     front_data["error"] = "not_front_id"
-                    return front_data
-                
+                    return front_data   
                 if 'id_number' in front_data_fields:
                     id_number = front_data_fields['id_number']
                     if id_number and len(id_number) < 12:
                         front_data_fields['id_number'] = id_number.zfill(12)
-
+                        
+                logging.info(f"ID Number after padding: {front_data_fields.get('id_number', '')}")
+                
                 if 'id_number_ar' in front_data_fields:
                     id_number_ar = front_data_fields['id_number_ar']
                     if id_number_ar and len(id_number_ar) < 12:
                         front_data_fields['id_number_ar'] = id_number_ar.rjust(12, '٠')
-
+                
+                logging.info(f"ID Number AR after padding: {front_data_fields.get('id_number_ar', '')}")
+                
                 front_data_fields['issuing_country'] = 'LBN'
                 front_data_fields['nationality'] = 'LBN'
                 required_keys = [
@@ -1919,7 +1057,6 @@ class IdentityVerification:
                     "dob_ar",
                     "place_of_birth_ar",
                     "id_number",
-                    "expiry_date",
                 ]
 
                 dob = front_data_fields.get('dob', '')
@@ -1944,7 +1081,7 @@ class IdentityVerification:
                 ]
 
                 if empty_string_keys:
-                    front_data["error"] = "covered_photo"
+                    front_data["error"] = "missing_key_fields"
                     logging.info(
                         f"Missing required keys in LBN front data: {empty_string_keys}"
                     )
@@ -1953,25 +1090,16 @@ class IdentityVerification:
                 image = np.array(processed_front_id)
 
                 st = time.time()
-                ## TODO: doc_on_pp and detect_photo_on_screen for LBN
-                # doc_on_pp_result = document_on_printed_paper(image)
                 doc_on_pp_result = "clear"
-                ## no logo for Lebanon ID's
                 logo_result = "clear"
                 template_result = "clear"
-                ## TODO: template matching for Lebanon ID's
-                # template_result = detect_logo(self.client, compressed_image_data, country, compare_type='template', side='front')
-                ## TODO: tampering result for Lebanon ID's - pending tampered samples
-                # tampered_result_front = calculate_error_difference(np.array(Image.open(io.BytesIO(base64.decodebytes(bytes(front_id, "utf-8"))))))
                 screenshot_result = detect_screenshot(self.client, front_id)
-                # photo_on_screen_result = detect_photo_on_screen(self.client, image)
                 photo_on_screen_result = "clear"
                 front_blurred, front_glare = self.get_blurred_and_glared_for_doc(image)
                 logging.info(
                     f"----------------Time taken for fraud detection attributes front: {time.time() - st} seconds\n"
                 )
 
-                # front_face_locations, front_face_encodings = self.load_and_process_image_fr(front_id)
                 st = time.time()
                 front_face_locations, front_face_encodings = (
                     load_and_process_image_deepface(front_id)
@@ -1980,6 +1108,11 @@ class IdentityVerification:
                     f"----------------Time taken for face extraction front: {time.time() - st} seconds\n"
                 )
 
+                if front_face_encodings is None or len(front_face_encodings) == 0:
+                    front_data['error'] = 'face_not_detected'
+                    logging.info("No face detected in front image")
+                    return front_data
+
                 front_face_locations_str = json.dumps(
                     [tuple(face_loc) for face_loc in front_face_locations]
                 )
@@ -1987,7 +1120,6 @@ class IdentityVerification:
                     [face_enc.tolist() for face_enc in front_face_encodings]
                 )
 
-                # print(f"TAMPERING: {tampered_result_front}")
 
                 front_data_temp = {
                     "front_extracted_data": "",
@@ -1996,7 +1128,6 @@ class IdentityVerification:
                     "front_doc_on_pp": doc_on_pp_result,
                     "front_logo_result": logo_result,
                     "front_template_result": template_result,
-                    # 'front_tampered_result': tampered_result_front,
                     "front_screenshot_result": screenshot_result,
                     "front_photo_on_screen_result": photo_on_screen_result,
                     "front_blurred": front_blurred,
@@ -2016,7 +1147,6 @@ class IdentityVerification:
                     "dob_ar",
                     "place_of_birth_ar",
                     "id_number",
-                    "expiry_date",
                 ]
 
                 empty_string_keys = [
@@ -2027,7 +1157,7 @@ class IdentityVerification:
                 ]
 
                 if empty_string_keys:
-                    front_data["error"] = "covered_photo"
+                    front_data["error"] = "missing_key_fields"
                     logging.info(
                         f"Missing required keys in LBN front data after update: {empty_string_keys}"
                     )
@@ -2040,43 +1170,33 @@ class IdentityVerification:
 
         if country == "SDN":
             logging.info("----------------Working on SDN\n")
+            from idvpackage.blur_detection import is_image_blur
+            from idvpackage.common import load_and_process_image_deepface
             from idvpackage.ocr_utils import (
-                detect_logo,
-                detect_photo_on_screen,
                 detect_screenshot,
                 document_on_printed_paper,
             )
-            from idvpackage.sudan_passport_extraction import get_response_from_openai_sdn
-            from idvpackage.common import load_and_process_image_deepface 
-            from idvpackage.blur_detection import is_image_blur
+            from idvpackage.sudan_passport_extraction import (
+                get_response_from_openai_sdn,
+            )
 
             front_data = {"error": "", "doc_type": "national_identity_card"}
 
             try:
                 st = time.time()
-                processed_front_id = self.image_conversion(front_id)
-                logging.info(
-                    f"----------------Time taken for image conversion front: {time.time() - st} seconds\n"
-                )
-
+                processed_front_id,compressed_image_data = self.image_conversion_and_compression(front_id)
+                logging.info(f"----------------Time taken for image conversion front: {time.time() - st} seconds\n")
+                
                 st = time.time()
-                compressed_image = BytesIO()
-                processed_front_id.save(
-                    compressed_image, format="JPEG", quality=90, optimize=True
-                )
-                compressed_image_data = compressed_image.getvalue()
                 image = np.array(processed_front_id)
-
                 doc_on_pp_result = "clear"
                 logo_result = "clear"
                 template_result = "clear"
                
-                front_data_fields = get_response_from_openai_sdn(compressed_image, 'front', self.openai_key)
+                front_data_fields = get_response_from_openai_sdn(compressed_image_data, 'front', self.openai_key)
 
                 logging.info(f"Extracted SDN front data fields: {json.dumps(front_data_fields, indent=2, ensure_ascii=False)}")
-                logging.info(
-                    f"----------------Time taken for data formatting and final extraction front: {time.time() - st} seconds\n"
-                )
+                logging.info(f"----------------Time taken for OpenAI and  final extraction front: {time.time() - st} seconds\n")
 
                 if not front_data_fields['header_verified']:
                     front_data_fields['error'] = 'not_front_id'
@@ -2086,7 +1206,6 @@ class IdentityVerification:
                 dob = front_data_fields.get('dob', '')
                 
                 if dob:
-
                     try:
                         from idvpackage.ocr_utils import is_age_18_above
                         is_legal_age = is_age_18_above(dob)
@@ -2097,11 +1216,6 @@ class IdentityVerification:
                     except Exception as e:
                         logging.error(f"Error in age calculation: {e}")
                         
-                else:
-                    front_data['error'] = 'covered_photo'
-                    logging.error(f"DOB date not found in the extracted data.")
-                    return front_data
-                
 
                 front_data_fields['occupation'] = front_data_fields.get('occupation_en', '')
                
@@ -2118,6 +1232,12 @@ class IdentityVerification:
                 front_face_locations, front_face_encodings = (
                     load_and_process_image_deepface(front_id, country="SDN")
                 )
+                
+                if front_face_encodings is None or len(front_face_encodings) == 0:
+                    front_data['error'] = 'face_not_detected'
+                    logging.info("No face detected in front image")
+                    return front_data
+                
                 logging.info(
                     f"----------------Time taken for face extraction front: {time.time() - st} seconds\n"
                 )
@@ -2129,7 +1249,6 @@ class IdentityVerification:
                     [face_enc.tolist() for face_enc in front_face_encodings]
                 )
 
-
                 front_data_temp = {
                     "front_extracted_data": "",
                     "translated_front_id_text": "",
@@ -2166,9 +1285,10 @@ class IdentityVerification:
 
             except Exception as e:
                 front_data["error"] = "bad_image"
-                print(f"-------------->> Something went wrong error trace:: {e}")
+                logging.info(f"-------------->> Something went wrong error trace:: {e}")
                 front_data["error_details"] = e
 
+            
 
             try:
                 list_1 = front_data["name_ar"].split(" ")
@@ -2227,217 +1347,101 @@ class IdentityVerification:
 
             return front_data
 
-    def extract_back_id_info(self, back_id, country, nationality=None, step_data=None):
-        if country == "UAE":
-            from idvpackage.ocr_utils import (
-                detect_photo_on_screen,
-                detect_screenshot,
-                document_on_printed_paper,
-            )
-            from idvpackage.common import (
-                remove_special_characters1,
-                remove_special_characters2,
-                func_id_number,
-                convert_date_format,
-                detect_id_card_uae,
-                convert_gender,
-                count_digits_after_pattern,
-                remove_special_characters_mrz2,
-                validate_string,
-            )
-            from idvpackage.ocr_utils import is_valid_and_not_expired
-            import pytesseract
-            import sys
-
-            back_data = {"error": "", "doc_type": "national_identity_card"}
+    def extract_back_id_info(self, back_id, country, nationality, step_data):
+        
+        step_data = step_data if step_data is not None else {}
 
+        if country == 'UAE':
             try:
-                processed_back_id = self.image_conversion(back_id)
-                id_infos = self.get_ocr_results(processed_back_id)
-                text = id_infos[0].description
-                pattern4 = r"(Card Number|<<|ILARE|IDARE|(?=.*\bOccupation\b).*|(?=.*\bEmployer\b).*|(?=.*\bIssuing Place\b).*)"
-                k = re.search(pattern4, text.replace(" ", ""), re.IGNORECASE)
-
-                if not k:
-                    back_data["error"] = "not_back_id"
+                from idvpackage.ocr_utils import (
+                    detect_photo_on_screen,
+                    detect_screenshot,
+                    document_on_printed_paper,
+                    
+                )
+                from idvpackage.uae_id_extraction import get_response_from_openai_uae
+                back_data = {"error": "", "doc_type": "national_identity_card"}
+                back_data['country'] = "UAE"
+                
+                st = time.time()
+                output = get_response_from_openai_uae(back_id, 'back', 'UAE', self.openai_key)
+                logging.info(f"Time Taken to get back id raw output from openai: {time.time() - st}")
+                
+                logging.info(f"back_data output from openai: {json.dumps(output, indent=2, ensure_ascii=False)}")
+                
+                if not output.get('back_header_verified', False):
+                    back_data['error'] = 'not_back_id'
                     return back_data
 
-                original_text = text
-
-                # print('this is original text:',original_text)
-
-                patterns = {
-                    "id_number": (
-                        r"(?:ILARE|IDARE)\s*([\d\s]+)",
-                        lambda match: match.group(0).replace(" ", "")[15:30]
-                        if match
-                        else "",
-                    ),
-                    "card_number": (
-                        r"(?:ILARE|IDARE)(\d{1,9})",
-                        lambda match: match.group(1) if match else "",
-                    ),
-                    "nationality": (
-                        r"([A-Z]+)<<",
-                        lambda match: match.group(1) if match else "",
-                    ),
-                    "gender": (
-                        r"(?<=\d)[A-Z](?=\d)",
-                        lambda match: match.group(0) if match else "",
-                    ),
-                    "dob": (
-                        r"(\d+)[MF]",
-                        lambda match: self.convert_dob(match.group(1)) if match else "",
-                    ),
-                    "expiry_date": (
-                        r"[MF](\d+)",
-                        lambda match: self.convert_expiry_date(match.group(1))
-                        if match
-                        else "",
-                    ),
-                    "name": (
-                        r"(.*[A-Za-z]+<[<]+[A-Za-z].*)",
-                        lambda match: match.group(0).replace("<", " ").strip()
-                        if match
-                        else "",
-                    ),
-                    # 'first_name': (r'<<([^<]+)', lambda match: match.group(0).replace("<", "") if match else ''),
-                    # 'last_name': (r'([^<]+)(?=<<)', lambda match: match.group(0).replace("<", "") if match else ''),
-                    # 'occupation': (r'Occupation:\s*([-\w\s.]+)', lambda match: match.group(1).strip().split('\n', 1)[0] if match else '', re.IGNORECASE),
-                    # 'employer': (r'Employer:\s*([\w\s.]+)', lambda match: match.group(1).strip().split('\n', 1)[0] if match else '', re.IGNORECASE),
-                    "place_of_issuance": (
-                        r"Issuing Place:\s*([\w\s.]+)",
-                        lambda match: match.group(1).strip().split("\n", 1)[0]
-                        if match
-                        else "",
-                        re.IGNORECASE,
-                    ),
-                    "issuing_place": (
-                        r"Issuing Place:\s*([\w\s.]+)",
-                        lambda match: match.group(1).strip().split("\n", 1)[0]
-                        if match
-                        else "",
-                        re.IGNORECASE,
-                    ),
+                update_doe_dob_gender_from_back_data = {
+                    "dob": step_data.get('dob', '') if step_data.get('dob', '') != '' else output.get('dob', ''),
+                    "expiry_date": step_data.get('expiry_date', '') if step_data.get('expiry_date',
+                                                                                     '') != '' else output.get(
+                        'expiry_date', ''),
+                    "gender": step_data.get('gender', '') if step_data.get('gender', '') != '' else output.get('gender', '')
+                    }
+
+                back_date_update_from_stepdata = {
+                    "name": step_data.get('name', ''),
+                    "first_name": step_data.get('first_name', ''),
+                    "last_name": step_data.get('last_name', ''),
+                    "nationality": step_data.get('nationality', '')
                 }
 
-                mrz_pattern = r"(ILAR.*\n*.*\n*.*\n*.*|IDAR.*\n*.*\n*.*\n*.*)"
+                logging.info("Time Taken to get back id info from openai: {}".format(time.time() - st))
+
+                back_data.update(output)
+                back_data.update(update_doe_dob_gender_from_back_data)
+                back_data.update(back_date_update_from_stepdata)
+                
+                # hard-coded as per Ola and Maryem's request.
+                back_data["issuing_country"] = "ARE"
+                
+                from idvpackage.ocr_utils import (
+                    normalize_date_generic,
+                    normalize_mrz_date
+                )
 
+                dob = back_data.get('dob', '')
                 try:
-                    mrz = re.findall(
-                        mrz_pattern,
-                        original_text.replace(" ", "").strip(),
-                        re.MULTILINE,
-                    )
-                    mrz_list = mrz[0].replace(" ", "").split("\n", 3)
-                    mrz1 = mrz_list[0]
+                    if  dob:
+                        logging.info(f"Extracted DOB for age verification: {dob}")
+                        from idvpackage.ocr_utils import is_age_18_above
+                        is_legal_age = is_age_18_above(dob)
+                        logging.info(f"Is legal age (18+): {is_legal_age}")
+                        if not is_legal_age:
+                            back_data['error'] = 'under_age'
+                            return back_data
 
-                except:
-                    mrz1 = ""
+                except Exception as e:
+                    logging.info(f"Error in age calculation: {e}")
 
-                #### EXTRACT mrz2
 
-                # try:
-                #     mrz2=mrz_list[1]
-                # except:
-                #     mrz2=''
+                expiry_date = back_data.get('expiry_date', '')
                 try:
-                    mrz2 = [
-                        s
-                        for s in [
-                            remove_special_characters1(ele).replace(" ", "")
-                            for ele in original_text.split("\n")
-                        ]
-                        if len(re.findall(r"<", s)) >= 2
-                        and not (re.fullmatch(r"[A-Za-z<]+", s))
-                    ][0]
+                    if expiry_date:
+                        logging.info(f"Extracted Expiry Date for expiry verification: {expiry_date}")
+                        from idvpackage.ocr_utils import is_expired_id
+                        if is_expired_id(expiry_date):
+                            back_data['error'] = 'expired_id'
+                            logging.info(f"ID is expired with expiry date: {expiry_date}")
+                            return back_data
+                except Exception as e:
+                    logging.info(f"Error in expiry date calculation: {e}")
+                    
+                back_data['mrz'] = [
+                    back_data.get('mrz1', '') + back_data.get('mrz2', '') + back_data.get('mrz3', '')]
 
-                except:
-                    mrz2 = ""
-                ### Extract mrz3
-                try:
-                    mrz3 = [
-                        s
-                        for s in [
-                            remove_special_characters1(ele).replace(" ", "")
-                            for ele in original_text.split("\n")
-                        ]
-                        if len(re.findall(r"<", s)) >= 2
-                        and re.fullmatch(r"[A-Za-z<]+", s)
-                    ][0]
-                    back_data["name"] = remove_special_characters2(mrz3[0]).strip()
-                    back_data["last_name"] = (
-                        remove_special_characters2(
-                            re.search(r"([^<]+)(?=<<)", mrz3).group(0)
-                        ).strip()
-                        if re.search(r"([^<]+)(?=<<)", mrz3)
-                        else ""
-                    )
-                    back_data["first_name"] = (
-                        remove_special_characters2(
-                            re.search(r"<<([^<]+)", mrz3).group(0)
-                        ).strip()
-                        if re.search(r"<<([^<]+)", mrz3)
-                        else ""
-                    )
+                mrz1 = back_data.get('mrz1', '')
+                
+                mrz1 = mrz1.strip()
+                back_data['id_number_mrz'] = mrz1[-15:]
+                back_data['card_number_mrz'] = mrz1[5:14]
 
-                except:
-                    (
-                        mrz3,
-                        back_data["name"],
-                        back_data["last_name"],
-                        back_data["first_name"],
-                    ) = "", "", "", ""
-
-                pattern = r"ARE\d{25}"
-
-                extracted_data_tesseract = ""
-
-                if not re.search(pattern, original_text.replace(" ", "")):
-                    img = self.image_conversion(back_id)
-                    # Decode the base64 string
-                    image_data = base64.b64decode(back_id)
-                    # Convert to an image
-                    with Image.open(io.BytesIO(image_data)) as image:
-                        # Use PyTesseract to do OCR on the image
-                        try:
-                            extracted_data_tesseract = pytesseract.image_to_string(
-                                image
-                            )
-                            match = re.search(
-                                pattern, extracted_data_tesseract.replace(" ", "")
-                            )
-                            mrz1 = (mrz1[:2] + match[0]).strip()
-                        except:
-                            pass
-
-                mrz1_keys = ["id_number", "card_number"]
-                mrz2_keys = ["nationality", "gender", "dob", "expiry_date"]
-                # mrz3_keys = [ 'first_name', 'last_name']
-
-                for key, value in patterns.items():
-                    pattern = value[0]
-                    transform_func = value[1]
-                    flags = value[2] if len(value) > 2 else 0
-
-                    text = original_text
-                    if key in mrz1_keys:
-                        text = mrz1
-                    if key in mrz2_keys:
-                        text = mrz2
-                    # if key in mrz3_keys:
-                    #     text = mrz3
-
-                    match = re.search(pattern, text, flags)
-                    back_data[key] = transform_func(match) if match else ""
-
-                back_data.update({"mrz1": mrz1, "mrz2": mrz2, "mrz3": mrz3})
-
-                # print("ths is gender :",back_data['gender'])
 
                 try:
-                    front_id_number = step_data.get("id_number_front", "")
-                    back_id_number = back_data.get("id_number", "")
+                    front_id_number = step_data.get("id_number", "")
+                    back_id_number = back_data.get("id_number_mrz", "")
 
                     logging.info(
                         f"Front ID number: {front_id_number}, Back ID number: {back_id_number}"
@@ -2458,263 +1462,83 @@ class IdentityVerification:
                         "error_details": f"Exception Thrown while comparing front id number with back id number: {e}",
                     }
 
-                ## extracting occupation and employer
-                occ_word = "Occupation"
-                occ = ""
-                emp_word = "Employer"
-                emp = ""
-                try:
-                    lines = original_text.split("\n")
-                    for line in lines:
-                        if occ_word in line:
-                            start_index = line.find(occ_word)
-                            end_index = start_index + len(occ_word)
-                            occ = line[end_index:]
-                            occ = self.clean_string(occ)
-
-                        if emp_word in line:
-                            start_index1 = line.find(emp_word)
-                            end_index1 = start_index1 + len(emp_word)
-                            emp = line[end_index1:]
-                            emp = self.clean_string(emp)
-                except:
-                    occ = ""
-                    emp = ""
-
-                family_sponsor_word = "Family Sponsor"
-                family_sponsor = ""
-                try:
-                    lines = original_text.split("\n")
-                    for line in lines:
-                        if family_sponsor_word in line:
-                            start_index = line.find(family_sponsor_word)
-                            end_index = start_index + len(family_sponsor_word)
-                            family_sponsor = line[end_index:]
-                            family_sponsor = self.clean_string(family_sponsor)
-                except:
-                    family_sponsor = ""
-
-                ### new rule
-                if len(str(back_data["id_number"])) != 15:
-                    back_data["id_number"] = ""
-
-                ### new rule
-                if len(str(back_data["card_number"])) != 9:
-                    back_data["card_number"] = ""
-
-                current_module = sys.modules[__name__]
-
-                for key in ["dob", "expiry_date", "card_number", "name", "nationality"]:
-                    # if not back_data[key] and key not in ['occupation', 'employer', 'first_name', 'last_name', 'issuing_place', 'error']:
-
-                    if not back_data[key]:
-                        transform_func_new = getattr(current_module, f"func_{key}")
-                        back_data[key] = transform_func_new(original_text)
-
-                for key in ["dob", "expiry_date"]:
-                    if not back_data[key]:
-                        transform_func_new = getattr(current_module, f"find_{key}")
-                        back_data[key] = transform_func_new(
-                            original_text, back_data["mrz2"]
-                        )
-
-                if not back_data["id_number"]:
-                    back_data["id_number"] = func_id_number(
-                        original_text, back_data["dob"]
-                    )
-
-                if (
-                    is_valid_and_not_expired(back_data.get("expiry_date"), country)
-                    == "consider"
-                ):
-                    back_data["error"] = "expired_document"
-
-                ### convert the date format
-                if back_data["dob"]:
-                    try:
-                        back_data["dob"] = convert_date_format(back_data["dob"])
-                    except:
-                        back_data["dob"] = ""
-
-                if back_data["expiry_date"]:
-                    try:
-                        back_data["expiry_date"] = convert_date_format(
-                            back_data["expiry_date"]
-                        )
-                    except:
-                        back_data["expiry_date"] = ""
-
-                img = self.image_conversion(back_id)
-                if hasattr(img, "_getexif"):
-                    orientation = 0x0112
-                    exif = img._getexif()
-                    if exif is not None and orientation in exif:
-                        orientation = exif[orientation]
-                        rotations = {
-                            3: Image.ROTATE_180,
-                            6: Image.ROTATE_270,
-                            8: Image.ROTATE_90,
-                        }
-                        if orientation in rotations:
-                            img = img.transpose(rotations[orientation])
-
-                image = np.array(img)
-                pil_image = Image.fromarray(image)
-
-                with io.BytesIO() as output:
-                    pil_image.save(output, format="PNG")
-                    image_data = output.getvalue()
+                back_data['id_number_front_back_mrz_match'] = False
+                back_data["card_number_back_mrz_match"] = False
+                back_data['dob_mrz_match']= False
+                back_data['expiry_date_mrz_match']= False
+                back_data['gender_mrz_match']= False
+                back_data['name_mrz_match']= False
+        
+                back_data['place_of_issuance'] = output.get('issuing_place'," ")
+                
+                id_number_mrz_str = back_data.get('id_number_mrz', '')
+                id_number_str= step_data.get('id_number', '')
+                logging.info(f"ID Number from front: {id_number_str} ID Number from MRZ: {id_number_mrz_str}")
+                if id_number_mrz_str and id_number_str and id_number_mrz_str == id_number_str:
+                    back_data['id_number_front_back_mrz_match'] = True
+                
+                card_number_mrz_str = back_data.get('card_number_mrz', '')
+                card_number_str= back_data.get('card_number', '')
+                
+                logging.info(f"Card Number from back: {card_number_str} Card Number from MRZ: {card_number_mrz_str}")
+                if card_number_mrz_str and card_number_str and card_number_mrz_str == card_number_str:
+                    back_data['card_number_back_mrz_match'] = True
 
-                tampered_result, third_part_text = detect_id_card_uae(
-                    self.client, image_data, id_infos, part="third"
-                )
-                back_data["back_tampered_result"] = tampered_result
-
-                ### layer of gender extraction
-                if not back_data["gender"]:
-                    # print(f"TEXT: {third_part_text}")
-                    mrz2 = re.search(r"\b\d{7}.*?(?:<<\d|<<\n)", third_part_text)
-                    mrz2 = mrz2.group(0) if mrz2 else None
-
-                    gender_ptrn = r"\d{7}([A-Z])\d{4,}"
-                    if mrz2:
-                        gender_match = re.search(gender_ptrn, mrz2)
-                        gender = gender_match.group(1)
-                        back_data["gender"] = gender
-                    else:
-                        gender_match = re.search(gender_ptrn, third_part_text)
-                        gender = gender_match.group(0)
-                        back_data["gender"] = gender
-
-                ### another layer of gender extraction + formatting
-                if not back_data["gender"]:
-                    extract_no_space = original_text.replace(" ", "")
-                    try:
-                        pattern = r"\sM|F"
-                        m = re.search(pattern, original_text)
-                        back_data["gender"] = m.group(0)[-1]
-                    except:
-                        pattern = r"\d{3}(?:M|F)\d"
-                        m = re.findall(pattern, extract_no_space)
-                        if len(m) != 0:
-                            back_data["gender"] = m[0][3:4]
-                        else:
-                            back_data["gender"] = ""
-
-                ### if still no gender then one more layer of gender extraction + formatting
-                if not back_data["gender"]:
-                    if not extracted_data_tesseract:
-                        # Decode the base64 string
-                        image_data = base64.b64decode(back_id)
-                        # Convert to an image
-                        with Image.open(io.BytesIO(image_data)) as image:
-                            # Use PyTesseract to do OCR on the image
-                            extracted_data_tesseract = pytesseract.image_to_string(
-                                image
-                            )
-
-                    mrzs_tesseract = [
-                        s
-                        for s in [
-                            ele.replace(" ", "")
-                            for ele in extracted_data_tesseract.split("\n")
-                        ]
-                        if re.search(r"<<{2,}", s)
-                    ]
-                    mrz3_tesseract = [
-                        s for s in mrzs_tesseract if re.fullmatch(r"[A-Za-z<]+", s)
-                    ]
-
-                    if mrzs_tesseract and mrz3_tesseract:
-                        mrz2_tesseract = list(
-                            set(mrzs_tesseract) - set(mrz3_tesseract)
-                        )[0]
-                        gender = mrz2_tesseract[7].lower()
-                        if gender in ["f", "m"]:
-                            back_data["gender"] = convert_gender(gender)
-                else:
-                    back_data["gender"] = convert_gender(back_data["gender"])
+                dob_back_str = back_data.get("dob", "")
+                dob_back_mrz_str = back_data.get("dob_mrz", "")
+                dob_back = normalize_date_generic(dob_back_str)
+                dob_back_mrz = normalize_date_generic(dob_back_mrz_str)
 
-                if back_data["name"]:
-                    back_data["name"] = re.sub(
-                        "[^a-zA-Z]", " ", back_data["name"]
-                    ).strip()
+                logging.info(f"Normalized DOB from back: {dob_back}, from MRZ: {dob_back_mrz}")
 
-                ### new rule
-                if len(str(back_data["id_number"])) != 15:
-                    back_data["id_number"] = ""
+                if dob_back == dob_back_mrz:
+                    back_data['dob_mrz_match']= True
 
-                ### new rule
-                if len(str(back_data["card_number"])) != 9:
-                    back_data["card_number"] = ""
+                exp_back_str = back_data.get("expiry_date", "")
+                exp_back_mrz_str = back_data.get("expiry_date_mrz", "")
+                exp_back = normalize_date_generic(exp_back_str)
+                exp_back_mrz = normalize_date_generic(exp_back_mrz_str)
+                logging.info(f"Normalized Expiry Date from back: {exp_back}, from MRZ: {exp_back_mrz}")
+                if exp_back == exp_back_mrz_str:
+                    back_data['expiry_date_mrz_match']= True
 
-                count = count_digits_after_pattern(mrz2)
+                 
+                if back_data.get('gender_mrz','')=='M':
+                    back_data['gender_mrz'] = 'Male'
+                elif back_data.get('gender_mrz','')=='F':
+                    back_data['gender_mrz'] = 'Female'
 
-                if count > 1:
-                    mrz2 = mrz2[: -int(count - 1)]
+                gender_mrz = back_data.get("gender_mrz", "")
+                gender_back = back_data.get('gender', '')
+                logging.info(f"Gender from card {gender_back} and gender from mrz: {gender_mrz}")
+                if gender_back == gender_mrz:
+                    logging.info(f"Gender from back: {gender_back}, gender from MRZ: {gender_mrz}")
+                    back_data['gender_mrz_match'] = True
 
-                ## fix a special case where O comes as zero
 
-                if re.sub(r"O([A-Z]{3})", r"0\1", mrz2):
-                    mrz2 = re.sub(r"O([A-Z]{3})", r"0\1", mrz2)
+                from idvpackage.ocr_utils import get_name_match_mrz
 
-                if not validate_string(remove_special_characters_mrz2(mrz2)):
-                    if (
-                        (back_data["mrz2"])
-                        and (back_data["gender"])
-                        and (back_data["mrz2"][-1].isdigit())
-                    ):
-                        try:
-                            # Regular expression to extract two sequences of 7 digits
-                            matches = re.findall(r"\d{7}", back_data["mrz2"])
-
-                            # Check if we found two sequences
-                            extracted_digits = (
-                                matches[:2] if len(matches) >= 2 else None
-                            )
-
-                            if extracted_digits:
-                                mrz2 = (
-                                    extracted_digits[0]
-                                    + back_data["gender"][:1]
-                                    + extracted_digits[-1]
-                                    + "<<<<<<<<<<<"
-                                    + back_data["mrz2"][-1]
-                                )
-                        except:
-                            mrz2 = ""
-                    else:
-                        mrz2 = ""
-
-                if len(back_data["nationality"]) > 3:
-                    back_data["nationality"] = back_data["nationality"][-3:]
-
-                ### check if teh extracted nationality is valid
-                valid_nationality_result = self.check_nationality_in_iso_list(
-                    back_data.get("nationality")
-                )
+                back_data["is_name_match_mrz"],back_data["name_mrz"] = get_name_match_mrz(back_data,'national_identity_card')
+                logging.info(f"Name from back: {back_data.get('name','')}, name from MRZ: {back_data.get('name_mrz','')}")
 
                 img = self.image_conversion(back_id)
                 image = np.array(img)
-                # pil_image = Image.fromarray(image)
 
                 doc_on_pp_result = document_on_printed_paper(image)
                 screenshot_result = detect_screenshot(self.client, back_id)
                 photo_on_screen_result = detect_photo_on_screen(self.client, back_id)
                 back_blurred, back_glare = self.get_blurred_and_glared_for_doc(image)
-                # print(f"blurred, glare: {back_blurred, back_glare}")
+
+                valid_nationality_result = self.check_nationality_in_iso_list(
+                    back_data.get("nationality")
+                )
 
                 back_data_update = {
+                    "back_extracted_data":"back_extracted_data",
                     "valid_nationality": valid_nationality_result,
-                    "back_extracted_data": original_text,
+                    "back_extracted_data": 'original_text',
                     "back_coloured": True,
-                    "mrz": mrz,
-                    "mrz1": mrz1,
-                    "mrz2": mrz2,
-                    "mrz3": mrz3,
-                    "occupation": occ,
-                    "employer": emp,
-                    "family_sponsor": family_sponsor,
+
                     "doc_on_pp": doc_on_pp_result,
                     "screenshot_result": screenshot_result,
                     "photo_on_screen_result": photo_on_screen_result,
@@ -2723,7 +1547,7 @@ class IdentityVerification:
                 }
 
                 back_data.update(back_data_update)
-                back_data["issuing_country"] = "ARE"
+
 
                 non_optional_keys = [
                     "id_number",
@@ -2745,194 +1569,240 @@ class IdentityVerification:
                 ]
 
                 if empty_string_keys:
-                    back_data["error"] = "covered_photo"
+                    back_data["error"] = "missing_key_fields"
+                    logging.info(f"Missing fields: {empty_string_keys}")
+
 
             except Exception as e:
-                back_data["error"] = "bad_image"
-                back_data["error_details"] = e
+                back_data = {"error": "bad_image"}
+                back_data["error_details"] = f"Exception Thrown: {e}"
+                return back_data
 
             return back_data
 
         if country == "IRQ":
-            back_data = {"error": "", "doc_type": "national_identity_card"}
+            result = {"error": "", "doc_type": "national_identity_card"}
+            from idvpackage.iraq_id_extraction_withopenai import (
+                get_response_from_openai_irq,
+            )
             from idvpackage.ocr_utils import (
                 detect_photo_on_screen,
                 detect_screenshot,
                 document_on_printed_paper,
             )
-            from idvpackage.iraq_id_extraction import (
-                iraq_back_id_extraction,
-                update_family_number_cases,
-                extract_family_number,
-            )
-
             try:
-                processed_back_id = self.image_conversion(back_id)
-
                 st = time.time()
-                compressed_image = BytesIO()
-                processed_back_id.save(
-                    compressed_image, format="JPEG", quality=75, optimize=True
-                )
-                compressed_image_data = compressed_image.getvalue()
-
-                id_infos = self.get_ocr_results(
-                    compressed_image_data, country="IRQ", side="front"
-                )
-                text = id_infos[0].description
-                logging.info(
-                    f"----------------Time taken for vision: {time.time() - st} seconds\n"
-                )
-
-                # print(f"\nORIGINAL TEXT: {text}\n")
-                # translated_id_text = self.translator.translate(text, from_lang='ar', to_lang='en').text
-                try:
-                    translated_id_text = self.translator.translate(
-                        text, src="ar", dest="en"
-                    ).text
-                except Exception as e:
-                    logging.info(
-                        f"--------------Fallback for translation keyword matching\n"
-                    )
-                    from deep_translator import GoogleTranslator
-
-                    translated_id_text = GoogleTranslator("ar", "en").translate(text)
-
-                # print(f"\nTRANS: {translated_id_text}\n")
-
-                pattern4 = r"(Register|Signature|IDIRQ|Family number|The Directorate of Nationality|IDIR)"
-                k = re.search(pattern4, translated_id_text, re.IGNORECASE)
-
-                if not k:
-                    k1 = re.search(pattern4, text, re.IGNORECASE)
-                    if not k1:
-                        back_data["error"] = "not_back_id"
-                        return back_data
-
-                original_text = text
-
-                # print('this is original text:',original_text)
-                image = np.array(processed_back_id)
-                # pil_image = Image.fromarray(image)
-
-                # with io.BytesIO() as output:
-                #     pil_image.save(output, format="PNG")
-                #     image_data = output.getvalue()
-
-                # template_result='clear'
-                # tampered_result_back = calculate_error_difference(np.array(Image.open(io.BytesIO(base64.decodebytes(bytes(back_id, "utf-8"))))))
-
-                # image_format = 'jpg'
-                image_format = self.detect_image_format(back_id)
-
+                processed_back_id,compressed_image_data = self.image_conversion_and_compression(back_id)
+                logging.info(f'----------------Time taken for image conversion: {time.time() - st} seconds\n')
+                
+                logging.info(f"starting the extraction using openai for IRQ back id")
                 st = time.time()
-                back_extraction_result = iraq_back_id_extraction(
-                    self.client,
-                    compressed_image_data,
-                    id_infos,
-                    original_text,
-                    image_format,
-                )
-                logging.info(
-                    f"--------------Time taken for back id data formatting final extraction: {time.time() - st}\n"
-                )
+                back_data = get_response_from_openai_irq(compressed_image_data, openai_key=self.openai_key,side="back")
+                logging.info(f'----------------Time taken for OpenAI and  final extraction back id: {time.time() - st} seconds\n')
 
-                error = ""
-                expiry_date = back_extraction_result.get("expiry_date", "")
-
-                if expiry_date:
-                    try:
-                        dt_obj = datetime.strptime(expiry_date, "%d/%m/%Y")
-                        tomorrow = datetime.today() + timedelta(days=1)
-                        if dt_obj < tomorrow:
-                            back_data["error"] = "expired_id"
-                    except:
-                        pass
-
-                back_data.update(back_extraction_result)
-                valid_nationality_result = self.check_nationality_in_iso_list(
-                    back_data.get("nationality")
-                )
-                # doc_on_pp_result = document_on_printed_paper(image)
-                doc_on_pp_result = "clear"
-                screenshot_result = detect_screenshot(self.client, back_id)
-                photo_on_screen_result = detect_photo_on_screen(self.client, back_id)
-                back_blurred, back_glare = self.get_blurred_and_glared_for_doc(image)
-
-                back_data_update = {
-                    # 'back_tampered_result': tampered_result_back,
-                    "valid_nationality": valid_nationality_result,
-                    "back_extracted_data": original_text,
-                    "translated_back_id_text": translated_id_text,
-                    "back_coloured": True,
-                    "occupation": "",
-                    "employer": "",
-                    "doc_on_pp": doc_on_pp_result,
-                    "screenshot_result": screenshot_result,
-                    "photo_on_screen_result": photo_on_screen_result,
-                    "back_blurred": back_blurred,
-                    "back_glare": back_glare,
-                }
+                logging.info(f"back_data: {json.dumps(back_data, ensure_ascii=False, indent=2)}")
+                if not back_data.get('header_verified', False):
+                    back_data["error"] = "not_back_id"
+                    return back_data
+                
+                doc_on_pp_result="clear"
+                
+                from idvpackage.ocr_utils import (
+                    normalize_date_generic,
+                )
 
-                back_data.update(back_data_update)
+                if back_data.get('dob', None):
+                    from idvpackage.ocr_utils import is_age_18_above
+                    if not is_age_18_above(back_data.get('dob', None)):
+                        result['error'] = 'under_age'
+                        logging.info("Passport holder is under age")
+                        return result
 
-                non_optional_keys = [
-                    "id_number",
-                    "card_number",
-                    "dob",
-                    "expiry_date",
-                    "nationality",
-                ]
-                empty_string_keys = [
-                    key
-                    for key, value in back_data.items()
-                    if key in non_optional_keys and value == ""
-                ]
+                if back_data.get('expiry_date'):
+                    from idvpackage.ocr_utils import is_expired_id
+                    if is_expired_id(back_data.get('expiry_date', None)):
+                        result['error'] = 'expired_id'
+                        logging.info(f"ID is expired with expiry date: {back_data.get('expiry_date','')}")
+                        return result
 
-                if empty_string_keys:
-                    back_data["error"] = "covered_photo"
 
-            except Exception as e:
-                back_data["error"] = "bad_image"
-                back_data["error_details"] = e
+                try:
+                    expiry_date_obj = datetime.strptime(back_data.get("expiry_date", ""), "%d/%m/%Y")
+                except Exception as e:
+                    logging.info(f"Error in parsing expiry date for IRQ ID: {e}")
+                    expiry_date_obj = None
+                try:
+                    expiry_date_mrz_obj = datetime.strptime(back_data.get("expiry_date_mrz", ""), "%d/%m/%Y")
+                except Exception as e:
+                    logging.info(f"Error in parsing expiry date mrz for IRQ ID: {e}")
+                    expiry_date_mrz_obj = None
+                try:
+                    issue_date_obj = datetime.strptime(back_data.get("issue_date", ""), "%d/%m/%Y")
+                except Exception as e:
+                    logging.info(f"Error in parsing issue date for IRQ ID: {e}")
+                    issue_date_obj = None
+                    
+                if expiry_date_obj and issue_date_obj:
+                    try:
+                        difference_in_days_mrz_obj = (expiry_date_obj - issue_date_obj).days
+                        back_data["valid_id_duration_mrz"] = difference_in_days_mrz_obj in [3651, 3652, 3653]
 
-            try:
-                family_num_dict = extract_family_number(text)
-                family_num_dict = update_family_number_cases(family_num_dict, text)
+                    except Exception as e:
+                        result['error'] = 'expiry_issue_date_difference_error'
+                        logging.info(f"Error in calculating difference between expiry and issue date for IRQ ID: {e}")
+                        return result
+                        
+                dob_back_str = back_data.get("dob", "")
+                dob_back_mrz_str = back_data.get("dob_mrz", "")
+                dob_back = normalize_date_generic(dob_back_str)
+                dob_back_mrz = normalize_date_generic(dob_back_mrz_str)
+            
+                if dob_back == dob_back_mrz:
+                    logging.info(f"Normalized DOB from back: {dob_back}, from MRZ: {dob_back_mrz}")
+                    back_data['dob_back_and_mrz_match']= True
+                else:
+                    logging.info(f"Mismatch in DOB from back: {dob_back}, from MRZ: {dob_back_mrz}")
+                    back_data['dob_back_and_mrz_match']= False
 
-                back_data["family_number_en"] = family_num_dict["family_number"]
-                back_data["family_number"] = family_num_dict["family_number"]
-            except Exception as e:
-                back_data["family_number_en"] = None
-                back_data["family_number"] = None
+                back_data['exp_date_back_and_mrz_match']= False
+                back_data['gender_front_and_back_match']= False 
+                back_data['id_number_front_back_match'] = False
+                back_data['card_number_front_back_match'] = False
+                
+                card_number_mrz_str = back_data.get('card_number_mrz', '')
+                card_number_str= step_data.get('card_number', '')
+                if card_number_mrz_str and card_number_str and card_number_mrz_str == card_number_str:
+                    back_data['card_number_front_back_match'] = True
+                
+                mrz1= back_data.get('mrz1', '')
+                mrz1 = mrz1.strip()
+                back_data['id_number_mrz'] = mrz1[15:27]
+                id_number_mrz_str = back_data.get('id_number_mrz', '')
+                logging.info(f"ID Number from front: {step_data.get('id_number', '')} ID Number from MRZ: {id_number_mrz_str}")
+                
+                id_number_str= step_data.get('id_number', '')
+                if id_number_mrz_str and id_number_str and id_number_mrz_str == id_number_str:
+                    back_data['id_number_front_back_match'] = True
+              
+                exp_back_str = back_data.get("expiry_date", "")
+                exp_back_mrz_str = back_data.get("expiry_date_mrz", "")
+                exp_back = normalize_date_generic(exp_back_str)
+                exp_back_mrz = normalize_date_generic(exp_back_mrz_str)
+                logging.info(f"Normalized Expiry Date from back: {exp_back}, from MRZ: {exp_back_mrz}")
+
+                if exp_back == exp_back_mrz:
+                    logging.info(f"Normalized Expiry Date from back: {exp_back}, from MRZ: {exp_back_mrz}")
+                    back_data['exp_date_back_and_mrz_match']= True
+                
+        
+                if back_data.get('gender_mrz','')=='M':
+                    back_data['gender_mrz'] = 'Male'
+                elif back_data.get('gender_mrz','')=='F':
+                    back_data['gender_mrz'] = 'Female' 
+                    
+                gender_mrz = back_data.get("gender_mrz", "")    
+                gender_front = step_data.get('gender','') if step_data else ''
+                
+                if gender_front:
+                    if gender_front == gender_mrz:
+                        back_data['gender_front_and_back_match']= True  
+                else:
+                    back_data['gender'] = gender_mrz
 
-                print("error in family number:", e)
+                back_data['mrz'] = [back_data.get('mrz1', '') + back_data.get('mrz2', '') + back_data.get('mrz3', '')]
 
-            return back_data
+                if len(back_data['mrz']) == 0 or not back_data['mrz'][0].strip():
+                    result['error']= "missing_mrz"
+                    result['error_details'] = "Missing or empty MRZ fields"
+                    return result
+
+                nationality_to_check = "IRQ"
+                logging.info(f"Nationality to check for back id: {nationality_to_check}")
+                valid_nationality_result = self.check_nationality_in_iso_list(nationality_to_check)
+                logging.info(f"valid_nationality_result for passport: {valid_nationality_result}")
+                back_data['valid_nationality'] = valid_nationality_result
+                
+                result.update(back_data)
+
+                non_optional_keys =['issuing_authority',
+                                    'issuing_authority_en',
+                                    'issue_date',
+                                    'expiry_date',
+                                    'place_of_birth',
+                                    'place_of_birth_en',
+                                    'dob',
+                                    'mrz1',
+                                    'mrz2',
+                                    'mrz3',
+                                    'family_number',
+                                    'doc_type',
+                                    'family_number_en',
+                                    'nationality',
+                                    'issuing_country',
+                                    'card_number',
+                                    'card_number_back',
+                                    'id_number',
+                                    'mrz',
+                                    'valid_expiry_issue',
+                                    'age_check',
+                                    'dob_match_mrz_dob',
+                                    'is_doc_expired',
+                                    'gender_back']
+               
+                empty_string_keys = [
+                    key for key, value in result.items() if key in non_optional_keys and (value == "" or value == [] or value == "[]")
+                ]
+                logging.info(f"Empty string keys in back_data: {empty_string_keys}")
+                
+                if empty_string_keys:
+                    result['error'] = 'missing_key_fields'
+                    logging.info(f"Empty string fields found in back_data: {empty_string_keys}")
+                    return result
 
+                back_data_update = {                 
+                    'valid_nationality': valid_nationality_result,
+                    'back_extracted_data': '',
+                    'translated_back_id_text': '',
+                    'back_coloured': True,
+                    'occupation': '',
+                    'employer': '',
+                    'doc_on_pp': doc_on_pp_result,
+                    'screenshot_result': 'screenshot_result',
+                    'photo_on_screen_result': 'photo_on_screen_result',
+                    'back_blurred': 'front_blurred',
+                    'back_glare': 'front_glare',
+                    'back_tampered_result': 'clear'
+                }
+                logging.info(f"Back data update before final update: ")
+                result.update(back_data_update)
+                logging.info(f"Final back_data completed succesfully")
+                
+                return result
+            
+            except Exception as e:
+                result['error'] = 'bad_image'
+                result['error_details'] = str(e)
+                logging.info(f"Exception details: {e}")
+                return result
+ 
         if country == "QAT":
             from idvpackage.qatar_id_extraction import get_response_from_openai_qat
-
             back_data = {"error": "", "doc_type": "national_identity_card"}
-            # is_colored2 = self.is_colored(back_id)
-            # if is_colored2:
             try:
                 processed_back_id = self.image_conversion(back_id)
-
                 compressed_image = BytesIO()
                 processed_back_id.save(
                     compressed_image, format="JPEG", quality=80, optimize=True
                 )
                 compressed_image_data = compressed_image.getvalue()
-
+                
+                st = time.time()
                 back_extraction_result = get_response_from_openai_qat(compressed_image_data, "back", country, self.openai_key)
+                logging.info(f"----------------Time taken for OpenAI and  final extraction back: {time.time() - st} seconds\n")
+                logging.info(f"back_extraction_result: {json.dumps(back_extraction_result, ensure_ascii=False, indent=2)}")
+               
                 back_data.update(back_extraction_result)
-
-                # image = np.array(processed_back_id)
-
-
                 back_data_update = {
-                    # 'back_tampered_result': tampered_result_back,
                     "back_extracted_data": "",
                     "back_coloured": True,
                     "doc_on_pp": "clear",
@@ -2943,8 +1813,7 @@ class IdentityVerification:
                 }
 
                 back_data.update(back_data_update)
-
-           
+                
             except Exception as e:
                 back_data["error"] = "bad_image"
                 back_data["error_details"] = e
@@ -2952,37 +1821,30 @@ class IdentityVerification:
             return back_data
 
         if country == "LBN":
+            from idvpackage.blur_detection import is_image_blur
+            from idvpackage.lebanon_id_extraction import lebanon_id_extraction_from_text
             from idvpackage.ocr_utils import (
                 detect_photo_on_screen,
                 detect_screenshot,
                 document_on_printed_paper,
             )
-            from idvpackage.lebanon_id_extraction import lebanon_id_extraction_from_text
-            from idvpackage.blur_detection import is_image_blur
 
             back_data = {"error": "", "doc_type": "national_identity_card"}
-
             try:
                 st = time.time()
                 processed_back_id = self.image_conversion(back_id)
-                logging.info(
-                    f"----------------Time taken for image conversion back: {time.time() - st} seconds\n"
-                )
-
+                logging.info(f"----------------Time taken for image conversion back: {time.time() - st} seconds\n")
                 st = time.time()
                 compressed_image = BytesIO()
                 processed_back_id.save(
                     compressed_image, format="JPEG", quality=100, optimize=True
                 )
                 compressed_image_data = compressed_image.getvalue()
-
                 back_id_text = self.get_ocr_results(compressed_image_data, country="LBN")
                 back_id_text_desc = back_id_text[0].description
 
                 logging.info(f"Extracted LBN back OCR text: {back_id_text_desc}")
-                logging.info(
-                    f"----------------Time taken for vision back: {time.time() - st} seconds\n"
-                )
+                logging.info(f"----------------Time taken for vision back: {time.time() - st} seconds\n")
 
                 # Check for blur using the new comprehensive method
                 image = np.array(processed_back_id)
@@ -2993,13 +1855,10 @@ class IdentityVerification:
                     fft_threshold=120,
                     bright_reflection_min_area=1.0,
                 ):
-                    print(
-                        f"Blur/Brightness issue detected in front image, marking as covered photo"
-                    )
+                    logging.info(f"Blur/Brightness issue detected in front image, marking as covered photo")
                     back_data["error"] = "blur_photo"
                     return back_data
 
-             
                 st = time.time()
                 back_extraction_result = lebanon_id_extraction_from_text(back_id_text_desc, compressed_image_data, "back", self.openai_key)
 
@@ -3021,18 +1880,15 @@ class IdentityVerification:
                         return back_data
 
                 logging.info(
-                    f"----------------Time taken for data formatting and final extraction back: {time.time() - st} seconds\n"
+                    f"----------------Time taken for OpenAI and  final extraction back: {time.time() - st} seconds\n"
                 )
 
                 back_data.update(back_extraction_result)
                 image = np.array(processed_back_id)
 
                 st = time.time()
-                ## TODO: doc_on_pp and detect_photo_on_screen for LBN
-                # doc_on_pp_result = document_on_printed_paper(image)
                 doc_on_pp_result = "clear"
                 screenshot_result = detect_screenshot(self.client, back_id)
-                # photo_on_screen_result = detect_photo_on_screen(self.client, image)
                 photo_on_screen_result = "clear"
                 back_blurred, back_glare = self.get_blurred_and_glared_for_doc(image)
                 logging.info(
@@ -3040,9 +1896,7 @@ class IdentityVerification:
                 )
 
                 back_data_update = {
-                    # 'back_tampered_result': tampered_result_back,
                     "back_extracted_data": "",
-                    # "translated_back_id_text": "",
                     "back_coloured": True,
                     "doc_on_pp": doc_on_pp_result,
                     "screenshot_result": screenshot_result,
@@ -3052,15 +1906,7 @@ class IdentityVerification:
                 }
 
                 back_data.update(back_data_update)
-
-                if (
-                    not back_data.get("gender")
-                    or not back_data.get("issue_date")
-                    or not back_data.get("card_number")
-                ):
-                    back_data["error"] = "covered_photo"
-
-                non_optional_keys = ["gender", "issue_date"]
+                non_optional_keys = ["gender", "issue_date", "card_number"]
                 empty_string_keys = [
                     key
                     for key, value in back_data.items()
@@ -3068,7 +1914,7 @@ class IdentityVerification:
                 ]
 
                 if empty_string_keys:
-                    back_data["error"] = "covered_photo"
+                    back_data["error"] = "missing_key_fields"
                     logging.info(f"Empty non-optional keys: {empty_string_keys}")
 
             except Exception as e:
@@ -3079,19 +1925,18 @@ class IdentityVerification:
             return back_data
 
         if country == "SDN":
+            from idvpackage.blur_detection import is_image_blur
             from idvpackage.ocr_utils import (
                 detect_photo_on_screen,
                 detect_screenshot,
                 document_on_printed_paper,
             )
-            from idvpackage.blur_detection import is_image_blur
 
             back_data = {"error": "", "doc_type": "national_identity_card"}
-
-            logging.info(f"Starting Sudan ID back side processing with step_data:{step_data.keys() if step_data else 'None'}")
+            logging.info(f"Starting Sudan ID back side processing with step_data:{step_data}")
             try:
                 st = time.time()
-                processed_back_id = self.image_conversion(back_id)
+                processed_back_id ,compressed_image_data = self.image_conversion_and_compression(back_id)
                 logging.info(
                     f"----------------Time taken for image conversion back: {time.time() - st} seconds\n"
                 )
@@ -3102,19 +1947,18 @@ class IdentityVerification:
                 )
                 compressed_image_data = compressed_image.getvalue()
                 image = np.array(processed_back_id)
-                st = time.time()
+             
                 from idvpackage.sudan_passport_extraction import get_response_from_openai_sdn
+                st = time.time()                
                 back_extraction_result = get_response_from_openai_sdn(compressed_image_data, "back", self.openai_key)
-
+                logging.info(f"----------------Time taken for vision back: {time.time() - st} seconds\n")
                 logging.info(f"back_extraction_result: {json.dumps(back_extraction_result, ensure_ascii=False, indent=2)}")
 
                 if not back_extraction_result.get('idsdn_verified', False):
                     back_data["error"] = "not_back_id"
                     return back_data
-                logging.info(
-                    f"----------------Time taken for data formatting and final extraction back: {time.time() - st} seconds\n"
-                )
-
+                logging.info(f"----------------Time taken for OpenAI and  final extraction back: {time.time() - st} seconds\n")
+                
                 if back_extraction_result.get("expiry_date"):
                     try:
                         expiry_date = back_extraction_result["expiry_date"]
@@ -3129,7 +1973,6 @@ class IdentityVerification:
 
                 back_data.update(back_extraction_result)
                 back_data['issuing_country'] = 'SDN'
-
                 from idvpackage.ocr_utils import normalize_date_generic
 
                 dob_front_str = step_data.get("dob", "") if step_data else ""
@@ -3178,35 +2021,28 @@ class IdentityVerification:
                
                 if back_data.get('issue_date', '') and back_data.get("expiry_date_mrz",''):
                     try:
-
                         logging.info(f"difference_in_days issue_date_mrz_obj: {issue_date_obj}, expiry_date_mrz_obj: {expiry_date_mrz_obj} differece is : {(expiry_date_mrz_obj - issue_date_obj).days}")
                         difference_in_days_mrz_obj = (expiry_date_mrz_obj - issue_date_obj).days
                         back_data["valid_id_duration_mrz"] = difference_in_days_mrz_obj in [1825, 1826, 1827]
-
                     except:
                         logging.info("Error in calculating date difference between issue and expiry dates for SDN ID")
                         pass
             
-
                 if back_data.get("issue_date",'') and back_data.get("expiry_date",''):
                     try:
-                       
                         logging.info(f"difference_in_days issue_date_obj: {issue_date_obj}, expiry_date_obj: {expiry_date_obj} differece is : {(expiry_date_obj - issue_date_obj).days}")
                         difference_in_days_obj = (expiry_date_obj - issue_date_obj).days
                         back_data["valid_id_duration"] = difference_in_days_obj in [1825, 1826, 1827]
-
                     except:
                         logging.info("Error in calculating date difference between issue and expiry dates from MRZ for SDN ID")
                         pass
                 
                 logging.info(f"dob_front_str: {dob_front_str}, dob_back_str: {dob_back_str}, dob_back_mrz_str: {dob_back_mrz_str}")
-
                 if dob_front_str and dob_back_str:
                     try: 
                         logging.info(f"dob_front: {dob_front}, dob_back: {dob_back}")
                         if dob_front == dob_back:
                             back_data["is_dob_front_back_match"] = True
-                        
                     except:
                         logging.info("Error in comparing DOB between front and back for SDN ID")
                         pass
@@ -3216,18 +2052,15 @@ class IdentityVerification:
                         logging.info(f"dob_front: {dob_front}, dob_back_mrz: {dob_back_mrz}")
                         if dob_front == dob_back_mrz:
                             back_data["is_dob_front_back_mrz_match"] = True
-                        
                     except:
                         logging.info("Error in comparing DOB between front and back MRZ for SDN ID")
                         pass
                 
-
                 if back_data.get('gender') and back_data.get('gender_mrz'):
                     try:
                         logging.info(f"gender: {back_data['gender']}, gender_mrz: {back_data['gender_mrz']}")
                         if back_data['gender'] == back_data['gender_mrz']:
                             back_data["is_gender_mrz_match"] = True
-                      
                     except:
                         logging.info("Error in comparing gender between front and back MRZ for SDN ID")
                         pass
@@ -3239,9 +2072,7 @@ class IdentityVerification:
                 from idvpackage.ocr_utils import get_name_match_mrz
                 back_data['nationality'] = 'SDN'
                 back_data['is_name_match_mrz'], back_data['name_mrz'] = get_name_match_mrz(back_data, "nationality_identity_card")
-
-
-                # get in this format  "2022/08/01",
+                logging.info(f"name from back: {back_data.get('name','')}, name from mrz: {back_data.get('name_mrz','')}")
                 back_data['issuance_date'] = back_data.get('issue_date', '')
                 if back_data.get("issuance_date",""):
                     try:
@@ -3253,11 +2084,8 @@ class IdentityVerification:
                 image = np.array(processed_back_id)
 
                 st = time.time()
-                ## TODO: doc_on_pp and detect_photo_on_screen for LBN
-                # doc_on_pp_result = document_on_printed_paper(image)
                 doc_on_pp_result = "clear"
                 screenshot_result = detect_screenshot(self.client, back_id)
-                # photo_on_screen_result = detect_photo_on_screen(self.client, image)
                 photo_on_screen_result = "clear"
                 back_blurred, back_glare = self.get_blurred_and_glared_for_doc(image)
                 logging.info(
@@ -3265,7 +2093,6 @@ class IdentityVerification:
                 )
 
                 back_data_update = {
-                    # 'back_tampered_result': tampered_result_back,
                     "back_extracted_data": "",
                     "translated_back_id_text": "",
                     "back_coloured": True,
@@ -3277,8 +2104,6 @@ class IdentityVerification:
                 }
 
                 back_data.update(back_data_update)
-
-
                 mrz1 = back_data.get('mrz1', '')
                 mrz2 = back_data.get('mrz2', '')
                 mrz3 = back_data.get('mrz3', '')
@@ -3292,286 +2117,208 @@ class IdentityVerification:
                     for key, value in back_data.items()
                     if key in non_optional_keys and value == ""
                 ]
-
                 if empty_string_keys:
                     back_data["error"] = "missing_key_fields"
 
             except Exception as e:
                 back_data["error"] = "bad_image"
-                print(f"-------------->> Something went wrong error trace:: {e}")
+                logging.info(f"-------------->> Something went wrong error trace:: {e}")
                 back_data["error_details"] = e
 
             return back_data
 
-    def exract_passport_info(self, passport, country, nationality, step_data=None):
-        if (nationality and nationality == "RUS") or (
-            not nationality and country == "RUS"
-        ):
-            logging.info("-------------Working on RUS Passport \n")
-            processed_passport = self.image_conversion(passport)
-            passport_text = self.get_ocr_results(processed_passport)
-            passport_text = passport_text[0].description
-
-            passport_details = {}
-
-            patterns = {
-                "passport_given_name": (
-                    r"Имя Given names\n(.*?)/",
-                    lambda match: self.translator.translate(
-                        match.group(1), src="ru", dest="en"
-                    ).text
-                    if match
-                    else "",
-                ),
-                "passport_surname": (
-                    r"RUS(.*?)<<(.*?)<.*",
-                    lambda match: match.group(1) if match else "",
-                ),
-                "passport_number": (
-                    r"(\d{7})",
-                    lambda match: match.group(1) if match else "",
-                ),
-                "passport_date_of_birth": (
-                    r"(\d+)[MF]",
-                    lambda match: self.convert_dob(match.group(1)) if match else "",
-                ),
-                "passport_date_of_expiry": (
-                    r"[MF](\d+)",
-                    lambda match: self.convert_expiry_date(match.group(1))
-                    if match
-                    else "",
-                ),
-                "passport_gender": (
-                    r"(\d)([A-Za-z])(\d)",
-                    lambda match: match.group(2) if match else "",
-                ),
-            }
-
-            mrz1_pattern = r"([A-Z<]+)<<([A-Z<]+)<<([\dA-Z<]+)"
-            mrz2_pattern = r"(\d{10}[A-Z]{3}\d{7}[\dA-Z<]+)"
-
-            mrz1_matches = re.findall(mrz1_pattern, passport_text)
-            mrz2_matches = re.findall(mrz2_pattern, passport_text)
-
-            if mrz1_matches:
-                mrz1 = " ".join(mrz1_matches[0])
-            else:
-                mrz1 = ""
-
-            if mrz2_matches:
-                mrz2 = mrz2_matches[0]
-            else:
-                mrz2 = ""
-
-            mrz1_keys = ["passport_surname"]
-            mrz2_keys = [
-                "passport_date_of_birth",
-                "passport_date_of_expiry",
-                "passport_gender",
-            ]
-
-            for key, value in patterns.items():
-                pattern = value[0]
-                transform_func = value[1]
-
-                text = passport_text
-                if key in mrz1_keys:
-                    text = mrz1
-                if key in mrz2_keys:
-                    text = mrz2
-
-                match = re.search(pattern, text)
-                passport_details[key] = transform_func(match) if match else ""
+    def extract_passport_info(self, passport, country, nationality, step_data=None):
 
-            passport_details["doc_type"] = "passport"
-            passport_details["nationality_received"] = nationality
-
-            return passport_details
+        step_data = step_data if step_data is not None else {}
 
         if (nationality and nationality == "IRQ") or (
-            not nationality and country == "IRQ"
-        ):
+            not nationality and country == "IRQ"):
             logging.info("-------------Working on IRQ Passport \n")
-            from idvpackage.ocr_utils import detect_photo_on_screen, detect_screenshot, document_on_printed_paper
-            from idvpackage.iraq_passport_extraction import iraq_passport_extraction, extract_mother_name_and_surname
-            from idvpackage.common import load_and_process_image_deepface
 
-            passport_data = {
-                'error': '',
-                'doc_type': 'passport'
-            }
+            from idvpackage.iraq_id_extraction_withopenai import (
+            get_response_from_openai_irq,
+        )
+
+            result = {'error': '', "error_details": '', 'doc_type': 'passport'}
+            if nationality == '':
+                nationality=None
 
             try:
-                processed_passport = self.image_conversion(passport)
-                id_infos = self.get_ocr_results(processed_passport)
-                passport_text = id_infos[0].description
-                print(passport_text)
-                pattern4 = r'(Republic of Iraq|Iraq)'  # |Passport
-                k = re.search(pattern4, passport_text, re.IGNORECASE)
-
-                if not k:
+                st = time.time()
+                processed_passport, compressed_image_data = self.image_conversion_and_compression(passport)
+                logging.info(f'----------------Time taken for image conversion: {time.time() - st} seconds\n')
+                logging.info(f"starting the extraction using openai for IRQ passport")
+                
+                st = time.time()
+                passport_data = get_response_from_openai_irq(compressed_image_data, openai_key=self.openai_key,side="page1")
+                logging.info(f'----------------Time taken for OpenAI and  final extraction passport: {time.time() - st} seconds\n')
+                logging.info(f"passport_data: {json.dumps(passport_data, ensure_ascii=False, indent=2)}")
+                
+                if not passport_data.get('header_verified', False):
                     passport_data["error"] = "not_passport"
                     return passport_data
+                 
+                if passport_data.get('full_name', '') and passport_data.get('last_name', ''):
+                    if passport_data['last_name'] not in passport_data['full_name']:  
+                        passport_data['full_name'] = passport_data.get('full_name', '') +' '+ passport_data.get('last_name', '')
+                
+                if passport_data.get('dob', None):
+                    from idvpackage.ocr_utils import is_age_18_above
+                    if not is_age_18_above(passport_data.get('dob', None)):
+                        passport_data['error'] = 'under_age'
+                        logging.info("Passport holder is under age")
+                        return passport_data
 
-                original_text = passport_text
-
-                passport_details = iraq_passport_extraction(passport_text)
-
-                passport_details['id_number'] = passport_details.get('passport_number',
-                                                                        passport_details.get('passport_number_mrz'))
-                if passport_details.get('passport_number_mrz') and passport_details['id_number'] != passport_details[
-                    'passport_number_mrz']:
-                    passport_details['id_number'] = passport_details['passport_number_mrz']
+                if passport_data.get('expiry_date'):
+                    from idvpackage.ocr_utils import is_expired_id
+                    if is_expired_id(passport_data.get('expiry_date', None)):
+                        passport_data['error'] = 'expired_id'
+                        logging.info(f"ID is expired with expiry date: {passport_data.get('expiry_date','')}")
+                        return passport_data
+                
+                if passport_data.get('mrz2', ''):
+                    mrz2 = passport_data.get('mrz2')
+                    logging.info(f"Original mrz2: {mrz2}")
+                    mrz2 = mrz2.replace('<', '')
+                    logging.info(f"Cleaned mrz2: {mrz2}")
+                    if len(mrz2) <28:
+                        passport_data['error'] = 'missing_mrz'
+                        logging.info("MRZ2 length is less than expected after cleaning")
+                        return passport_data
 
-                if not passport_details.get('passport_date_of_birth_generic') and passport_details.get('dob_mrz'):
-                    passport_details['dob'] = passport_details['dob_mrz']
-                else:
-                    passport_details['dob'] = passport_details['passport_date_of_birth_generic']
-
-                if (not passport_details.get('full_name') and passport_details.get('full_name_generic')) or (
-                        len(passport_details.get('full_name')) < len(passport_details.get('full_name_generic'))):
-                    if passport_details['full_name_generic'].startswith('IRQ'):
-                        passport_details['full_name'] = passport_details['full_name_generic'][3:].strip()
-                    else:
-                        passport_details['full_name'] = passport_details['full_name_generic']
-
-                # if len(passport_details.get('full_name')) < len(passport_details.get('full_name_generic')):
-                #     passport_details['full_name'] = passport_details['full_name_generic']
-
-                if not passport_details.get('last_name') and passport_details.get('surname_generic'):
-                    if passport_details['surname_generic'].startswith('IRQ'):
-                        passport_details['last_name'] = passport_details['surname_generic'][3:].strip()
-                    else:
-                        passport_details['last_name'] = passport_details['surname_generic']
-
-                if not passport_details.get('passport_date_of_expiry_generic') and passport_details.get(
-                        'expiry_date_mrz'):
-                    passport_details['expiry_date'] = passport_details['expiry_date_mrz']
-                else:
-                    passport_details['expiry_date'] = passport_details['passport_date_of_expiry_generic']
+                passport_data['gender'] = ''
+                if passport_data.get('gender_letter', None):
+                    if passport_data.get('gender_letter', '') in ['M', 'm']:
+                        passport_data['gender'] = 'Male'
+                    elif passport_data.get('gender_letter', '') in ['F', 'f']:
+                        passport_data['gender'] = 'Female'
+                
+                if passport_data.get('mrz1') and passport_data.get('mrz2'):
+                     passport_data['mrz'] = passport_data['mrz1'] + passport_data['mrz2']
 
-                keys_to_delete = ['expiry_date_mrz', 'passport_date_of_expiry_generic', 'dob_mrz',
-                                    'passport_date_of_birth_generic',
-                                    'passport_number', 'passport_number_mrz', 'full_name_generic', 'surname_generic']
+                if len(passport_data['mrz']) == 0 or not passport_data['mrz'][0].strip():
+                    return {
+                        "error": "missing_mrz",
+                        "error_details": "Missing or empty MRZ fields"
+                    } 
+
+                passport_data['issuing_country'] = 'IRQ'   
+                nationality_to_check = passport_data.get('nationality')
+                valid_nationality_result = self.check_nationality_in_iso_list(nationality_to_check)
+                logging.info(f"valid_nationality_result for passport: {valid_nationality_result}")
+                passport_data['valid_nationality'] = valid_nationality_result
+
+                logging.info(f"checking all passport_data fields for empty non-optional keys{passport_data.keys()}")
+                empty_string_keys = [ key for key, value in passport_data.items() if value == "" or value == [] or value == "[]"]
+                if empty_string_keys:
+                    passport_data['error'] = 'missing_key_fields'
+                    logging.warning(f"Empty string fields found in passport_data: {empty_string_keys}")
 
-                for key in keys_to_delete:
-                    if key in passport_details.keys():
-                        del passport_details[key]
+                st = time.time()
+                from idvpackage.common import load_and_process_image_deepface
+                front_face_locations, front_face_encodings = load_and_process_image_deepface(passport)
+                logging.info(f'----------------Time taken for face extraction: {time.time() - st} seconds\n')
 
-                passport_data.update(passport_details)
 
-                image = np.array(processed_passport)
-                doc_on_pp_result = document_on_printed_paper(image)
-                screenshot_result = detect_screenshot(self.client, passport)
-                photo_on_screen_result = detect_photo_on_screen(self.client, passport)
-                blurred, glare = self.get_blurred_and_glared_for_doc(image)
-                valid_nationality_result = self.check_nationality_in_iso_list(passport_details.get('nationality'))
+                if front_face_encodings is None or len(front_face_encodings) == 0:
+                    passport_data['error'] = 'face_not_detected'
+                    logging.info("No face detected in passport image")
+                    return passport_data
 
-                front_face_locations, front_face_encodings = load_and_process_image_deepface(passport)
+                front_face_locations_str = json.dumps([tuple(face_loc) for face_loc in front_face_locations])
+                front_face_encodings_str = json.dumps([face_enc.tolist() for face_enc in front_face_encodings])
+                
+                result.update(passport_data)
 
                 if step_data:
                     try:
-                        print(f"Matching face from Passport with National ID")
-                        national_id_front_face_locations = step_data.get("front_face_locations")
+                        logging.info(f"Matching face from Passport with National ID")
+                        national_id_front_face_locations = json.loads(step_data.get("front_face_locations"))
                         national_id_front_face_encodings = json.loads(step_data.get("front_face_encodings"))
-                        national_id_front_face_encodings = np.array(national_id_front_face_encodings[0],
-                                                                    dtype=np.float32)
-                        similarity = self.calculate_similarity(national_id_front_face_encodings,
-                                                                front_face_encodings[0])
-                        passport_data["similarity_score"] = similarity
-                        if similarity <= 0.65:
-                            passport_data["error"] = 'face_mismatch'
-                    except Exception as e:
-                        passport_data["error"] = 'National ID Image Not Found'
-                        passport_data["error_details"] = e
+                        st = time.time()
 
-                front_face_locations_str = json.dumps([tuple(face_loc) for face_loc in front_face_locations])
-                front_face_encodings_str = json.dumps([face_enc.tolist() for face_enc in front_face_encodings])
+                        largest_face_index1 = national_id_front_face_locations.index(
+                            max(national_id_front_face_locations, key=lambda loc: (loc[2] - loc[0]) * (loc[3] - loc[1])))
+                        largest_face_index2 = front_face_locations.index(
+                            max(front_face_locations, key=lambda loc: (loc[2] - loc[0]) * (loc[3] - loc[1])))
 
-                passport_data_update = {
-                    # 'back_tampered_result': tampered_result_back,
-                    'passport_data': original_text,
-                    'front_coloured': True,
-                    'back_coloured': True,
-                    'front_logo_result': 'clear',
-                    'front_doc_on_pp': doc_on_pp_result,
-                    'front_screenshot_result': screenshot_result,
-                    'front_photo_on_screen_result': photo_on_screen_result,
-                    'doc_on_pp': doc_on_pp_result,
-                    'screenshot_result': screenshot_result,
-                    'photo_on_screen_result': photo_on_screen_result,
-                    'front_blurred': blurred,
-                    'front_glare': glare,
-                    'back_blurred': blurred,
-                    'back_glare': glare,
-                    'front_face_locations': front_face_locations_str,
-                    'front_face_encodings': front_face_encodings_str,
-                    'valid_nationality': valid_nationality_result,
-                    'issuing_country': 'IRQ',
-                    'nationality_received': nationality
-                }
+                        face_encoding1 = national_id_front_face_encodings[largest_face_index1]
+                        face_encoding2 = front_face_encodings[largest_face_index2]
 
-                passport_data.update(passport_data_update)
+                        similarity = self.calculate_similarity(face_encoding1, face_encoding2)
 
-                non_optional_keys = ["gender", "passport_date_of_birth", "id_number", "passport_date_of_expiry"]
-                empty_string_keys = [key for key, value in passport_data.items() if
-                                        key in non_optional_keys and value == '']
+                        logging.info(
+                            f'----------------Time taken for face extraction for matching passport with front id: {time.time() - st} seconds\n')
+                        result["similarity_score"] = similarity
+                        logging.info(f"Front ID and Passport similarity score: {similarity}")
+                        if similarity < 0.5:
+                            result["error"] = 'face_mismatch'
+                            return {"error": "face_mismatch",
+                                    "error_details": "Front ID and Passport Face dont match."}
+                        
+                    except Exception as e:
+                        logging.info(f"Error in face matching between passport and national id: {e}")
+                        result["error"] = 'covered_photo'
+                        result["error_details"] = e
+                        return result
+                        
+                data_temp = {
+                        'passport_data': "",
+                        'front_coloured': True,
+                        'back_coloured': True,
+                        'front_logo_result': 'clear',
+                        'front_doc_on_pp': "clear",
+                        'front_screenshot_result': 'clear',
+                        'front_photo_on_screen_result': 'clear',
+                        'doc_on_pp': "clear",
+                        'screenshot_result': 'clear',
+                        'photo_on_screen_result': 'clear',
+                        'front_blurred': 'front_blurred',
+                        'front_glare': 'front_glare',
+                        'back_blurred': 'front_blurred',
+                        'back_glare': 'front_glare',
+                        'front_face_locations': front_face_locations_str,
+                        'front_face_encodings': front_face_encodings_str,
+                        'issuing_country': 'IRQ',
+                        'nationality_received': nationality_to_check
+                    }
+                
+                result.update(data_temp)
 
-                if empty_string_keys:
-                    passport_data['error'] = 'covered_photo'
+                from idvpackage.ocr_utils import validation_checks_passport
+                validation_results = validation_checks_passport(passport_data, 'id_number', 'IRQ')
+                logging.info(f"validation_results: {json.dumps(validation_results, ensure_ascii=False, indent=2)}")
+                result.update(validation_results)
 
-            except Exception as e:
-                passport_data['error'] = 'bad_image'
-                passport_data['error_details'] = e
+                from idvpackage.ocr_utils import get_name_match_mrz
+                result['is_name_match_mrz'], result['name_mrz'] = get_name_match_mrz(result, "passport")
 
-            try:
-                new_dict = extract_mother_name_and_surname(passport_text)
-                new_dict['mother_first_name_en'] = new_dict.get('mother_first_name', '')
-                new_dict['mother_last_name_en'] = new_dict.get('mother_last_name', '')
-
-                from deep_translator import GoogleTranslator
-                new_dict['mother_first_name'] = GoogleTranslator('en', 'ar').translate(
-                    new_dict.get('mother_first_name', ''))
-                new_dict['mother_last_name'] = GoogleTranslator('en', 'ar').translate(
-                    new_dict.get('mother_last_name', ''))
-                passport_data.update(new_dict)
+                return result
+            
             except Exception as e:
-                passport_data['mother_first_name'], passport_data['mother_first_name_en'] = '', ''
-                passport_data['mother_last_name'], passport_data['mother_last_name_en'] = '', ''
-                print("error:", e)
-
-            return passport_data 
+                result['error'] = 'bad_image'
+                result['error_details'] = str(e)
+                logging.info(f"Exception details: {e}")
+                return result
 
         if (nationality and nationality == "LBN") or (
             not nationality and country == "LBN"
         ):
             logging.info("-------------Working on LBN Passport \n")
-            from idvpackage.ocr_utils import (
-                detect_photo_on_screen,
-                detect_screenshot,
-                document_on_printed_paper,
-            )
-         
-            from idvpackage.lebanon_id_extraction import get_response_from_openai_lbn
-            from idvpackage.common import load_and_process_image_deepface
             from idvpackage.blur_detection import is_image_blur
+            from idvpackage.common import load_and_process_image_deepface
+            from idvpackage.lebanon_id_extraction import get_response_from_openai_lbn
+            from idvpackage.ocr_utils import  detect_screenshot
 
             passport_data = {"error": "", "doc_type": "passport"}
 
             try:
-                st = time.time()
-                processed_passport = self.image_conversion(passport)
-                logging.info(
-                    f"----------------Time taken for image conversion passport: {time.time() - st} seconds\n"
-                )
-
-                st = time.time()
-                compressed_image = BytesIO()
-                processed_passport.save(
-                    compressed_image, format="PNG", quality=100, optimize=True
-                )
-                compressed_image_data = compressed_image.getvalue()
+                st = time.time()
+                processed_passport, compressed_image_data = self.image_conversion_and_compression(passport)
+                logging.info(f"----------------Time taken for image conversion passport: {time.time() - st} seconds\n")
 
+                st = time.time()
+                
                 image = np.array(processed_passport)
                 if country == "LBN":
                     if is_image_blur(
@@ -3581,21 +2328,17 @@ class IdentityVerification:
                         fft_threshold=120,
                         bright_reflection_min_area=1.0,
                     ):
-                        print(
-                            f"Blur/Brightness issue detected in front image, marking as covered photo"
-                        )
+                        logging.info(f"Blur/Brightness issue detected in front image, marking as covered photo")
                         passport_data["error"] = "blur_photo"
                         return passport_data
 
                 st = time.time()
                 passport_details = get_response_from_openai_lbn(compressed_image_data,"first", self.openai_key)
                 logging.info(
-                    f"----------------Time taken for data formatting and final extraction passport: {time.time() - st} seconds\n"
+                    f"----------------Time taken for OpenAI and  final extraction passport: {time.time() - st} seconds\n"
                 )
-
                 logging.info(f"passport_details: {json.dumps(passport_details, ensure_ascii=False, indent=2)}")
 
-
                 if not passport_details['header_verified'] :
                     passport_data["error"] = "not_passport"
                     return passport_data
@@ -3609,7 +2352,7 @@ class IdentityVerification:
                         passport_details['error'] = 'under_age'
                         return passport_details
                 else:
-                    passport_details['error'] = 'covered_photo'
+                    passport_details['error'] = 'missing_key_fields'
                     logging.error(f"DOB date not found in the extracted data.")
                     return passport_details
 
@@ -3625,51 +2368,40 @@ class IdentityVerification:
                     logging.info(f"Error in expiry date check: {e}")
 
                 passport_details['mrz'] = passport_details.get('mrz1', '')  + passport_details.get('mrz2', '')
-
+                passport_details['full_name'] = passport_details.get('first_name', '') + ' ' + passport_details.get('last_name', '')
                 passport_details['issuing_country'] = 'LBN'
-
                 passport_data.update(passport_details)
 
-                # if passport_data["id_number"][:2] == "PR":
-                #     passport_data["error"] = "not_allowed"
-                #     return passport_data
-
                 from idvpackage.ocr_utils import validation_checks_passport
-
-
                 validation_results = validation_checks_passport(passport_data, 'id_number', 'LBN')
                 logging.info(f"validation_results: {json.dumps(validation_results, ensure_ascii=False, indent=2)}")
                 passport_data.update(validation_results)
 
-                
                 from idvpackage.ocr_utils import get_name_match_mrz
                 passport_data['is_name_match_mrz'], passport_data['name_mrz'] = get_name_match_mrz(passport_data, "passport")
 
-
                 image = np.array(processed_passport)
-
                 st = time.time()
-                
                 doc_on_pp_result = "clear"
                 screenshot_result = detect_screenshot(self.client, passport)
-                # photo_on_screen_result = detect_photo_on_screen(self.client, passport)
                 photo_on_screen_result = "clear"
                 blurred, glare = self.get_blurred_and_glared_for_doc(image)
-                print(f"Nationality: {passport_details.get('nationality')}")
+                logging.info(f"Nationality: {passport_details.get('nationality')}")
                 valid_nationality_result = self.check_nationality_in_iso_list(
                     passport_details.get("nationality")
                 )
-                logging.info(
-                    f"----------------Time taken for fraud detection attributes passport: {time.time() - st} seconds\n"
-                )
+                logging.info(f"----------------Time taken for fraud detection attributes passport: {time.time() - st} seconds\n")
 
                 st = time.time()
-                front_face_locations, front_face_encodings = (
-                    load_and_process_image_deepface(passport)
-                )
+                front_face_locations, front_face_encodings = (load_and_process_image_deepface(passport))
                 logging.info(
                     f"----------------Time taken for face extraction passport: {time.time() - st} seconds\n"
                 )
+                
+                if front_face_encodings is None or len(front_face_encodings) == 0:
+                    passport_data["error"] = "face_not_detected"
+                    logging.info("No face detected in passport image")
+                    return passport_data
 
                 front_face_locations_str = json.dumps(
                     [tuple(face_loc) for face_loc in front_face_locations]
@@ -3679,7 +2411,6 @@ class IdentityVerification:
                 )
 
                 passport_data_update = {
-                    # 'back_tampered_result': tampered_result_back,
                     "passport_data": "",
                     "front_coloured": True,
                     "back_coloured": True,
@@ -3701,7 +2432,6 @@ class IdentityVerification:
                 }
 
                 passport_data.update(passport_data_update)
-
                 non_optional_keys = [
                     "front_face_locations",
                     "id_number",
@@ -3718,11 +2448,10 @@ class IdentityVerification:
                 ]
 
                 if empty_string_keys:
-                    passport_data["error"] = "covered_photo"
+                    passport_data["error"] = "missing_key_fields"
                     logging.info(f"Empty keys found: {empty_string_keys}")
                    
 
-
             except Exception as e:
                 passport_data["error"] = "bad_image"
                 passport_data["error_details"] = e
@@ -3734,34 +2463,20 @@ class IdentityVerification:
             not nationality and country == "SDN"
         ):
             logging.info("-------------Working on SDN Passport \n")
-            from idvpackage.ocr_utils import (
-                detect_photo_on_screen,
-                detect_screenshot,
-                document_on_printed_paper,
-            )
-            from idvpackage.sudan_passport_extraction import get_response_from_openai_sdn
-            from idvpackage.common import load_and_process_image_deepface
             from idvpackage.blur_detection import is_image_blur
-
+            from idvpackage.common import load_and_process_image_deepface
+            from idvpackage.ocr_utils import  detect_screenshot
+            from idvpackage.sudan_passport_extraction import get_response_from_openai_sdn
+            
             passport_data = {"error": "", "doc_type": "passport"}
-
             try:
                 st = time.time()
-                processed_passport = self.image_conversion(passport)
+                processed_passport, compressed_image_data = self.image_conversion_and_compression(passport)
                 logging.info(
                     f"----------------Time taken for image conversion passport: {time.time() - st} seconds\n"
                 )
 
                 st = time.time()
-                compressed_image = BytesIO()
-
-                
-                processed_passport.save(
-                    compressed_image, format="JPEG", quality=95, optimize=True
-                )
-                compressed_image_data = compressed_image.getvalue()
-               
-
                 image = np.array(processed_passport)
                 if country == "SDN":
                     if is_image_blur(
@@ -3772,18 +2487,15 @@ class IdentityVerification:
                         bright_reflection_threshold=100,
                         bright_reflection_min_area=1.0,
                     ):
-                        print(f"Passport Document is blurry, marking as covered photo")
+                        logging.info(f"Passport Document is blurry, marking as covered photo")
                         passport_data["error"] = "blur_photo"
                         return passport_data
 
                 st = time.time()
-                # passport_details = sdn_passport_extraction(passport_text)
                 passport_details = get_response_from_openai_sdn(compressed_image_data, "passport", self.openai_key)
 
                 logging.info(f"Passport details extracted: {json.dumps(passport_details, ensure_ascii=False, indent=2)}")
-                logging.info(
-                    f"----------------Time taken for data formatting and final extraction passport: {time.time() - st} seconds\n"
-                )
+                logging.info(f"----------------Time taken for OpenAI and  final extraction passport: {time.time() - st} seconds\n")
 
                 if not passport_details['header_verified']:
                     passport_data["error"] = "not_passport"
@@ -3794,6 +2506,8 @@ class IdentityVerification:
                 passport_details['nationality_received'] = 'SDN'
                 passport_details['nationality'] = 'SDN'
 
+                passport_details['full_name'] = passport_details.get('first_name','')+ passport_details.get('last_name','')   
+
                 from idvpackage.ocr_utils import is_expired_id
                 if is_expired_id(passport_details.get('expiry_date', None)):
                     passport_data['error'] = 'expired_id'
@@ -3810,15 +2524,11 @@ class IdentityVerification:
 
                 if passport_details.get("name_en", ""):
                     passport_details['full_name_generic'] = passport_details.get("name_en","")
-                
                 else:
                     passport_details['full_name_generic'] = passport_details.get("first_name","") + " " + passport_details.get("middle_name","") + " " + passport_details.get("last_name","")
                 
                 passport_data.update(passport_details)
 
-               
-                
-
                 from idvpackage.ocr_utils import validation_checks_passport
                 validation_results = validation_checks_passport(passport_data, 'passport_number', 'SDN')
                 logging.info(f"validation_results: {json.dumps(validation_results, ensure_ascii=False, indent=2)}")
@@ -3828,27 +2538,18 @@ class IdentityVerification:
                 from idvpackage.ocr_utils import get_name_match_mrz
                 passport_data['is_name_match_mrz'], passport_data['name_mrz'] = get_name_match_mrz(passport_data, "passport")
 
-                
                 if passport_data.get("issue_date"):
                     passport_data["issuance_date"] = passport_details["issue_date"]
 
-
-              
-
                 image = np.array(processed_passport)
 
                 st = time.time()
-               
                 doc_on_pp_result = "clear"
                 screenshot_result = detect_screenshot(self.client, passport)
                 photo_on_screen_result = "clear"
                 blurred, glare = self.get_blurred_and_glared_for_doc(image)
-                valid_nationality_result = self.check_nationality_in_iso_list(
-                    passport_details.get("nationality")
-                )
-                logging.info(
-                    f"----------------Time taken for fraud detection attributes passport: {time.time() - st} seconds\n"
-                )
+                valid_nationality_result = self.check_nationality_in_iso_list(passport_details.get("nationality"))
+                logging.info(f"----------------Time taken for fraud detection attributes passport: {time.time() - st} seconds\n")
 
                 st = time.time()
                 front_face_locations, front_face_encodings = (
@@ -3857,6 +2558,11 @@ class IdentityVerification:
                 logging.info(
                     f"----------------Time taken for face extraction passport: {time.time() - st} seconds\n"
                 )
+                
+                if front_face_encodings is None or len(front_face_encodings) == 0:
+                    passport_data["error"] = "face_not_detected"
+                    logging.info("No face detected in passport image")
+                    return passport_data
 
                 front_face_locations_str = json.dumps(
                     [tuple(face_loc) for face_loc in front_face_locations]
@@ -3866,7 +2572,6 @@ class IdentityVerification:
                 )
 
                 passport_data_update = {
-                    # 'back_tampered_result': tampered_result_back,
                     "passport_data": "",
                     "front_coloured": True,
                     "back_coloured": True,
@@ -3889,11 +2594,10 @@ class IdentityVerification:
                 }
 
                 passport_data.update(passport_data_update)
-
                 non_optional_keys = [
-                    "passport_number_mrz",
-                    "dob_mrz",
-                    "expiry_date_mrz",
+                    "passport_number",
+                    "dob",
+                    "expiry_date",
                     "gender",
                 ]
                 empty_string_keys = [
@@ -3904,10 +2608,11 @@ class IdentityVerification:
 
                 if empty_string_keys:
                     passport_data["error"] = "missing_key_fields"
+                    logging.info(f"Empty keys found: {empty_string_keys}")
 
             except Exception as e:
                 passport_data["error"] = "bad_image"
-                print(f"-------------->> Something went wrong error trace:: {e}")
+                logging.info(f"-------------->> Something went wrong error trace:: {e}")
                 passport_data["error_details"] = e
 
             return passport_data
@@ -3916,97 +2621,93 @@ class IdentityVerification:
             not nationality and country == "SYR"
         ):
             logging.info("-------------Working on SYR Passport \n")
-            from idvpackage.ocr_utils import (
-                detect_photo_on_screen,
-                detect_screenshot,
-                document_on_printed_paper,
-            )
-            # from idvpackage.syr_passport_extraction import (
-            #     syr_passport_extraction_front,
-            #     syr_passport_extraction_back,
-            # )
-
-            from idvpackage.syr_passport_extraction import get_response_from_openai_syr
-
             from idvpackage.common import load_and_process_image_deepface
+            from idvpackage.ocr_utils import detect_screenshot
 
+            from idvpackage.syr_passport_extraction import get_response_from_openai_syr
             passport_data = {"error": "", "doc_type": "passport"}
 
             try:
                 st = time.time()
                 processed_passport = self.image_conversion(passport)
-                # logging.info(
-                #     f"----------------Time taken for image conversion passport: {time.time() - st} seconds\n"
-                # )
-
-                # st = time.time()
-                # id_infos = self.get_ocr_results(processed_passport)
-                # passport_text = id_infos[0].description
-                # logging.info(
-                #     f"----------------Time taken for vision passport: {time.time() - st} seconds\n"
-                # )
-
-                # # print(f"\nPassport text: {passport_text}\n")
-
-                # pattern4 = r"(Syrian Arab Republic|REPUBLIQUE ARABE SYRIEH|Syrian|Syrienne|SYR)"
-                # k = re.search(pattern4, passport_text, re.IGNORECASE)
-
-                # if not k:
-                #     passport_data["error"] = "not_passport"
-                #     return passport_data
-
                 st = time.time()
-                # passport_details = syr_passport_extraction_front(
-                #     passport_text, self.gemini_key
-                # )
                 passport_details = get_response_from_openai_syr(processed_passport, "page1", country, self.openai_key)
+                logging.info(
+                    f"----------------Time taken for OpenAI and  final extraction passport: {time.time() - st} seconds\n"
+                )
+                logging.info(f"passport_details: {json.dumps(passport_details, ensure_ascii=False, indent=2)}")
 
                 if not passport_details['header_verified']:
                     passport_data["error"] = "not_passport"
                     return passport_data
-
+                
+                if passport_details.get('dob', None):
+                    from idvpackage.ocr_utils import is_age_18_above
+                    if not is_age_18_above(passport_details.get('dob', None)):
+                        passport_data['error'] = 'under_age'
+                        logging.info("Passport holder is under age")
+                        return passport_data
+                       
+                passport_details['full_name'] = passport_details.get('first_name','') + ' ' + passport_details.get('last_name','')
                 passport_details['nationality'] = "SYR"
                 passport_details['mrz'] = passport_details.get('mrz1','') + passport_details.get('mrz2','')
+                
+                #check is passport_number is all digits and len is 9
+                if  passport_details.get('passport_number', '').isdigit() and  len(passport_details.get('passport_number', '')) == 9:
+                    passport_details['id_number'] = passport_details.get('passport_number','')        
+                elif passport_details.get('passport_number_mrz', '').isdigit() and len(passport_details.get('passport_number_mrz', '')) == 9:
+                    passport_details['id_number'] = passport_details.get('passport_number_mrz','')
+                else:
+                    passport_details['id_number'] = ''
+
+                mrz2 = passport_details.get('mrz2','')
+                mrz2 = mrz2.strip()
+                logging.info(f"mrz2: {mrz2}")
+                mrz2 = mrz2[-16:]
+                logging.info(f"mrz2: last 16 chars: {mrz2}")
+                mrz2 = mrz2.split("<<")[0].replace('<','')
+                passport_details['national_number_mrz_regex'] = mrz2
+                
+                if passport_details.get('gender_mrz','').upper() =='M':
+                    passport_details['gender_mrz'] = 'MALE'
+                elif passport_details.get('gender_mrz','').upper() =='F':
+                    passport_details['gender_mrz'] = 'FEMALE'
+                    
+                if passport_details.get('gender','').upper() =='M':
+                    passport_details['gender'] = 'MALE'
+                elif passport_details.get('gender','').upper() =='F':
+                    passport_details['gender'] = 'FEMALE'
 
-                try:
-                    passport_details['passport_number'] = passport_details.get('mrz2')[0:9]
-                    passport_details['id_number'] = passport_details['passport_number']
-                except KeyError:
-                    passport_details['passport_number'] = ""
-                    passport_details['id_number'] = ""
-
+                from idvpackage.ocr_utils import get_name_match_mrz
+                passport_details['is_name_match_mrz'], passport_details['name_mrz'] = get_name_match_mrz(passport_details, "passport")
+            
                 logging.info(f"----------------Passport details front: {json.dumps(passport_details, indent=2, ensure_ascii=False)}\n")
-                logging.info(
-                    f"----------------Time taken for data formatting and final extraction passport: {time.time() - st} seconds\n"
-                )
+                logging.info(f"----------------Time taken for OpenAI and  final extraction passport: {time.time() - st} seconds\n")
 
                 passport_data.update(passport_details)
 
                 image = np.array(processed_passport)
 
                 st = time.time()
-                ## TODO: doc_on_pp and detect_photo_on_screen for LBN
-                # doc_on_pp_result = document_on_printed_paper(image)
+              
                 doc_on_pp_result = "clear"
                 screenshot_result = detect_screenshot(self.client, passport)
-                # photo_on_screen_result = detect_photo_on_screen(self.client, passport)
                 photo_on_screen_result = "clear"
                 blurred, glare = self.get_blurred_and_glared_for_doc(image)
                 valid_nationality_result = self.check_nationality_in_iso_list(
                     passport_details.get("nationality")
                 )
-                logging.info(
-                    f"----------------Time taken for fraud detection attributes passport: {time.time() - st} seconds\n"
-                )
-
+                logging.info(f"----------------Time taken for fraud detection attributes passport: {time.time() - st} seconds\n")
                 st = time.time()
-                front_face_locations, front_face_encodings = (
-                    load_and_process_image_deepface(passport)
-                )
-                logging.info(
-                    f"----------------Time taken for face extraction passport: {time.time() - st} seconds\n"
-                )
+                front_face_locations, front_face_encodings = (load_and_process_image_deepface(passport))
+                logging.info(f"----------------Time taken for face extraction passport: {time.time() - st} seconds\n")
+
 
+                if front_face_encodings is None or len(front_face_encodings) == 0:
+                    passport_data["error"] = "face_not_detected"
+                    logging.info("No face detected in passport image")
+                    return passport_data
+                
                 front_face_locations_str = json.dumps(
                     [tuple(face_loc) for face_loc in front_face_locations]
                 )
@@ -4015,7 +2716,6 @@ class IdentityVerification:
                 )
 
                 passport_data_update = {
-                    # 'back_tampered_result': tampered_result_back,
                     "passport_data": "",
                     "front_coloured": True,
                     "back_coloured": True,
@@ -4045,9 +2745,8 @@ class IdentityVerification:
                     for key, value in passport_data.items()
                     if key in non_optional_keys and value == ""
                 ]
-
                 if empty_string_keys:
-                    passport_data["error"] = "covered_photo"
+                    passport_data["error"] = "missing_key_fields"
 
             except Exception as e:
                 passport_data["error"] = "bad_image"
@@ -4060,58 +2759,23 @@ class IdentityVerification:
             not nationality and country == "JOR"
         ):
             logging.info("-------------Working on JOR Passport \n")
-            from idvpackage.ocr_utils import (
-                detect_photo_on_screen,
-                detect_screenshot,
-                document_on_printed_paper,
-            )
-            from idvpackage.jor_passport_extraction import get_response_from_openai_jor
             from idvpackage.common import load_and_process_image_deepface
-
+            from idvpackage.jor_passport_extraction import get_response_from_openai_jor
+            from idvpackage.ocr_utils import  detect_screenshot
+            
             passport_data = {"error": "", "doc_type": "passport"}
-
             try:
                 st = time.time()
                 logging.info("Performing image conversion for passport")
                 processed_passport = self.image_conversion(passport)
-                logging.info("Image conversion completed for passport")
-
-                logging.info(
-                    f"----------------Time taken for image conversion passport: {time.time() - st} seconds\n"
-                )
-
-                # try:
-                #     st = time.time()
-                #     id_infos = self.get_ocr_results(processed_passport)
-                #     passport_text = id_infos[0].description
-                #     logging.info(f"Passport text extracted: {passport_text}")
-                #     logging.info(
-                #         f"----------------Time taken for vision passport: {time.time() - st} seconds\n"
-                #     )
-                # except Exception as e:
-                #     logging.error(f"Error during get_ocr_results: {e}")
-                #     passport_text = ""
-                #     pass
-
-                # pattern4 = r"(Hashemite Kingdom of Jordan|Hashemite Kingdom|Jordan)"
-                # k = re.search(pattern4, passport_text, re.IGNORECASE)
-
-                # if not k:
-                #     passport_data["error"] = "not_passport"
-                #     return passport_data
+                logging.info(f"----------------Time taken for image conversion passport: {time.time() - st} seconds\n")
 
                 logging.info("Performing OCR for passport using openai.......")
                 st = time.time()
-                passport_details = get_response_from_openai_jor(
-                    processed_passport, "first", nationality, self.openai_key
-                )
+                passport_details = get_response_from_openai_jor(processed_passport, "first", nationality, self.openai_key)
 
-                logging.info(
-                    f"----------------Passport details: {json.dumps(passport_details, indent=4, ensure_ascii=False)}\n"
-                )
-                logging.info(
-                    f"----------------Time taken for openai final extraction passport: {time.time() - st} seconds\n"
-                )
+                logging.info(f"----------------Passport details: {json.dumps(passport_details, indent=4, ensure_ascii=False)}\n")
+                logging.info(f"----------------Time taken for openai final extraction passport: {time.time() - st} seconds\n")
 
                 if (
                     passport_details.get("header_verified", "")
@@ -4120,77 +2784,59 @@ class IdentityVerification:
                     passport_data["error"] = "not_passport"
                     return passport_data
 
+                from idvpackage.ocr_utils import is_expired_id
+                if is_expired_id(passport_details.get('expiry_date', None)):
+                    passport_data['error'] = 'expired_id'
+                    logging.info(f"ID is expired with expiry date: {passport_details.get('expiry_date','')}")
+                    return passport_data
 
-                # append mrz1 and mrz2 from passport_details and make it as list
+                if passport_details.get('dob', None):
+                    from idvpackage.ocr_utils import is_age_18_above
+                    if not is_age_18_above(passport_details.get('dob', None)):
+                        passport_data['error'] = 'under_age'
+                        logging.info("Passport holder is under age")
+                        return passport_data
+                
+                  # append mrz1 and mrz2 from passport_details and make it as list
+                passport_details['full_name'] = passport_details.get('first_name','') +  ' ' + passport_details.get('last_name','')  
                 passport_details['mrz'] = [passport_details.get('mrz1','') + ' ' + passport_details.get('mrz2','')]
                 passport_details['issuing_place'] = passport_details.get('place_of_issue','')
+                passport_details['issue_date'] = passport_details.get('issuing_date','')
+                        
                 passport_data.update(passport_details)
-                passport_data['issue_date'] = passport_data.get('issuing_date','')
-
+               
                 from idvpackage.ocr_utils import validation_checks_passport
-
-
                 validation_results = validation_checks_passport(passport_data, 'passport_number', 'JOR')
                 logging.info(f"validation_results: {json.dumps(validation_results, ensure_ascii=False, indent=2)}")
                 passport_data.update(validation_results)
                 logging.info(f"Passport data after validation checks: {json.dumps(passport_data, ensure_ascii=False, indent=2)}")
-                #extract name from mrz1
-                # mrz1 = passport_details.get('mrz1', '')
-
                 
-                # logging.info(f"MRZ1 extracted: {mrz1}")
-                # if mrz1:
-                #     try:
-                #         mrz1 = mrz1[5:]
-                #         logging.info(f"Processed MRZ1: {mrz1}")
-                #         name_mrz = []
-                #         for word in mrz1.split("<"):
-                #             if word and  word.isalpha():
-                #                 name_mrz.append(word)
-
-                #         passport_data['name_mrz'] = " ".join(name_mrz)
-                #         logging.info(f"Name from MRZ1 parts: {name_mrz}")
-
-                #         name = passport_data.get("name", "") 
-                #         name = name.split(" ")
-                #         from idvpackage.ocr_utils import get_name_match_mrz
-                #         passport_data['is_name_match_mrz'] = get_name_match_mrz(name, name_mrz)
-                #         logging.info(f"is_name_match_mrz: {passport_data['is_name_match_mrz']}")
-
-                #     except Exception as e:
-                #         logging.info(f"Error in processing Name from MRZ1: {e}")
-                #         passport_data['is_name_match_mrz'] = False
-                #         pass
                 from idvpackage.ocr_utils import get_name_match_mrz
                 passport_data['is_name_match_mrz'], passport_data['name_mrz'] = get_name_match_mrz(passport_data, "passport")
 
-
-
                 image = np.array(processed_passport)
 
                 st = time.time()
-                ## TODO: doc_on_pp and detect_photo_on_screen for LBN
-                # doc_on_pp_result = document_on_printed_paper(image)
                 doc_on_pp_result = "clear"
                 screenshot_result = detect_screenshot(self.client, passport)
-                # photo_on_screen_result = detect_photo_on_screen(self.client, passport)
                 photo_on_screen_result = "clear"
                 blurred, glare = self.get_blurred_and_glared_for_doc(image)
                 valid_nationality_result = self.check_nationality_in_iso_list(
                     passport_details.get("nationality")
                 )
-                logging.info(
-                    f"----------------Time taken for fraud detection attributes passport: {time.time() - st} seconds\n"
-                )
+                logging.info(f"----------------Time taken for fraud detection attributes passport: {time.time() - st} seconds\n")
 
                 st = time.time()
                 front_face_locations, front_face_encodings = (
                     load_and_process_image_deepface(passport)
                 )
-                logging.info(
-                    f"----------------Time taken for face extraction passport: {time.time() - st} seconds\n"
-                )
+                logging.info(f"----------------Time taken for face extraction passport: {time.time() - st} seconds\n")
 
+                if front_face_encodings is None or len(front_face_encodings) == 0:
+                    passport_data["error"] = "face_not_detected"
+                    logging.info("No face detected in passport image")
+                    return passport_data
+                
                 front_face_locations_str = json.dumps(
                     [tuple(face_loc) for face_loc in front_face_locations]
                 )
@@ -4199,7 +2845,6 @@ class IdentityVerification:
                 )
 
                 passport_data_update = {
-                    # 'back_tampered_result': tampered_result_back,
                     "passport_data": "",
                     "front_coloured": True,
                     "back_coloured": True,
@@ -4231,7 +2876,7 @@ class IdentityVerification:
                 ]
 
                 if empty_string_keys:
-                    passport_data["error"] = "covered_photo"
+                    passport_data["error"] = "missing_key_fields"
 
             except Exception as e:
                 passport_data["error"] = "bad_image"
@@ -4244,44 +2889,16 @@ class IdentityVerification:
             not nationality and country == "PSE"
         ):
             logging.info("-------------Working on PSE Passport \n")
-            from idvpackage.ocr_utils import (
-                detect_photo_on_screen,
-                detect_screenshot,
-                document_on_printed_paper,
-            )
-            # from idvpackage.pse_passport_extraction import palestine_passport_extraction
-            from idvpackage.pse_passport_extraction import get_response_from_openai_pse
-
             from idvpackage.common import load_and_process_image_deepface
-
+            from idvpackage.ocr_utils import detect_screenshot
+            from idvpackage.pse_passport_extraction import get_response_from_openai_pse
             passport_data = {"error": "", "doc_type": "passport"}
 
             try:
                 st = time.time()
                 processed_passport = self.image_conversion(passport)
-                # logging.info(
-                #     f"----------------Time taken for image conversion passport: {time.time() - st} seconds\n"
-                # )
-
-                # st = time.time()
-                # id_infos = self.get_ocr_results(processed_passport)
-                # passport_text = id_infos[0].description
-                # logging.info(
-                #     f"----------------Time taken for vision passport: {time.time() - st} seconds\n"
-                # )
-
-                # pattern4 = r"(PALESTINIAN AUTHORITY|PALESTINE|P<PSE|PSE)"  # PASSPORT
-                # k = re.search(pattern4, passport_text, re.IGNORECASE)
-
-                # if not k:
-                #     passport_data["error"] = "not_passport"
-                #     return passport_data
-
                 st = time.time()
                 try:
-                    # passport_details = palestine_passport_extraction(
-                    #     passport_text, self.gemini_key
-                    # )
                     passport_details = get_response_from_openai_pse(
                         processed_passport,"first",nationality,self.openai_key
                     )
@@ -4289,29 +2906,39 @@ class IdentityVerification:
                 except Exception as e:
                     logging.error(f"Error in PSE passport extraction: {e}")
                 logging.info(
-                    f"----------------Time taken for data formatting and final extraction passport: {time.time() - st} seconds\n"
+                    f"----------------Time taken for OpenAI and  final extraction passport: {time.time() - st} seconds\n"
                 )
 
                 if passport_details.get("header_verified", "") == "False":
                     passport_data["error"] = "not_passport"
                     return passport_data
 
+                if passport_details.get('dob', None):
+                    from idvpackage.ocr_utils import is_age_18_above
+                    if not is_age_18_above(passport_details.get('dob', None)):
+                        passport_data['error'] = 'under_age'
+                        logging.info("Passport holder is under age")
+                        return passport_data
+                    
+                from idvpackage.ocr_utils import is_expired_id
+                if is_expired_id(passport_details.get('expiry_date', None)):
+                    passport_data['error'] = 'expired_id'
+                    logging.info(f"ID is expired with expiry date: {passport_details.get('expiry_date','')}")
+                    return passport_data
+                
                 passport_details['nationality'] = 'PSE'
-
+                passport_details['full_name_openai'] = passport_details.get('full_name','')
+                passport_details['full_name'] = passport_details.get('first_name','') + ' ' + passport_details.get('last_name','')
                 passport_details['mrz'] = passport_details.get('mrz1','') + passport_details.get('mrz2','')
+                passport_details['issue_date'] = passport_details.get('issuing_date','')
+                
                 
                 passport_data.update(passport_details)
-
-
-
                 image = np.array(processed_passport)
 
                 st = time.time()
-                ## TODO: doc_on_pp and detect_photo_on_screen for LBN
-                # doc_on_pp_result = document_on_printed_paper(image)
                 doc_on_pp_result = "clear"
                 screenshot_result = detect_screenshot(self.client, passport)
-                # photo_on_screen_result = detect_photo_on_screen(self.client, passport)
                 photo_on_screen_result = "clear"
                 blurred, glare = self.get_blurred_and_glared_for_doc(image)
                 valid_nationality_result = self.check_nationality_in_iso_list(
@@ -4320,7 +2947,6 @@ class IdentityVerification:
                 logging.info(
                     f"----------------Time taken for fraud detection attributes passport: {time.time() - st} seconds\n"
                 )
-
                 st = time.time()
                 front_face_locations, front_face_encodings = (
                     load_and_process_image_deepface(passport)
@@ -4328,7 +2954,12 @@ class IdentityVerification:
                 logging.info(
                     f"----------------Time taken for face extraction passport: {time.time() - st} seconds\n"
                 )
-
+                
+                if front_face_encodings is None or len(front_face_encodings) == 0:
+                    passport_data["error"] = "face_not_detected"
+                    logging.info("No face detected in passport image")
+                    return passport_data
+                
                 front_face_locations_str = json.dumps(
                     [tuple(face_loc) for face_loc in front_face_locations]
                 )
@@ -4337,7 +2968,6 @@ class IdentityVerification:
                 )
 
                 passport_data_update = {
-                    # 'back_tampered_result': tampered_result_back,
                     "passport_data": "",
                     "front_coloured": True,
                     "back_coloured": True,
@@ -4361,15 +2991,23 @@ class IdentityVerification:
 
                 passport_data.update(passport_data_update)
 
+                from idvpackage.ocr_utils import validation_checks_passport
+                validation_results = validation_checks_passport(passport_details, 'id_number', 'PSE')
+                logging.info(f"validation_results: {json.dumps(validation_results, ensure_ascii=False, indent=2)}")
+                passport_data.update(validation_results)
+
+                from idvpackage.ocr_utils import get_name_match_mrz
+                passport_data['is_name_match_mrz'], passport_data['name_mrz'] = get_name_match_mrz(passport_data, "passport")
+                
+                
                 non_optional_keys = ["passport_number", "dob", "expiry_date", "gender"]
                 empty_string_keys = [
                     key
                     for key, value in passport_data.items()
                     if key in non_optional_keys and value == ""
                 ]
-
                 if empty_string_keys:
-                    passport_data["error"] = "covered_photo"
+                    passport_data["error"] = "missing_key_fields"
 
             except Exception as e:
                 passport_data["error"] = "bad_image"
@@ -4377,11 +3015,10 @@ class IdentityVerification:
 
             return passport_data
 
-    def exract_passport_info_back(self, passport, country, nationality, step_data=None):
+    def extract_passport_info_back(self, passport, country, nationality, step_data=None):
         if (nationality and nationality == "SYR") or (
             not nationality and country == "SYR"
         ):
-            # from idvpackage.syr_passport_extraction import syr_passport_extraction_back
             from idvpackage.syr_passport_extraction import get_response_from_openai_syr
 
             passport_data = {"error": "", "doc_type": "passport"}
@@ -4389,34 +3026,33 @@ class IdentityVerification:
             try:
                 st = time.time()
                 processed_passport = self.image_conversion(passport)
-                # logging.info(
-                #     f"----------------Time taken for image conversion passport: {time.time() - st} seconds\n"
-                # )
-
-                # st = time.time()
-                # id_infos = self.get_ocr_results(processed_passport)
-                # passport_text = id_infos[0].description
-                # logging.info(
-                #     f"----------------Time taken for vision passport: {time.time() - st} seconds\n"
-                # )
-
-                # # print(f"\nPassport text: {passport_text}\n")
-
+                logging.info(f"----------------Time taken for image conversion passport: {time.time() - st} seconds\n")
+                
                 st = time.time()
-                # passport_details = syr_passport_extraction_back(
-                #     passport_text, self.gemini_key
-                # )
                 passport_details = get_response_from_openai_syr(processed_passport, "page2", country, self.openai_key)
-
                 logging.info(f"----------------Passport details back: {json.dumps(passport_details, indent=2, ensure_ascii=False)}\n")
-                logging.info(
-                    f"----------------Time taken for data formatting and final extraction passport: {time.time() - st} seconds\n"
-                )
+                logging.info(f"----------------Time taken for OpenAI and  final extraction passport: {time.time() - st} seconds\n")
 
                 passport_details['issuing_date'] = passport_details.get('issue_date','')
                 passport_data.update(passport_details)
-
-
+                
+                from idvpackage.ocr_utils import is_expired_id
+                if is_expired_id(passport_details.get('expiry_date', None)):
+                    passport_data['error'] = 'expired_id'
+                    logging.info(f"ID is expired with expiry date: {passport_details.get('expiry_date','')}")
+                    return passport_data
+                                
+                temp_data = {}
+                if step_data:
+                    temp_data.update(step_data)
+                temp_data.update(passport_details)
+                
+                logging.info(f"temp_data for validation: {json.dumps(temp_data, ensure_ascii=False, indent=2)}")
+                from idvpackage.ocr_utils import validation_checks_passport
+                validation_results = validation_checks_passport(temp_data, 'passport_number', 'SYR')
+                logging.info(f"validation_results: {json.dumps(validation_results, ensure_ascii=False, indent=2)}")
+                passport_data.update(validation_results)
+                
                 non_optional_keys = ["issuing_date", "expiry_date"]
                 empty_string_keys = [
                     key
@@ -4425,7 +3061,7 @@ class IdentityVerification:
                 ]
 
                 if empty_string_keys:
-                    passport_data["error"] = "covered_photo"
+                    passport_data["error"] = "missing_key_fields"
 
             except Exception as e:
                 passport_data["error"] = "bad_image"
@@ -4433,25 +3069,8 @@ class IdentityVerification:
 
         return passport_data
 
-    def replace_keywords(self, report, target="consider", replacement="clear"):
-        if isinstance(report, dict):
-            # Iterate through dictionary items
-            for key, value in report.items():
-                # If the value matches the target, replace it
-                if value == target:
-                    report[key] = replacement
-                else:
-                    # Recursively call for nested dictionaries or lists
-                    self.replace_keywords(value, target, replacement)
-        elif isinstance(report, list):
-            # If the report is a list, iterate through elements
-            for index, item in enumerate(report):
-                # Recursively call for each element
-                self.replace_keywords(item, target, replacement)
-
     def extract_ocr_info(self, data, video, country, report_names, back_img=None):
 
-
         try:
             if data.get("uae_pass_data", ""):
                 uae_pass_data = data.get("uae_pass_data", {})
@@ -4505,11 +3124,6 @@ class IdentityVerification:
                 if not data.get("gender") and data.get("gender_back"):
                     data["gender"] = data.get("gender_back")
 
-                # validation_result = self.validate_fields_id(data, country)
-                # if not validation_result:
-                #     tampering_result = 'consider'
-                #     data['tampering_result'] = tampering_result
-
             colour_picture = "consider"
             if data.get("front_coloured") and data.get("back_coloured"):
                 colour_picture = "clear"
@@ -4553,7 +3167,7 @@ class IdentityVerification:
                         selfie, front_face_locations, front_face_encodings
                     )
                 except Exception as e:
-                    print("issue in extracting face and computing similarity")
+                    logging.info("issue in extracting face and computing similarity")
                     selfie = None
                     similarity = 0
 
@@ -4561,8 +3175,6 @@ class IdentityVerification:
                 selfie = None
                 similarity = 0
 
-            # front_face_locations, front_face_encodings = data.get('front_face_locations'), data.get('front_face_encodings')
-            # processed_selfie = self.process_image(selfie)
             if country == "SAU" or data.get("doc_type") == "passport":
                 back_id_text = ""
             else:
@@ -4599,7 +3211,7 @@ class IdentityVerification:
                         back_img,
                     )
                 else:
-                    print(
+                    logging.info(
                         f"\nNationality not present, picking country: {country} for generating Document Report"
                     )
                     document_report = form_final_data_document_report(
@@ -4627,25 +3239,18 @@ class IdentityVerification:
                     if country == "IRQ" and nationality in ["SDN"]:
                         document_report["properties"].pop("passport_number_mrz", None)
 
-            if "facial_similarity_video" in report_names:
+            if 'facial_similarity_video' in report_names:
                 if video:
                     logging.info("--------------Checking for liveness-----------\n")
-                    liveness_result = self.check_for_liveness(
-                        similarity, video, face_match_threshold
-                    )
-                    logging.info(
-                    f"--------------Liveness Result from portal video: {liveness_result}\n"
-                )
+                    liveness_result = self.check_for_liveness(video)
+                    logging.info(f"--------------Liveness Result from portal video: {liveness_result}\n")
                     if country == "IRQ":
                         liveness_result = "clear"
-                    # print(f"LIVE RES: {liveness_result}")
                 else:
                     liveness_result = None
 
                 if nationality:
-                    print(
-                        f"\nNationality present, pikcing nationality: {nationality} for generating Video Report"
-                    )
+                    logging.info(f"Nationality present, picking nationality: {nationality} for generating Video Report")
                     facial_report = form_final_facial_similarity_report(
                         data,
                         selfie,
@@ -4655,9 +3260,7 @@ class IdentityVerification:
                         nationality,
                     )
                 else:
-                    print(
-                        f"\nNationality not present, pikcing country: {country} for generating Document Report"
-                    )
+                    logging.info(f"Nationality not present, picking country: {country} for generating Video Report")
                     facial_report = form_final_facial_similarity_report(
                         data,
                         selfie,
@@ -4667,10 +3270,7 @@ class IdentityVerification:
                         country,
                     )
 
-            logging.info(
-                f"--------------Time taken for Extract Ocr Function in IDV package: {time.time() - st} seconds\n"
-            )
-
+            logging.info(f"--------------Time taken for Extract Ocr Function in IDV package: {time.time() - st} seconds\n")
             return document_report, facial_report
 
         except Exception as e:
@@ -4678,15 +3278,13 @@ class IdentityVerification:
             raise Exception("Error occurred in extract_ocr_info")
 
     def generate_facial_report_portal(self, data, latest_portal_video):
-        from idvpackage.ocr_utils import form_final_facial_similarity_report
         from idvpackage.common import load_and_process_image_deepface_topup
+        from idvpackage.ocr_utils import form_final_facial_similarity_report
 
         try:
             if latest_portal_video:
                 logging.info("--------------Checking for liveness-----------\n")
-                latest_vid_liveness_result = self.check_for_liveness(
-                    0, latest_portal_video
-                )
+                latest_vid_liveness_result = self.check_for_liveness(latest_portal_video)
                 logging.info(
                     f"--------------Liveness Result from portal video: {latest_vid_liveness_result}\n"
                 )
@@ -4703,10 +3301,7 @@ class IdentityVerification:
             else:
                 latest_selfie = latest_selfie_str
 
-
-
             onboarding_face_encodings =  load_and_process_image_deepface_topup(initial_selfie)
-
             top_up_face_encodings = load_and_process_image_deepface_topup(latest_selfie)
 
             if not onboarding_face_encodings:
@@ -4719,13 +3314,9 @@ class IdentityVerification:
                 similarity_score = self.calculate_similarity(
                     onboarding_face_encodings[0], top_up_face_encodings[0]
                 )
-
                 similarity_score = min(1, similarity_score)
 
-                logging.info(
-                    f"--------------Face Similarity Computed: {similarity_score}\n"
-                )
-
+                logging.info(f"--------------Face Similarity Computed: {similarity_score}\n")
 
             facial_report = form_final_facial_similarity_report(
                 data,
@@ -4737,15 +3328,8 @@ class IdentityVerification:
             )
 
             logging.info(f"Facial Report: {facial_report}")
-
             return facial_report
 
         except Exception as e:
-            logging.info(
-                f"--------------Error occurred in generate_facial_report_portal: {e}\n"
-            )
-            raise Exception("Error occurred in generate_facial_report for portal")
-
-
-
-    
+            logging.info(f"--------------Error occurred in generate_facial_report_portal: {e}\n")
+            raise Exception("Error occurred in generate_facial_report for portal")