Semapp 1.0.5__py3-none-any.whl

semapp/Processing/klarf_reader.py

@@ -0,0 +1,461 @@
+"""
+Module for reading and parsing KLARF files (.001) to extract defect positions.
+Supports KRONOS, COMPLUS4T, and normal modes.
+"""
+
+import os
+import re
+import pandas as pd
+
+
+def extract_positions(filepath, wafer_id=None):
+    """
+    Extract defect positions from KLARF file.
+    Main function that detects the mode and calls the appropriate parser.
+    
+    Args:
+        filepath: Path to the KLARF (.001) file
+        wafer_id: Specific wafer ID to extract (for COMPLUS4T files with multiple wafers)
+                 If None, extracts all defects (normal mode)
+    
+    Returns:
+        pd.DataFrame: DataFrame with columns ["defect_id", "X", "Y", "defect_size"]
+    """
+    # Detect mode by reading the file
+    mode = _detect_mode(filepath)
+    
+    if mode == "KRONOS":
+        return _extract_positions_kronos(filepath)
+    elif mode == "COMPLUS4T":
+        return _extract_positions_complus4t(filepath, wafer_id)
+    else:
+        return _extract_positions_normal(filepath)
+
+
+def _detect_mode(filepath):
+    """
+    Detect the mode of the KLARF file (KRONOS, COMPLUS4T, or normal).
+    
+    Args:
+        filepath: Path to the KLARF (.001) file
+    
+    Returns:
+        str: "KRONOS", "COMPLUS4T", or "NORMAL"
+    """
+    try:
+        with open(filepath, "r", encoding="utf-8", errors="ignore") as f:
+            for i, line in enumerate(f):
+                if i >= 20:  # Only check first 20 lines
+                    break
+                line_stripped = line.strip()
+                
+                # Check for KRONOS format
+                if line_stripped.startswith("WaferID"):
+                    if re.search(r'WaferID\s+"Read Failed\.(\d+)"', line_stripped):
+                        return "KRONOS"
+                    if re.search(r'WaferID\s+"@(\d+)"', line_stripped):
+                        return "COMPLUS4T"
+                
+                # Check for COMPLUS4T keyword
+                if 'COMPLUS4T' in line_stripped:
+                    return "COMPLUS4T"
+                
+                # Check for KRONOS keyword
+                if 'KRONOS' in line_stripped:
+                    return "KRONOS"
+    except Exception:
+        pass
+    
+    return "NORMAL"
+
+
+def _extract_positions_kronos(filepath):
+    """
+    Extract defect positions from KRONOS format KLARF file.
+    
+    Args:
+        filepath: Path to the KLARF (.001) file
+    
+    Returns:
+        pd.DataFrame: DataFrame with columns ["defect_id", "X", "Y", "defect_size"]
+    """
+    data = {
+        "SampleSize": None,
+        "DiePitch": {"X": None, "Y": None},
+        "DieOrigin": {"X": None, "Y": None},
+        "SampleCenterLocation": {"X": None, "Y": None},
+        "Defects": []
+    }
+    
+    dans_defect_list = False
+    detected_numbers = set()
+    kronos_wafer_id = None
+    
+    with open(filepath, "r", encoding="utf-8") as f:
+        lines = f.readlines()
+    
+    # First pass: detect KRONOS and extract wafer ID, load detection CSV
+    for line in lines:
+        line_stripped = line.strip()
+        if line_stripped.startswith("WaferID"):
+            kronos_match = re.search(r'WaferID\s+"Read Failed\.(\d+)"', line_stripped)
+            if kronos_match:
+                kronos_wafer_id = int(kronos_match.group(1))
+                print(f"[KRONOS] Detected wafer ID: {kronos_wafer_id}")
+                
+                # Load detection_results.csv from the wafer subdirectory
+                file_dir = os.path.dirname(filepath)
+                file_dir_basename = os.path.basename(file_dir)
+                if file_dir_basename == str(kronos_wafer_id):
+                    detection_csv_path = os.path.join(file_dir, "detection_results.csv")
+                else:
+                    detection_csv_path = os.path.join(file_dir, str(kronos_wafer_id), "detection_results.csv")
+                
+                if os.path.exists(detection_csv_path):
+                    print(f"[KRONOS] Loading detection results from: {detection_csv_path}")
+                    try:
+                        detection_df = pd.read_csv(detection_csv_path)
+                        detected_numbers = set()
+                        for num_str in detection_df['Detected_Number']:
+                            if num_str and str(num_str) != 'None' and str(num_str) != 'nan':
+                                try:
+                                    num = int(float(str(num_str).strip()))
+                                    detected_numbers.add(num)
+                                except (ValueError, TypeError):
+                                    pass
+                        print(f"[KRONOS] Found {len(detected_numbers)} unique detected numbers: {sorted(detected_numbers)}")
+                    except Exception as e:
+                        print(f"[KRONOS] Error loading detection CSV: {e}")
+                else:
+                    print(f"[KRONOS] Warning: detection_results.csv not found at {detection_csv_path}")
+                break
+    
+    # Second pass: parse the file
+    for i, line in enumerate(lines):
+        line = line.strip()
+        
+        if line.startswith("WaferID"):
+            continue
+        
+        if line.startswith("SampleSize"):
+            match = re.search(r"SampleSize\s+1\s+(\d+)", line)
+            if match:
+                data["SampleSize"] = int(match.group(1))
+        
+        elif line.startswith("DiePitch"):
+            match = re.search(r"DiePitch\s+([0-9.e+-]+)\s+([0-9.e+-]+);", line)
+            if match:
+                data["DiePitch"]["X"] = float(match.group(1))
+                data["DiePitch"]["Y"] = float(match.group(2))
+        
+        elif line.startswith("DieOrigin"):
+            match = re.search(r"DieOrigin\s+([0-9.e+-]+)\s+([0-9.e+-]+);", line)
+            if match:
+                data["DieOrigin"]["X"] = float(match.group(1))
+                data["DieOrigin"]["Y"] = float(match.group(2))
+        
+        elif line.startswith("SampleCenterLocation"):
+            match = re.search(r"SampleCenterLocation\s+([0-9.e+-]+)\s+([0-9.e+-]+);", line)
+            if match:
+                data["SampleCenterLocation"]["X"] = float(match.group(1))
+                data["SampleCenterLocation"]["Y"] = float(match.group(2))
+        
+        elif line.startswith("DefectList"):
+            dans_defect_list = True
+            continue
+        
+        elif dans_defect_list:
+            if re.match(r"^\d+\s", line):
+                value = line.split()
+                if len(value) >= 12:
+                    real_defect_id = int(value[0])
+                    
+                    # Only add defect if its ID is in detected_numbers
+                    if real_defect_id not in detected_numbers:
+                        continue
+                    
+                    defect = {f"val{i+1}": float(val) for i, val in enumerate(value[:11])}
+                    defect["defect_id"] = real_defect_id
+                    data["Defects"].append(defect)
+            elif line.startswith("EndOfFile") or line.startswith("}"):
+                dans_defect_list = False
+    
+    return _calculate_coordinates(data, is_kronos=True, filepath=filepath, kronos_wafer_id=kronos_wafer_id)
+
+
+def _extract_positions_complus4t(filepath, wafer_id=None):
+    """
+    Extract defect positions from COMPLUS4T format KLARF file.
+    
+    Args:
+        filepath: Path to the KLARF (.001) file
+        wafer_id: Specific wafer ID to extract (required for COMPLUS4T)
+    
+    Returns:
+        pd.DataFrame: DataFrame with columns ["defect_id", "X", "Y", "defect_size"]
+    """
+    data = {
+        "SampleSize": None,
+        "DiePitch": {"X": None, "Y": None},
+        "DieOrigin": {"X": None, "Y": None},
+        "SampleCenterLocation": {"X": None, "Y": None},
+        "Defects": []
+    }
+    
+    dans_defect_list = False
+    current_wafer_id = None
+    target_wafer_found = False
+    reading_target_wafer = False
+    
+    with open(filepath, "r", encoding="utf-8") as f:
+        lines = f.readlines()
+    
+    for i, line in enumerate(lines):
+        line = line.strip()
+        
+        # Detect WaferID
+        if line.startswith("WaferID"):
+            match = re.search(r'WaferID\s+"@(\d+)"', line)
+            if match:
+                current_wafer_id = int(match.group(1))
+                
+                # If looking for a specific wafer
+                if wafer_id is not None:
+                    if current_wafer_id == wafer_id:
+                        target_wafer_found = True
+                        reading_target_wafer = True
+                        data["Defects"] = []
+                    elif target_wafer_found:
+                        break
+                    else:
+                        reading_target_wafer = False
+                else:
+                    reading_target_wafer = True
+            continue
+        
+        # If looking for specific wafer, skip lines until finding the right wafer
+        if wafer_id is not None and not reading_target_wafer and not line.startswith("DefectList"):
+            if current_wafer_id is None:
+                pass
+            else:
+                continue
+        
+        if line.startswith("SampleSize"):
+            match = re.search(r"SampleSize\s+1\s+(\d+)", line)
+            if match:
+                data["SampleSize"] = int(match.group(1))
+        
+        elif line.startswith("DiePitch"):
+            match = re.search(r"DiePitch\s+([0-9.e+-]+)\s+([0-9.e+-]+);", line)
+            if match:
+                data["DiePitch"]["X"] = float(match.group(1))
+                data["DiePitch"]["Y"] = float(match.group(2))
+        
+        elif line.startswith("DieOrigin"):
+            match = re.search(r"DieOrigin\s+([0-9.e+-]+)\s+([0-9.e+-]+);", line)
+            if match:
+                data["DieOrigin"]["X"] = float(match.group(1))
+                data["DieOrigin"]["Y"] = float(match.group(2))
+        
+        elif line.startswith("SampleCenterLocation"):
+            match = re.search(r"SampleCenterLocation\s+([0-9.e+-]+)\s+([0-9.e+-]+);", line)
+            if match:
+                data["SampleCenterLocation"]["X"] = float(match.group(1))
+                data["SampleCenterLocation"]["Y"] = float(match.group(2))
+        
+        elif line.startswith("DefectList"):
+            dans_defect_list = True
+            continue
+        
+        elif dans_defect_list:
+            # If in DefectList, filter by wafer if necessary
+            if wafer_id is not None and not reading_target_wafer:
+                if line.startswith("EndOfFile") or line.startswith("}"):
+                    dans_defect_list = False
+                continue
+            
+            if re.match(r"^\d+\s", line):
+                value = line.split()
+                if len(value) >= 12:
+                    # For COMPLUS4T: Check if next line has exactly 2 columns
+                    if i + 1 < len(lines):
+                        next_line = lines[i + 1].strip()
+                        next_values = next_line.split()
+                        if len(next_values) == 2:
+                            real_defect_id = int(next_values[0])
+                            defect = {f"val{i+1}": float(val) for i, val in enumerate(value[:10])}
+                            defect["defect_id"] = real_defect_id
+                            data["Defects"].append(defect)
+            elif line.startswith("EndOfFile") or line.startswith("}"):
+                dans_defect_list = False
+    
+    return _calculate_coordinates(data, is_kronos=False, filepath=filepath, wafer_id=wafer_id)
+
+
+def _extract_positions_normal(filepath):
+    """
+    Extract defect positions from normal format KLARF file.
+    
+    Args:
+        filepath: Path to the KLARF (.001) file
+    
+    Returns:
+        pd.DataFrame: DataFrame with columns ["defect_id", "X", "Y", "defect_size"]
+    """
+    data = {
+        "SampleSize": None,
+        "DiePitch": {"X": None, "Y": None},
+        "DieOrigin": {"X": None, "Y": None},
+        "SampleCenterLocation": {"X": None, "Y": None},
+        "Defects": []
+    }
+    
+    dans_defect_list = False
+    
+    with open(filepath, "r", encoding="utf-8") as f:
+        lines = f.readlines()
+    
+    for i, line in enumerate(lines):
+        line = line.strip()
+        
+        if line.startswith("SampleSize"):
+            match = re.search(r"SampleSize\s+1\s+(\d+)", line)
+            if match:
+                data["SampleSize"] = int(match.group(1))
+        
+        elif line.startswith("DiePitch"):
+            match = re.search(r"DiePitch\s+([0-9.e+-]+)\s+([0-9.e+-]+);", line)
+            if match:
+                data["DiePitch"]["X"] = float(match.group(1))
+                data["DiePitch"]["Y"] = float(match.group(2))
+        
+        elif line.startswith("DieOrigin"):
+            match = re.search(r"DieOrigin\s+([0-9.e+-]+)\s+([0-9.e+-]+);", line)
+            if match:
+                data["DieOrigin"]["X"] = float(match.group(1))
+                data["DieOrigin"]["Y"] = float(match.group(2))
+        
+        elif line.startswith("SampleCenterLocation"):
+            match = re.search(r"SampleCenterLocation\s+([0-9.e+-]+)\s+([0-9.e+-]+);", line)
+            if match:
+                data["SampleCenterLocation"]["X"] = float(match.group(1))
+                data["SampleCenterLocation"]["Y"] = float(match.group(2))
+        
+        elif line.startswith("DefectList"):
+            dans_defect_list = True
+            continue
+        
+        elif dans_defect_list:
+            if re.match(r"^\d+\s", line):
+                value = line.split()
+                if len(value) >= 12:
+                    # For Normal: Check if next line has exactly 2 columns
+                    if i + 1 < len(lines):
+                        next_line = lines[i + 1].strip()
+                        next_values = next_line.split()
+                        if len(next_values) == 2:
+                            real_defect_id = int(next_values[0])
+                            defect = {f"val{i+1}": float(val) for i, val in enumerate(value[:10])}
+                            defect["defect_id"] = real_defect_id
+                            data["Defects"].append(defect)
+            elif line.startswith("EndOfFile") or line.startswith("}"):
+                dans_defect_list = False
+    
+    return _calculate_coordinates(data, is_kronos=False, filepath=filepath, wafer_id=None)
+
+
+def _calculate_coordinates(data, is_kronos, filepath, wafer_id=None, kronos_wafer_id=None):
+    """
+    Calculate corrected coordinates from parsed defect data.
+    
+    Args:
+        data: Dictionary containing parsed KLARF data
+        is_kronos: Boolean indicating if this is KRONOS mode
+        filepath: Path to the KLARF file (for saving CSV)
+        wafer_id: Wafer ID for COMPLUS4T mode
+        kronos_wafer_id: Wafer ID for KRONOS mode
+    
+    Returns:
+        pd.DataFrame: DataFrame with columns ["defect_id", "X", "Y", "defect_size"]
+    """
+    pitch_x = data["DiePitch"]["X"]
+    pitch_y = data["DiePitch"]["Y"]
+    Xcenter = data["SampleCenterLocation"]["X"]
+    Ycenter = data["SampleCenterLocation"]["Y"]
+    
+    # Check if required values are not None
+    if pitch_x is None or pitch_y is None or Xcenter is None or Ycenter is None:
+        print(f"Warning: Missing required values in KLARF file. pitch_x={pitch_x}, pitch_y={pitch_y}, x_center={Xcenter}, y_center={Ycenter}")
+        return pd.DataFrame(columns=["defect_id", "X", "Y", "defect_size"])
+    
+    corrected_positions = []
+    for d in data["Defects"]:
+        real_defect_id = d["defect_id"]
+        
+        # For KRONOS: shift columns by 1
+        if is_kronos:
+            val2 = d["val4"]
+            val3 = d["val5"]
+            val4_scaled = d["val6"] * pitch_x - Xcenter
+            val5_scaled = d["val7"] * pitch_y - Ycenter
+            defect_size = d["val11"]
+        else:
+            # COMPLUS4T/Normal mode: use original columns
+            val2 = d["val2"]
+            val3 = d["val3"]
+            val4_scaled = d["val4"] * pitch_x - Xcenter
+            val5_scaled = d["val5"] * pitch_y - Ycenter
+            defect_size = d["val9"]
+        
+        x_corr = round((val2 + val4_scaled) / 10000, 1)
+        y_corr = round((val3 + val5_scaled) / 10000, 1)
+        
+        corrected_positions.append({
+            "defect_id": real_defect_id,
+            "X": x_corr,
+            "Y": y_corr,
+            "defect_size": defect_size
+        })
+    
+    coordinates = pd.DataFrame(corrected_positions, columns=["defect_id", "X", "Y", "defect_size"])
+    
+    # Print coordinates for debugging
+    print("\n" + "="*80)
+    print("COORDINATES EXTRACTED:")
+    print("="*80)
+    print(f"Total defects: {len(coordinates)}")
+    if len(coordinates) > 0:
+        print("\nCoordinates DataFrame:")
+        print(coordinates.to_string())
+    else:
+        print("No defects found.")
+    print("="*80 + "\n")
+    
+    # Save mapping to CSV
+    file_dir = os.path.dirname(filepath)
+    
+    # If KRONOS mode, save to wafer subfolder
+    if is_kronos and kronos_wafer_id is not None:
+        file_dir_basename = os.path.basename(file_dir)
+        if file_dir_basename == str(kronos_wafer_id):
+            csv_folder = file_dir
+        else:
+            csv_folder = os.path.join(file_dir, str(kronos_wafer_id))
+        os.makedirs(csv_folder, exist_ok=True)
+        csv_path = os.path.join(csv_folder, "mapping.csv")
+        print(f"[KRONOS] Saving mapping to: {csv_path}")
+    # If wafer_id is specified (COMPLUS4T mode), save to wafer subfolder
+    elif wafer_id is not None:
+        csv_folder = os.path.join(file_dir, str(wafer_id))
+        os.makedirs(csv_folder, exist_ok=True)
+        csv_path = os.path.join(csv_folder, "mapping.csv")
+    else:
+        # Normal mode: save in same folder as .001 file
+        csv_path = os.path.join(file_dir, "mapping.csv")
+    
+    coordinates.to_csv(csv_path, index=False)
+    
+    if is_kronos:
+        print(f"[KRONOS] Extracted {len(coordinates)} defects (filtered by detected numbers)")
+    
+    return coordinates