petal-qc 0.0.12__py3-none-any.whl → 0.0.15__py3-none-any.whl

petal_qc/test/createMetrologyFile.py

@@ -0,0 +1,77 @@
+#!/usr/bin/env python3
+"""Locates all raw data files."""
+
+import os
+import sys
+import fnmatch
+import re
+import shutil
+from pathlib import Path
+import argparse
+
+def all_files(root, patterns='*', single_level=False, yield_folders=False):
+    """A generator that reruns all files in the given folder.
+
+    Args:
+    ----
+        root (file path): The folder
+        patterns (str, optional): The pattern of the files. Defaults to '*'.
+        single_level (bool, optional): If true, do not go into sub folders. Defaults to False.
+        yield_folders (bool, optional): If True, return folders as well. Defaults to False.
+
+    Yields
+    ------
+        str: file path name
+
+    """
+    patterns = patterns.split(';')
+    for path, subdirs, files in os.walk(root):
+        if yield_folders:
+            files.extend(subdirs)
+
+        files.sort()
+        for name in files:
+            for pattern in patterns:
+                if fnmatch.fnmatch(name, pattern):
+                    yield os.path.join(path, name)
+                    break
+
+        if single_level:
+            break
+
+def main(options):
+    """Locate raw data files in input folder and copies them in out folder
+
+    Args:
+        folder_in (Path): Input folder
+        folder_oyt (Path): Output folder
+    """
+    reg = re.compile(r"(PPC\.[0-9]+)-(\w+)")
+    outF = Path(options.output).expanduser().resolve()
+    with open(outF, "w", encoding="utf-8") as fout:
+        for folder_in in options.files:
+            inF =  Path(folder_in).expanduser().resolve()
+            if not inF.exists():
+                print("Input folder does not exist. {}".format(inF))
+                continue
+
+            for fnam in all_files(inF, "PPC*.txt", yield_folders=False):
+                R = reg.search(fnam)
+                if R is None:
+                    continue
+                
+                petal_id = R.group(1)
+                side = R.group(2)
+                fout.write("{ifile} {pid}-{side} {pid}\n".format(ifile=fnam, side=side, pid=petal_id))
+        
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument('files', nargs='*', help="Input files")
+    parser.add_argument("--output", dest="output", help="Output file", default="out.txt")
+    opts = parser.parse_args()
+    if len(opts.files)==0:
+        print("I need at least one input folder")
+        sys.exit(-1)
+
+    main(opts)

petal_qc/test/desyModuleBow.py

@@ -0,0 +1,126 @@
+#!/usr/bin/env python3
+"""Get module bow from desy metrology files"""
+import sys
+import re
+from pathlib import Path
+import matplotlib.pyplot as plt
+import numpy as np
+import numpy.linalg as linalg
+
+
+from petal_qc.utils.all_files import all_files
+from petal_qc.metrology import DataFile
+from petal_qc.metrology import DataFile
+from petal_qc.utils.Geometry import fit_plane
+from petal_qc.utils.Geometry import project_to_plane
+from petal_qc.utils.Geometry import remove_outliers_indx
+
+
+rgx = "serialNo =\\s+([A-Z0-9]+)"
+serial_no = re.compile(rgx)
+
+mtype = re.compile("Project Name:\\s+([A-Za-z0-9]+)_bow")
+
+
+def get_array_range(A):
+    """Gets mean and range of given array."""
+    avg = np.mean(A)
+    imin = np.argmin(A)
+    vmin = A[imin]
+    imax = np.argmax(A)
+    vmax = A[imax]
+    stdv = np.std(A)
+    return avg, stdv, [vmin, vmax], [imin, imax]
+
+
+
+def create_bow_figure(SN, mtype, pout, width) -> plt.Figure:
+    """Create the sensor bow figure.
+
+    Args:
+        options: Program options.
+        pout: set of points
+        width: the actual bow.
+
+    Returns:
+        plt.Figure: The bow figure.
+    """
+    fig_bow, ax = plt.subplots(subplot_kw={'projection': '3d'})
+    fig_bow.suptitle(r"{}_{} - Sensor bow {:.1f} $\mu$m".format(SN, mtype, width))
+
+    zplt = 1000*pout[:, 2]
+    # surf = ax.plot_trisurf(pout[:, 0], pout[:, 1], zplt, cmap=plt.cm.jet, edgecolor="black", linewidths=0.2)
+    surf = ax.scatter(pout[:, 0], pout[:, 1], zplt, c=zplt, marker="o", edgecolor='none')
+    ax.set_xlabel("X")
+    ax.set_ylabel("Y")
+    ax.set_zlabel("Z")
+    cbar = fig_bow.colorbar(surf, shrink=0.5, aspect=5, location="left")
+    cbar.set_label(r"Z ($\mu$m)")
+    return fig_bow
+
+
+def module_bow(ifile):
+    """Compute module bow."""
+    SN = None
+    mod_type = None
+    print(Path(ifile).name)
+    with open(ifile, "r", encoding="utf-8") as fin:
+        ss = fin.read()
+        mtx = serial_no.search(ss)
+        if mtx:
+            SN = mtx.group(1)
+            
+        mtp = mtype.search(ss)
+        if mtp:
+            mod_type = mtp.group(1)
+
+    if SN is None:
+        return None, None, None
+    
+    print("{} - {}".format(SN, mod_type))
+    
+    data = DataFile.read(ifile, "(Bow|Sensor)")
+    if len(data) == 0:
+        return None, None, None
+    
+    # Try to remove locator points and get the plane
+    indx = remove_outliers_indx(data[:, 2])
+    M, TM, *_ = fit_plane(data[indx], use_average=False)
+
+    # project all data to the plane
+    Zmean = np.mean(M[:, 2])
+    M[:, 2] -= Zmean
+    
+    mean, stdev, rng, irng = get_array_range(M[:, 2])
+    width = 1000*(rng[1]-rng[0])
+    
+    # Check which one is closest to the center
+    min2center = linalg.norm(M[irng[0], 0:2])
+    max2center = linalg.norm(M[irng[1], 0:2])
+    # If center above the top, bow negative
+    if max2center < min2center:
+        width = -width
+
+    print("Sensor bow\nAverage: {:.1f} min {:.1f} max {:.1f} rng {:.1f}\n".format(
+        mean, 1000*rng[0], 1000*rng[1], width))
+
+    fig_bow = create_bow_figure(SN, mod_type, M, width)
+    fig_bow.savefig("{}-{}-bow.png".format(SN, mod_type), dpi=300)
+    plt.close(fig_bow)
+    del fig_bow
+    return SN, mod_type, width
+
+
+def main(folder):
+    """Main entry"""
+    fout = open("module-bow.txt", "w", encoding="utf-8")
+    for fnam in all_files(folder, "*.txt"):
+        SN, mod_type, width = module_bow(fnam)
+        if SN:
+            fout.write("{}-{} : {:.3f}\n".format(SN, mod_type, width))
+            
+    fout.close()
+
+if __name__ == "__main__":
+    main("/Users/lacasta/Downloads/InterposerPetal")
+    #main("/Users/lacasta/Downloads/kkdvk")

petal_qc/test/findRawData.py

@@ -0,0 +1,92 @@
+#!/usr/bin/env python3
+"""Locates all raw data files."""
+
+import os
+import sys
+import fnmatch
+import re
+import shutil
+from pathlib import Path
+import argparse
+
+def all_files(root, patterns='*', single_level=False, yield_folders=False):
+    """A generator that reruns all files in the given folder.
+
+    Args:
+    ----
+        root (file path): The folder
+        patterns (str, optional): The pattern of the files. Defaults to '*'.
+        single_level (bool, optional): If true, do not go into sub folders. Defaults to False.
+        yield_folders (bool, optional): If True, return folders as well. Defaults to False.
+
+    Yields
+    ------
+        str: file path name
+
+    """
+    patterns = patterns.split(';')
+    for path, subdirs, files in os.walk(root):
+        if yield_folders:
+            files.extend(subdirs)
+
+        files.sort()
+        for name in files:
+            for pattern in patterns:
+                if fnmatch.fnmatch(name, pattern):
+                    yield os.path.join(path, name)
+                    break
+
+        if single_level:
+            break
+
+def main(folder_in, folder_out):
+    """Locate raw data files in input folder and copies them in out folder
+
+    Args:
+        folder_in (Path): Input folder
+        folder_oyt (Path): Output folder
+    """
+    inF =  Path(folder_in).expanduser().resolve()
+    if not inF.exists():
+        print("Input folder does not exist. {}".format(inF))
+        return
+    
+    reg = re.compile(r"(PPC\.[0-9]+).*metr")
+    
+    outF = Path(folder_out).expanduser().resolve()
+    if not outF.exists():
+        os.makedirs(outF.as_posix())
+        
+    list_file = open("{}/raw-data-metrology-cores.txt".format(outF.as_posix()), "w", encoding="UTF-8")
+    for fnam in all_files(inF, "PPC*.txt", yield_folders=False):
+        R = reg.search(fnam)
+        if R is None:
+            continue
+        
+        petal_id = R.group(1)
+        if "back" in fnam:
+            side = "{}-back".format(petal_id)
+            ofile = "{}-back.txt".format(petal_id)
+        elif "front" in fnam:
+            side = "{}-front".format(petal_id)
+            ofile = "{}-front.txt".format(petal_id)
+        else:
+            print("Invalid file {}".format(fnam))
+            continue
+        
+        out_name = outF / ofile
+        list_file.write("{} {} {}\n".format(out_name.as_posix(), side, petal_id))
+        shutil.copy(fnam, out_name)
+        
+    list_file.close()
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--input-folder", dest="input", default=None, help="Input folder")
+    parser.add_argument("--output-folder", dest="output", help="Outout fodler", default=None)
+    opts = parser.parse_args()
+    if opts.input is None or opts.output is None:
+        print("I need both an input and an output folder.")
+        sys.exit(-1)
+
+    main(opts.input, opts.output)

petal_qc/test/getAVStests.py

@@ -1,18 +1,20 @@
 #!/usr/bin/env python3
 """Analize AVS metrology tests."""
+import sys
+from pathlib import Path
 import numpy as np
 import matplotlib.pyplot as plt
 
 try:
-    import itkdb_gtk
+    import petal_qc
 
 except ImportError:
-    import sys
-    from pathlib import Path
     cwd = Path(__file__).parent.parent
     sys.path.append(cwd.as_posix())
 
-from itkdb_gtk import ITkDBlogin, ITkDButils
+from itkdb_gtk import ITkDBlogin
+import petal_qc.utils.docx_utils as docx_utils
+
 
 
 def get_value(results, code):
@@ -108,24 +110,40 @@ def do_weighing(results, weights):
         weights.setdefault(value["code"][ipos+1:], []).append(value["value"])
 
 
-def plot_metrology(M_values, Mould_values, petal_ids):
+def plot_metrology(M_values, Mould_values, petal_ids, document):
     """Plot metrology values."""
+    fsize = np.zeros(2)
+
+    document.add_heading('Deviation from nominal positions', level=1)
     for key, values in M_values.items():
         fig, ax = plt.subplots(ncols=1, nrows=1, tight_layout=True)
         ax.hist(values, bins=15, range=(0, 0.150))
         ax.set_title(key)
-
+        ax.grid(True)
+        ax.set_xlabel("Distance (mm)")
+        fsize = fig.get_size_inches()
+        if key != "FIDUCIAL1":
+            document.add_picture(fig, True, 14, caption=key)
+
+    fsize[1] = fsize[0]/0.82
+    document.add_heading('Dependency with moulds', level=1)
     for obj in M_values.keys():
-        fig, ax = plt.subplots(ncols=1, nrows=4, tight_layout=True)
+        fig, ax = plt.subplots(ncols=1, nrows=4, tight_layout=True, figsize=fsize)
         fig.suptitle("{} - Mould".format(obj))
         for mould, m_values in Mould_values.items():
             im = int(mould) - 1
             ax[im].hist(m_values[obj], bins=15, range=(0, 0.150), label="Mould {}".format(mould))
 
         for i in range(4):
+            ax[i].grid(True)
+            ax[i].set_xlabel("Distance (mm)")
             ax[i].set_title("Mould {}".format(i+1))
 
-def plot_weighing(weights, tick_labels, show_total=False):
+        if obj != "FIDUCIAL1":
+            document.add_picture(fig, True, 14, caption=obj)
+
+
+def plot_weighing(weights, tick_labels, document, show_total=False):
     """Make the plot of weights."""
     labels = ["COOLINGLOOPASSEMBLY", "LOCATOR_A", "LOCATOR_B", "LOCATOR_C",
               "HONEYCOMBSET", "FACING_FRONT", "FACING_BACK",
@@ -169,13 +187,19 @@ def main(session):
     }
 
     core_list = session.get("listComponents", json=payload)
-    core_tests = ["METROLOGY_AVS", "WEIGHING", "MANUFACTURING"]
+    core_tests = ["METROLOGY_AVS", "WEIGHING", "MANUFACTURING", "DELAMINATION", "GROUNDING_CHECK", "VISUAL_INSPECTION"]
 
     weights = {}
     petal_ids = []
     M_values = {}
     Mould_values = {}
     mould_id = None
+    i = 0
+    bad_cores = {}
+    counter = {"TOTAL":0}
+    for T in core_tests:
+        counter[T]=0
+
     for core in core_list:
         SN = core["serialNumber"]
         altid = core['alternativeIdentifier']
@@ -222,6 +246,7 @@ def main(session):
         if mould_id not in Mould_values:
             Mould_values[mould_id] = {}
 
+        counter["TOTAL"] += 1
         for ttype, T in good_tests.items():
 
             if ttype == "WEIGHING":
@@ -233,20 +258,50 @@ def main(session):
             elif ttype == "MANUFACTURING":
                 continue
 
-            else:
-                for value in T["results"]:
-                    print("\t{} - {}".format(value["code"], value["value"]))
+            # else:
+            #     if T["results"]:
+            #         for value in T["results"]:
+            #             print("\t{} - {}".format(value["code"], value["value"]))
 
             if not T["passed"]:
                 print("## test {} FAILED".format(T["testType"]["code"]))
+                bad_cores.setdefault(altid, []).append({ttype: T["defects"]})
+
+            else:
+                try:
+                    counter[ttype] += 1
+                except KeyError:
+                    pass
 
             if len(T["defects"]):
                 print("+ Defects:")
                 for D in T["defects"]:
                     print("\t{} - {}".format(D["name"], D["description"]))
 
-    plot_weighing(weights, petal_ids)
-    plot_metrology(M_values, Mould_values, petal_ids)
+    document = docx_utils.Document()
+    document.add_page_numbers()
+    document.styles['Normal'].font.name = "Calibri"
+    document.add_heading("AVS QC tests.", 0)
+
+    document.add_heading('Results', level=1)
+    document.add_paragraph("Number of bad cores: {}.".format(len(bad_cores)))
+    document.add_heading("Bad cores", level=2)
+    for key, lst in bad_cores.items():
+        p = document.add_paragraph()
+        bf = p.add_run("{}:".format(key))
+        bf.bold = True
+        bf.italic = True
+        for item in lst:
+            for ttype, defects in item.items():
+                msg = "{}:".format(ttype)
+                for  D in defects:
+                    msg += "\r{} - {}".format(D["name"], D["description"])
+                document.add_paragraph(msg, style="List Bullet")
+
+
+    plot_weighing(weights, petal_ids, document)
+    plot_metrology(M_values, Mould_values, petal_ids, document)
+    document.save("AVStests.docx")
     plt.show()
 
 if __name__ == "__main__":
@@ -254,10 +309,6 @@ if __name__ == "__main__":
     dlg = ITkDBlogin.ITkDBlogin()
     session = dlg.get_client()
 
-    try:
-        main(session)
-
-    except Exception as E:
-        print(E)
+    main(session)
 
     dlg.die()

petal_qc/test/getPetalCoreTestSummary.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 """Get a summery of Petal core TEsts."""
-
+import json
 try:
     import itkdb_gtk
     
@@ -14,7 +14,7 @@ from itkdb_gtk import ITkDBlogin, ITkDButils
 from itkdb_gtk.dbGtkUtils import replace_in_container, DictDialog, ask_for_confirmation
 
 
-def main(session):
+def main(session, options):
     """Main entry point."""
 
     # find all cores
@@ -24,17 +24,23 @@ def main(session):
             #"componentType": ["BT"],
             "componentType": ["CORE_PETAL"],
             "type": ["CORE_AVS"],
-            "currentLocation": ["IFIC"],
+            # "currentLocation": ["IFIC"],
         },
         "sorterList": [
             {"key": "alternativeIdentifier", "descending": False }
         ],
     }
+    suff = "ALL"
+    if options.institute:
+        payload["filterMap"]["currentLocation"] = options.institute
+        suff = options.institute
+        
     core_list = session.get("listComponents", json=payload)
     core_tests = ["PETAL_METROLOGY_FRONT", "PETAL_METROLOGY_BACK", "XRAYIMAGING", "THERMAL_EVALUATION", "BTTESTING"]
     
     do_check_stage = "AT_QC_SITE"
     #do_check_stage = None
+    petal_id_db = {}
     
     for core in core_list:
         SN = core["serialNumber"]
@@ -42,7 +48,7 @@ def main(session):
         if "PPC" not in altid:
             continue
         
-
+        petal_id_db[altid] = SN
         location = core["currentLocation"]['code']
         coreStage = core["currentStage"]['code']
         if do_check_stage:
@@ -73,14 +79,21 @@ def main(session):
             for D in T["defects"]:
                 print("\t{} - {}".format(D["name"], D["description"]))
 
+    with open("petal_ID_db_{}.json".format(suff), "w", encoding="utf-8") as fOut:
+        json.dump(petal_id_db, fOut, indent=3)
 
 if __name__ == "__main__":
+    from argparse import ArgumentParser
+    parser = ArgumentParser()
+    parser.add_argument("--institute", default=None, help="The petal current location")
+    options = parser.parse_args()
+    
     # ITk_PB authentication
     dlg = ITkDBlogin.ITkDBlogin()
     session = dlg.get_client()
 
     try:
-        main(session)
+        main(session, options)
 
     except Exception as E:
         print(E)

petal_qc/test/prepareDESYfiles.py

@@ -1,8 +1,10 @@
 #!/usr/bin/env python3
 """Prepare raw data input files from DESY so that we can use them with this code."""
 import os
+import sys
 import fnmatch
 import re
+import shutil
 from pathlib import Path
 import argparse
 
@@ -51,17 +53,26 @@ def main(folder, out_folder):
         folder: the input folder where to find the original files.
         out_folder: the folder where the new fiels will be stored.
     """
+    inF =  Path(folder).expanduser().resolve()
+    if not inF.exists():
+        print("Input folder does not exist. {}".format(inF))
+        return
+    
+    print("Reading folder: {}".format(inF))
+    print("output in {}".format(out_folder))
     outF = Path(out_folder).expanduser().resolve()
     if not outF.exists():
+        print("creating {}".format(outF))
         os.mkdir(outF)
 
     rgx = re.compile(r"Project Name: (\w+)side_.*AlternativeID=PPC[-_](\d+)", re.MULTILINE|re.DOTALL)
     petal_cores = {}
-    for fnam in all_files(folder, "*.txt"):
+    for fnam in all_files(inF, "*.txt"):
         P = Path(fnam).expanduser().resolve()
+        print(P.name)
         with open(fnam, "r", encoding="UTF-8") as ff:
             R = rgx.search(ff.read())
-            if R: 
+            if R:
                 petal_id = "PPC.{}".format(R.group(2))
                 side = R.group(1).lower()
                 if "_2D_" in P.name:
@@ -86,7 +97,7 @@ def main(folder, out_folder):
             for data_type, fnam in values.items():
                 if fnam is None:
                     print("This should not happen.")
-                    
+
                 with open(fnam, "r", encoding="UTF-8") as ifile:
                     data += ifile.read()
 
@@ -94,10 +105,14 @@ def main(folder, out_folder):
                 ofile.write(data)
 
     list_file.close()
-    
+
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
-    parser.add_argument("--input-folder", default=None, help="Input folder")
-    parser.add_argument("--output-folder", help="Outout fodler", default=None)
-    
-    main("/Users/lacasta/Downloads/Report_QC-desy_Sep2024", "/tmp/desy")
+    parser.add_argument("--input-folder", dest="input", default=None, help="Input folder")
+    parser.add_argument("--output-folder", dest="output", help="Outout fodler", default=None)
+    opts = parser.parse_args()
+    if opts.input is None or opts.output is None:
+        print("I need both an input and an output folder.")
+        sys.exit(-1)
+
+    main(opts.input, opts.output)

petal_qc/test/reportFromJSon.py

@@ -0,0 +1,51 @@
+#!/usr/bin/env python3
+"""List failing cores from JSon files."""
+import sys
+import argparse
+from pathlib import Path
+import json
+
+
+
+def main(options):
+    """main entry."""
+    petal_cores = {}
+    for fnam in options.files:
+        ifile = Path(fnam).expanduser().resolve()
+        with open(ifile, "r", encoding="utf-8") as fin:
+            data = json.load(fin)
+
+        if not data["passed"]:
+            petalId = data["component"]
+            if petalId not in petal_cores:
+                petal_cores[petalId] = {"FRONT": [], "BACK": []}
+
+            side = "FRONT" if "FRONT" in data["testType"] else "BACK"
+            for D in data["defects"]:
+                petal_cores[petalId][side].append("{}: {}".format(D["name"], D["description"]))
+
+
+    keys = sorted(petal_cores.keys())
+    for petalId in keys:
+        print(petalId)
+        for side in ["FRONT","BACK"]:
+            if len(petal_cores[petalId][side])>0:
+                print("+-", side)
+            for D in petal_cores[petalId][side]:
+                print("  ", D)
+
+        print("\n")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument('files', nargs='*', help="Input files")
+    opts = parser.parse_args()
+    
+    from petal_qc.utils.all_files import all_files
+    
+    opts.files = []
+    for fnam in all_files(Path("~/tmp/petal-metrology/Production/Results").expanduser(), "*.json"):
+        opts.files.append(fnam)
+    
+    main(opts)

petal_qc/thermal/create_core_report.py

@@ -39,12 +39,19 @@ def create_report(options):
 
     print("\n## {} - {}".format(options.SN, options.alias))
 
+    # Do the core analysis.
+    core = create_IR_core(options)
+
+    if options.golden is None:
+        return core
+    
     goldenFile = Path(options.golden).expanduser().resolve()
     if not goldenFile.exists():
         goldenFile = utils.output_folder(options.folder, options.golden)
         goldenFile = Path(goldenFile).expanduser().resolve()
         if not goldenFile.exists():
             print("I need a golden file.")
+            return core
 
     with open(goldenFile, "r", encoding='utf-8') as fp:
         golden = golden_from_json(json.load(fp))
@@ -65,9 +72,6 @@ def create_report(options):
     P = document.add_paragraph("Golden: {}".format(goldenFile.name), "Subtitle")
     P.alignment = docx_utils.paragraph_align_center()
 
-
-    core = create_IR_core(options)
-
     figures = get_IRcore_plots()
     document.add_heading('Original image', level=1)
     document.add_picture(figures["original"], True, 14, caption="Original Thermal image.")