petal-qc 0.0.20__py3-none-any.whl → 0.0.22__py3-none-any.whl

petal_qc/readTemplateTable.py

@@ -0,0 +1,313 @@
+#!/usr/bin/env python3
+"""Read Python table with reception tests."""
+
+import sys
+import math
+import argparse
+import pandas as pd
+import json
+from pathlib import Path
+
+from itkdb_gtk import ITkDBlogin, ITkDButils
+
+def check_value(value):
+    """Check that value is valid."""
+    if value is None:
+        return False
+
+    if isinstance(value, float):
+        if math.isnan(value):
+            return False
+
+    return True
+
+
+def string_to_boolean(s):
+    """Converts a string to a boolean."""
+    val = s.lower()
+    if val=="pass" or val=="true" or val=="1":
+        return True
+    elif val=="fail" or val=="false" or val=="0":
+        return False
+    else:
+        raise ValueError
+
+def parse_value(value):
+    """Parse value in cell."""
+    try:
+        spassed, comment = value.split(",")
+    except ValueError:
+        spassed = value
+        comment = ""
+
+    passed = string_to_boolean(spassed)
+    if passed is None:
+        return None, None
+
+    return passed, comment
+
+def create_dto(session, SN, test_name):
+    """REturns the test DTO."""
+    user =  ITkDButils.get_db_user(session)
+    defaults = {
+            "component": SN,
+            "institution": user["institutions"][0]["code"],
+            "runNumber": "1",
+        }
+    dto = ITkDButils.get_test_skeleton(session, "CORE_PETAL", test_name, defaults)
+    return dto
+
+def do_visual_inspection(session, core, value):
+    """Uploads visual inspection."""
+    if not check_value(value):
+       return None
+
+    SN = core["serialNumber"]
+    dto = create_dto(session, SN, "VISUAL_INSPECTION")
+
+    passed, comment = parse_value(value)
+
+    if passed is None:
+        print("Wrong value fpr PASS/FAIL {} in {} for {}".format(value, "VISUAL_INSPECTION", core["alternativeIdentifier"]) )
+        return None
+
+    if passed:
+        dto["passed"] = True
+    else:
+        dto["passed"] = False
+        dto["defects"].append({
+            "name": "VISUAL",
+            "description": comment,
+            "properties": {}
+        })
+
+    return dto
+
+def do_grounding(session, core, value):
+    """Uploads grounding check"""
+    if not check_value(value):
+        return None
+
+    SN = core["serialNumber"]
+    dto = create_dto(session, SN, "GROUNDING_CHECK")
+
+    fb, pipes, pipe_gnd = [float(x) for x in value.split(',')]
+    if fb > 2.0:
+        dto["passed"] = False
+        dto["defects"].append({
+            "name": "GROUND_FB",
+            "description": "resistance front-back is {} > 2 Ohm".format(fb),
+            "properties": {}
+        })
+
+    if pipes>0 and pipes<20.0e6:
+        dto["passed"] = False
+        dto["defects"].append({
+            "name": "GROUND_PIPES",
+            "description": "resistance between pipes is {} < 20 MOhm".format(pipes),
+            "properties": {}
+        })
+
+    if pipe_gnd>0 and pipe_gnd<20.0e6:
+        dto["passed"] = False
+        dto["defects"].append({
+            "name": "GROUND_PIPE_GND",
+            "description": "resistance between pipes and GNDis {} < 20 MOhm".format(pipe_gnd),
+            "properties": {}
+        })
+
+    dto["results"]["RESISTANCE_FB"] = fb
+    dto["results"]["RESISTANCE_PIPES"] = pipes
+    dto["results"]["RESISTANCE_PIPE_GND"] = pipe_gnd
+
+    return dto
+
+def do_bending(session, core, value):
+    """Uploads bending."""
+    if not check_value(value):
+        return None
+
+    SN = core["serialNumber"]
+    dto = create_dto(session, SN, "BENDING120")
+    passed, comment = parse_value(value)
+    if passed is None:
+        print("Wrong value fpr PASS/FAIL {} in {} for {}".format(value, "BENDING120", core["alternativeIdentifier"]) )
+        return None
+
+    if passed:
+        dto["passed"] = True
+    else:
+        dto["passed"] = False
+        dto["defects"].append({
+            "name": "BENDING120",
+            "description": comment,
+            "properties": {}
+        })
+
+    return dto
+
+def do_weight(session, core, value):
+    """Uploads weight."""
+    if not check_value(value):
+        return None
+
+    SN = core["serialNumber"]
+    dto = create_dto(session, SN, "PETAL_CORE_WEIGHT")
+
+    weight = float(value)
+    dto["results"]["WEIGHT"] = weight
+    if abs(weight-250)>25:
+        dto["passed"] = False
+        dto["defects"].append({
+            "name": "WEIGHT",
+            "description": "Petal core wights {:.1f} more than 25 gr. beyond 250.".format(weight),
+            "properties": {}
+        })
+    else:
+        dto["passed"] = True
+
+    return dto
+
+
+def do_thickness(session, core, value):
+    """Uploads thickness."""
+    if not check_value(value):
+        return None
+
+    SN = core["serialNumber"]
+    dto = create_dto(session, SN, "CORE_THICKNESS")
+    thickness = float(value)
+    dto["results"]["THICKNESS"] = thickness
+
+    dto["passed"] = True
+    if abs(thickness-5.9)>0.25:
+        dto["problems"] = True
+        dto["comments"].append("Petal core wights {:.1f} more than 25 gr. beyond 250.".format(thickness))
+
+    return dto
+
+
+def do_metrology_template(session, core, value):
+    """Uploads metrology template."""
+    if not check_value(value):
+        return None
+
+    SN = core["serialNumber"]
+    dto = create_dto(session, SN, "METROLOGY_TEMPLATE")
+    passed, comment = parse_value(value)
+    if passed is None:
+        print("Wrong value fpr PASS/FAIL {} in {} for {}".format(value, "TEMPLATE", core["alternativeIdentifier"]) )
+        return None
+
+    if passed:
+        dto["passed"] = True
+    else:
+        dto["passed"] = False
+        dto["defects"].append({
+            "name": "TEMPLATE",
+            "description": comment,
+            "properties": {}
+        })
+
+    return dto
+
+
+def do_delamination(session, core, value):
+    """Uploads delamination."""
+    if not check_value(value):
+        return None
+
+    SN = core["serialNumber"]
+    dto = create_dto(session, SN, "DELAMINATION")
+
+    passed, comment = parse_value(value)
+    if passed is None:
+        print("Wrong value fpr PASS/FAIL {} in {} for {}".format(value, "DELAMINATION", core["alternativeIdentifier"]) )
+        return None
+
+    if passed:
+        dto["passed"] = True
+    else:
+        dto["passed"] = False
+        dto["defects"].append({
+            "name": "DELAMINATION",
+            "description": comment,
+            "properties": {}
+        })
+
+    return dto
+
+
+def readTemplateTable(session, options):
+    """Main entry.
+
+    Args:
+        session (itkdb.Session): The PDB client.
+        options: program options
+    """
+
+    core_tests = {
+        "VISUAL_INSPECTION": do_visual_inspection,
+        "GROUNDING_CHECK": do_grounding,
+        "BENDING120": do_bending,
+        "PETAL_CORE_WEIGHT": do_weight,
+        "CORE_THICKNESS": do_thickness,
+        "METROLOGY_TEMPLATE": do_metrology_template,
+        "DELAMINATION": do_delamination
+    }
+    try:
+        sheet = int(options.sheet)
+    except ValueError:
+        sheet = options.sheet
+        
+    df = pd.read_excel(options.files[0], sheet_name=sheet)
+    #print(df)
+
+    for row in df.itertuples():
+        petal_id = row.CORE_ID
+        if not check_value(petal_id):
+            break
+
+        print("\n\n### {}".format(petal_id))
+        core = ITkDButils.get_DB_component(session, petal_id)
+        for test, func in core_tests.items():
+            data = func(session, core, getattr(row, test))
+            if data is None:
+                continue
+
+            data["properties"]["OPERATOR"] = options.operator
+            
+            #print(json.dumps(data, indent=3))
+            rc = ITkDButils.upload_test(session, data, check_runNumber=True)
+            if rc:
+                print("\n*** Could not upload test {} for {}".format(test, petal_id))
+                print(rc)
+                print()
+
+def main():
+    """Entry point."""
+    parser = argparse.ArgumentParser()
+    parser.add_argument("files", nargs='*', help="The template spreadsheet")
+    parser.add_argument("--sheet", default="0", help="Sheet to read from excel file.")
+    parser.add_argument("--operator", default="Oihan Elesgaray", help="Name of operator.")
+
+    args = parser.parse_args()
+    if len(args.files) == 0:
+        print("I need an input file")
+        sys.exit()
+
+    if not Path(args.files[0]).exists():
+        print("Input file does not exist.")
+        sys.exit()
+
+
+    # ITk_PB authentication
+    dlg = ITkDBlogin.ITkDBlogin()
+    pdb_session = dlg.get_client()
+
+    readTemplateTable(pdb_session, args)
+
+    dlg.die()
+
+if __name__ == "__main__":
+    main()

petal_qc/test/analyzeMetrologyTable.py

@@ -1,5 +1,9 @@
 #!/usr/bin/env python3
-"""Analyze the table generated by createMetrologyTable."""
+"""Analyze the table generated by createMetrologyTable.
+
+   It should also work with the table created by analyzeMetrologyTable, which is
+   the new default.
+"""
 
 import sys
 from pathlib import Path
@@ -13,7 +17,22 @@ from lmfit.models import LinearModel
 
 from petal_qc.utils.fit_utils import draw_best_fit
 
+fig_width = 12.0
+fig_height = 1.2*fig_width/3.0
+
+
+
 def distance(P1, P2, P3):
+    """Distance of P3 to line defined by P1-P2.
+
+    Args:
+        P1: Point 1 of the line
+        P2: Point 2 of the line
+        P3: Actual point
+
+    Returns:
+        float: distance
+    """
     C = np.cross(P2-P1, P3-P1)
     D = C/norm(P2-P1)
     return D
@@ -38,14 +57,41 @@ def remove_outliers_indx(data, cut=2.0, debug=False):
     indx = np.where(s < cut)[0]
     return indx
 
+def draw_delta_scatter(ax, x, y, title, x_lim=None, y_lim=None, radius=None):
 
-def main(options):
-    """main entry."""
-    fig_width = 12.0
-    fig_height = 1.2*fig_width/3.0
+    if x_lim is None:
+        x_lim=(-150, 150)
 
-    T = pd.read_csv(options.files[0])
+    if y_lim is None:
+        y_lim=(-150, 150)
+
+    if radius is None:
+        radius = (25, 100)
+
+    ax.set_title(title)
+    ax.set_aspect('equal', adjustable='box')
+    ax.set_xlim(x_lim)
+    ax.set_ylim(y_lim)
+    circle = plt.Circle((0,0), radius[1], color="red", alpha=0.25)
+    ax.add_patch(circle)
+    circle = plt.Circle((0,0), radius[0], color="green", alpha=0.25)
+    ax.add_patch(circle)
 
+    ax.set_xlabel("X (µm)")
+    ax.set_ylabel("Y (µm)")
+    ax.grid()
+
+    ax.scatter(x, y, marker='.')
+
+
+def analyze_fiducials(options, T):
+    """Analyze relative position of locator fiducials.
+
+    Args:
+        options (_type_): Program options
+        T (DataFrame): Pandas data frame.
+    """
+    side = T["side"].values[0]
     if options.mould > 0:
         x = 1000*T.loc[T['mould'] == options.mould, 'fd_dx'].values
         y = 1000*T.loc[T['mould'] == options.mould, 'fd_dy'].values
@@ -54,22 +100,23 @@ def main(options):
         x = 1000*T['fd_dx'].values
         y = 1000*T['fd_dy'].values
 
-    fig, ax = plt.subplots(nrows=1, ncols=3, tight_layout=True, figsize=(fig_width, fig_height))
-    fig.suptitle("Relative Position FD01-FD02")
-    ax[0].set_title("FD01-FD02")
-    ax[0].set_aspect('equal', adjustable='box')
-    ax[0].set_xlim(-150, 150)
-    ax[0].set_ylim(-150, 150)
-    circle = plt.Circle((0,0), 75, color="red", alpha=0.25)
-    ax[0].add_patch(circle)
-    circle = plt.Circle((0,0), 25, color="green", alpha=0.25)
-    ax[0].add_patch(circle)
-
-    ax[0].set_xlabel("X (µm)")
-    ax[0].set_ylabel("Y (µm)")
-    ax[0].grid()
-
-    ax[0].scatter(x, y, marker='.')
+    fig, ax = plt.subplots(nrows=1, ncols=2, tight_layout=True, figsize=(fig_width, fig_height))
+    fig.suptitle("Relative Position FD01-FD02 [{}]".format(side))
+    draw_delta_scatter(ax[0], x, y, "FD01-FD02")
+    # ax[0].set_title("FD01-FD02")
+    # ax[0].set_aspect('equal', adjustable='box')
+    # ax[0].set_xlim(-150, 150)
+    # ax[0].set_ylim(-150, 150)
+    # circle = plt.Circle((0,0), 100, color="red", alpha=0.25)
+    # ax[0].add_patch(circle)
+    # circle = plt.Circle((0,0), 25, color="green", alpha=0.25)
+    # ax[0].add_patch(circle)
+    #
+    # ax[0].set_xlabel("X (µm)")
+    # ax[0].set_ylabel("Y (µm)")
+    # ax[0].grid()
+    #
+    # ax[0].scatter(x, y, marker='.')
 
     model = LinearModel()
     params = model.guess(y, x=x)
@@ -94,15 +141,94 @@ def main(options):
     angle = 180*math.atan( result.best_values['slope'])/math.pi
     print("angle {:.5f} deg.".format(angle))
 
-    ax[1].set_xlim(-150, 150)
-    ax[1].set_xlabel("X (µm)")
+    ax[1].set_xlim(-50, 50)
+    ax[1].set_xlabel("distance to line (µm)")
     ax[1].grid()
-    ax[1].hist(x)
+    ax[1].hist(values)
+
+    # ax[2].set_xlim(-75, 75)
+    # ax[2].set_xlabel("delta_X (µm)")
+    # ax[2].grid()
+    # ax[2].hist(x)
+
+    # ax[3].set_xlim(-150, 150)
+    # ax[3].set_xlabel("delta_Y (µm)")
+    # ax[3].grid()
+    # ax[3].hist(y)
+
+def analyze_delta_pos(options, T):
+    """ Study dependency of delta pos in PL01-FD01 with the other locators.
+
+    Args:
+        options (_type_): Program options
+        T (DataFrame): Pandas data frame.
+    """
+    side = T["side"].values[0]
+    x = T["pl01_dx"].values
+    y = T["pl01_dy"].values
+    delta_pl01 = 1000*np.column_stack((x, y))
+    norm = np.array([ np.linalg.norm(P)  for P in delta_pl01])
+    indx = np.where(norm<75.0)
+
+    delta_pl02 = 1000*np.column_stack((T["pl02_dx"].values, T["pl02_dy"].values))
+    delta_pl03 = 1000*np.column_stack((T["pl03_dx"].values, T["pl03_dy"].values))
+
+    fig, ax = plt.subplots(nrows=1, ncols=2, tight_layout=True, figsize=(fig_width, fig_height))
+    fig.suptitle("Delta from PL01 [{}]".format(side))
+    draw_delta_scatter(ax[0], delta_pl02[indx, 0], delta_pl02[indx, 1], "PL02")
+    draw_delta_scatter(ax[1], delta_pl03[indx, 0], delta_pl03[indx, 1], "PL02")
+
+
+def analyze_parallelism(options, T):
+    """Analyze parallelism.
+
+    Args:
+        options (_type_): Program options
+        T (DataFrame): Pandas data frame.
+    """
+    if options.institute is None:
+        df = T
+    else:
+        df = T.loc[T["institute"] == options.institute]
+
+
+    side = df["side"].values[0]
+    labels = []
+    for L in df["petal_id"].values:
+        ival = int(L[5:])
+        labels.append(ival)
+
+    x = 1000*df["flatness"].values
+    y = 1000*df["parallelism"].values
+
+    fig, ax = plt.subplots(nrows=1, ncols=1, tight_layout=True)
+    fig.suptitle("Parallelism .vs. flatness [{}]".format(side))
+
+    ax.scatter(x, y, marker='.')
+    for xp, yp, lbl in zip(x, y, labels):
+        ax.text(xp, yp, "{}".format(lbl))
+
+    ax.set_xlabel("flatness (µm)")
+    ax.set_ylabel("parallelism (µm)")
+    ax.grid()
+
+    fig, ax = plt.subplots(nrows=1, ncols=1, tight_layout=True)
+    fig.suptitle("Offset [{}]".format(side))
+    ax.hist(1000*T["offset"].values)
+
+
+def main(options):
+    """main entry."""
+    T = pd.read_csv(options.files[0])
+
+    # analyze fiducials
+    analyze_fiducials(options, T)
+
+    # Understand deviation from nominal.
+    analyze_delta_pos(options, T)
 
-    ax[2].set_xlim(-150, 150)
-    ax[2].set_xlabel("Y (µm)")
-    ax[2].grid()
-    ax[2].hist(y)
+    # Correlate planarity y parallelism
+    analyze_parallelism(options, T)
 
     plt.show()
 
@@ -110,6 +236,9 @@ if __name__ == "__main__":
     parser = argparse.ArgumentParser()
     parser.add_argument('files', nargs='*', help="Input files")
     parser.add_argument('--mould', default=-1, type=int, help="mould index")
+    parser.add_argument("--institute",
+                    default=None,
+                    help="Either IFIC or DESY to treat the different files")
 
     opts = parser.parse_args()
     if len(opts.files) == 0:

petal_qc/test/getAVStests.py

@@ -323,10 +323,10 @@ if __name__ == "__main__":
     parser.add_argument("--cores", dest="cores", action=RangeListAction, default=[],
                         help="Create list of cores to analyze. The list is made with  numbers or ranges (ch1:ch2 or ch1:ch2:step) ")
 
-    
+    opts = parser.parse_args()
     dlg = ITkDBlogin.ITkDBlogin()
     session = dlg.get_client()
 
-    main(session, options)
+    main(session, opts)
 
     dlg.die()

petal_qc/test/reportFromJSon.py

@@ -38,15 +38,17 @@ def main(options):
             data = json.load(fin)
 
         if not data["passed"]:
-            petalId = data["component"]
-            if petalId is None:
+            if data["component"] is None:
                 print("Petal {} has bad Serial number".format(petal_id))
-            if petal_id not in petal_cores:
-                petal_cores[petal_id] = {"FRONT": [], "BACK": []}
+                continue
+            
+            petalId = "{}-{}".format(petal_id, data["component"])
+            if petalId not in petal_cores:
+                petal_cores[petalId] = {"FRONT": [], "BACK": []}
 
             side = "FRONT" if "FRONT" in data["testType"] else "BACK"
             for D in data["defects"]:
-                petal_cores[petal_id][side].append("{}: {}".format(D["name"], D["description"]))
+                petal_cores[petalId][side].append("{}: {}".format(D["name"], D["description"]))
 
 
     keys = sorted(petal_cores.keys())
@@ -69,8 +71,8 @@ if __name__ == "__main__":
     opts = parser.parse_args()
     
     
-    folder = Path("/tmp/petal-metrology/results")
-    # folder = Path("~/tmp/petal-metrology/Production/Results").expanduser()
+    #folder = Path("/tmp/petal-metrology/results")
+    folder = Path("~/tmp/petal-metrology/Production/Results").expanduser()
     opts.files = []
     for fnam in all_files(folder, "*.json"):
         opts.files.append(fnam)

petal_qc/test/testMitutoyo.py

@@ -0,0 +1,10 @@
+#!/usr/bin/env python
+"""Test."""
+
+from pathlib import Path
+from petal_qc.metrology import DataFile
+
+ifile = Path("/tmp/petals/re-measured/test-xx.txt")
+
+data = DataFile.read(ifile, r"Punto(-|(Vision-))\d", "Punto")
+print(data)

petal_qc/thermal/IRDataGetter.py

@@ -51,6 +51,7 @@ class IRDataGetter(object):
 
     def fine_tune_params(self, param):
         """Set default values for the parameters."""
+        param.sensor_max = -5
         return
 
     def get_IR_data(self, image, **kargs):
@@ -298,6 +299,7 @@ class IRDataDESY(IRDataGetter):
         """Set default values for the parameters."""
         param.distance = 16
         param.width = 16
+        param.sensor_max = -5
 
     def get_IR_data(self, image, **kargs):
         """Get the data from the image in the proper orientation.

petal_qc/thermal/IRPetal.py

@@ -515,7 +515,7 @@ def find_slice_minimum(T, X):
     return Tmin, Pmin
 
 
-def get_T_profile(data, A, B, npts=10, do_fit=False, npdim=7, debug=False):
+def get_T_profile(data, A, B, npts=10, do_fit=False, npdim=6, debug=False):
     """Get the temperature profile between A and B.
 
     Args:
@@ -569,13 +569,16 @@ def get_T_profile(data, A, B, npts=10, do_fit=False, npdim=7, debug=False):
 
         # Do the polinomialfit.
         if ndof > npdim:
-            c, stats = Polynomial.polyfit(X, T, npdim, full=True)
+            indx = remove_outliers(np.array(T), 6, indx=True)
+            TT = np.array(T)
+            XX = np.array(X)
+            c, stats = Polynomial.polyfit(XX[indx], TT[indx], npdim, full=True)
             pfunc = Polynomial.Polynomial(c)
 
             # Get the polinomial derivative roots and curvature.
             # This is done to catch double minimums
 
-            # valid_roots, valid_values, valid_curv = get_derivatives(T_min, X[1], X[-1], c)
+            valid_roots, valid_values, valid_curv = get_derivatives(T_min, X[1], X[-1], c)
 
             #
             # TODO: temporary remove the search for double minima in the segment.
@@ -585,6 +588,13 @@ def get_T_profile(data, A, B, npts=10, do_fit=False, npdim=7, debug=False):
                 print("--- Multi-minimum segment.")
                 for r, v, crv in zip(valid_roots, valid_values, valid_curv):
                     print("{} -> {} [{}]".format(r, v, crv))
+                    
+                x = np.linspace(x_min, x_max, 3*npts)
+                debug_plot.plot('TProf', X, T, 'o', x, pfunc(x), '-', valid_roots, valid_values, 'o')
+                if len(valid_roots) > 1:
+                    debug_plot.setup_debug('TProfMulti')
+                    debug_plot.plot('TProfMulti', X, T, 'o', x, pfunc(x), '-', valid_roots, valid_values, 'o')
+                    debug_plot.set_title('TProfMulti', "Curvature: {}".format(str(valid_curv)))
 
             # Find the polynomial minimum with `minimize` (redundant, probably)
             X0 = X[np.argmin(T)]
@@ -722,7 +732,7 @@ def get_derivatives(T_min, x_min, x_max, coeff):
     valid_values = pfunc(valid_roots)
     valid_curv = deriv2(valid_roots)
 
-    indx = np.where(valid_curv > 0.1)
+    indx = np.where(valid_curv > 1e-3)
     valid_roots = valid_roots[indx]
     valid_values = valid_values[indx]
     valid_curv = valid_curv[indx]
@@ -983,13 +993,14 @@ def get_IR_pipe(data, params):
     return min_path
 
 
-def remove_outliers(data, cut=2.0, debug=False):
+def remove_outliers(data, cut=2.0, indx=False, debug=False):
     """Remove points far away form the rest.
 
     Args:
     ----
         data : The data
         cut: max allowed distance
+        indx: if True return indices rather than a new array
         debug: be bverbose if True
 
     """
@@ -997,7 +1008,10 @@ def remove_outliers(data, cut=2.0, debug=False):
     d = np.abs(data - np.median(data))
     mdev = np.median(d)
     s = d / (mdev if mdev else 1.)
-    return data[s < cut]
+    if indx:
+        return np.where(s < cut)[0]
+    else:
+        return data[s < cut]
 
 
 def get_T_along_path(irpath, data, width, norm=True):