petal-qc 0.0.15__py3-none-any.whl → 0.0.17__py3-none-any.whl

petal_qc/test/analyzeMetrologyTable.py

@@ -0,0 +1,119 @@
+#!/usr/bin/env python3
+"""Analyze the table generated by createMetrologyTable."""
+
+import sys
+from pathlib import Path
+import math
+import argparse
+import pandas as pd
+import numpy as np
+from numpy.linalg import norm
+import matplotlib.pyplot as plt
+from lmfit.models import LinearModel
+
+from petal_qc.utils.fit_utils import draw_best_fit
+
+def distance(P1, P2, P3):
+    C = np.cross(P2-P1, P3-P1)
+    D = C/norm(P2-P1)
+    return D
+
+def remove_outliers_indx(data, cut=2.0, debug=False):
+    """Remove points far away form the rest.
+
+    Args:
+    ----
+        data : The data
+        cut: max allowed distance
+        debug: be verbose if True.
+
+    Returns
+    -------
+        index of valid pints in data array.
+
+    """
+    d = np.abs(data - np.median(data))
+    mdev = np.median(d)
+    s = d / (mdev if mdev else 1.)
+    indx = np.where(s < cut)[0]
+    return indx
+
+
+def main(options):
+    """main entry."""
+    fig_width = 12.0
+    fig_height = 1.2*fig_width/3.0
+
+    T = pd.read_csv(options.files[0])
+
+    if options.mould > 0:
+        x = 1000*T.loc[T['mould'] == options.mould, 'fd_dx'].values
+        y = 1000*T.loc[T['mould'] == options.mould, 'fd_dy'].values
+
+    else:
+        x = 1000*T['fd_dx'].values
+        y = 1000*T['fd_dy'].values
+
+    fig, ax = plt.subplots(nrows=1, ncols=3, tight_layout=True, figsize=(fig_width, fig_height))
+    fig.suptitle("Relative Position FD01-FD02")
+    ax[0].set_title("FD01-FD02")
+    ax[0].set_aspect('equal', adjustable='box')
+    ax[0].set_xlim(-150, 150)
+    ax[0].set_ylim(-150, 150)
+    circle = plt.Circle((0,0), 75, color="red", alpha=0.25)
+    ax[0].add_patch(circle)
+    circle = plt.Circle((0,0), 25, color="green", alpha=0.25)
+    ax[0].add_patch(circle)
+
+    ax[0].set_xlabel("X (µm)")
+    ax[0].set_ylabel("Y (µm)")
+    ax[0].grid()
+
+    ax[0].scatter(x, y, marker='.')
+
+    model = LinearModel()
+    params = model.guess(y, x=x)
+    result = model.fit(y, params, x=x)
+
+
+    P1 = np.array([0, result.eval(x=0)])
+    P2 = np.array([1, result.eval(x=1)])
+    values = []
+    for v, w in zip(x, y):
+        P3 = np.array([v, w])
+        values.append(distance(P1, P2, P3))
+
+    indx = remove_outliers_indx(values)
+
+    xx = x[indx]
+    yy = y[indx]
+    params = model.guess(yy, xx)
+    result = model.fit(yy, params, x=xx)
+    result.plot_fit(ax=ax[0])
+    print("slope {:.5f}, intercept {:.5f}".format(result.best_values['slope'], result.best_values["intercept"]))
+    angle = 180*math.atan( result.best_values['slope'])/math.pi
+    print("angle {:.5f} deg.".format(angle))
+
+    ax[1].set_xlim(-150, 150)
+    ax[1].set_xlabel("X (µm)")
+    ax[1].grid()
+    ax[1].hist(x)
+
+    ax[2].set_xlim(-150, 150)
+    ax[2].set_xlabel("Y (µm)")
+    ax[2].grid()
+    ax[2].hist(y)
+
+    plt.show()
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument('files', nargs='*', help="Input files")
+    parser.add_argument('--mould', default=-1, type=int, help="mould index")
+
+    opts = parser.parse_args()
+    if len(opts.files) == 0:
+        print("I need at least one input file")
+        sys.exit()
+
+    main(opts)

petal_qc/test/checkAVStests.py

@@ -169,8 +169,8 @@ if __name__ == "__main__":
     client = dlg.get_client()
 
     try:
-        # main(session)
-        the_test = analyze_avs_metrology(client, "20USEBC1000124", "PPC.015")
+        main(client)
+        # the_test = analyze_avs_metrology(client, "20USEBC1000124", "PPC.015")
         #rc = ITkDButils.upload_test(client, the_test, check_runNumber=True)
         #if rc:
         #    print(rc)

petal_qc/test/checkCoreShipments.py

@@ -0,0 +1,57 @@
+"""List shipments."""
+import sys
+from pathlib import Path
+import numpy as np
+
+from itkdb_gtk import ITkDBlogin, ITkDButils
+
+
+def main(session):
+    """List shipments from AVS to CERN and IFIC containing CORES."""
+    
+    payload = {
+            "filterMap": {
+                "sender": "AVS",
+                "recipient": ["CERN", "IFIC"],
+                "status": "delivered"
+            }
+        }
+    shpmts = session.get("listShipmentsByInstitution", json=payload)
+    for s in shpmts:
+        items = session.get("listShipmentItems", json={"shipment": s["id"]})
+        cores = []
+        for it in items:
+            if it["component"]["componentType"]['code'] == "CORE_PETAL":
+                if "PPC." in it["component"]["alternativeIdentifier"]:
+                    cores.append(it["component"]["alternativeIdentifier"])
+                
+        if len(cores) == 0:
+            continue
+        
+        print("Shipment {}".format(s["name"]))
+        print("AVS -> {}".format(s["recipient"]["code"]))
+        print(":> {}".format(s["sentTs"]))
+        for c in cores:
+            print("\t{}".format(c))
+
+        print("\n")
+
+if __name__ == "__main__":
+    # ITk_PB authentication
+    dlg = ITkDBlogin.ITkDBlogin()
+    client = dlg.get_client()
+
+    try:
+        main(client)
+        # the_test = analyze_avs_metrology(client, "20USEBC1000124", "PPC.015")
+        #rc = ITkDButils.upload_test(client, the_test, check_runNumber=True)
+        #if rc:
+        #    print(rc)
+
+    except Exception as E:
+        print(E)
+
+    dlg.die()
+
+    
+    

petal_qc/test/createMetrologyTable.py

@@ -0,0 +1,87 @@
+#!/usr/bin/env python3
+"""Create  table for metrology analysis."""
+
+import sys
+import re
+import json
+from pathlib import Path
+from argparse import ArgumentParser
+from petal_qc.utils.all_files import all_files
+from petal_qc.utils.ArgParserUtils import RangeListAction
+
+r_petal_id = re.compile("PPC.([0-9]*)")
+
+db_file = Path("/Users/lacasta/cernbox/workspace/Petal-QC/PetalMould.csv")
+
+petal_db = {}
+with open(db_file, "r", encoding="utf-8") as fin:
+    for i, line in enumerate(fin):
+        if i==0:
+            continue
+
+        values = [x.strip() for x in line.split(",")]
+        petal_db[values[0]] = values[1:]
+
+def main(options):
+    """Main entry."""
+    ifolder = Path(options.input)
+    if not ifolder.exists():
+        print("Input folder does not exist.\n{}".format(ifolder))
+        return
+
+    has_list = len(options.cores) != 0
+
+    ofile = open(options.out, "w", encoding="utf-8")
+    ofile.write("PetalID,SN,side,mould,fd_dx,fd_dy,paralelism,R0,R1,R2,R3S0,R3S1,R4S0,R4S1,R5S0,R5S1,flatness\n")
+
+    for fnam in all_files(ifolder.as_posix(), "*.json"):
+        fstem = fnam.stem
+        if "PPC." not in fstem:
+            continue
+
+        R = r_petal_id.search(fstem)
+        if R is None:
+            continue
+
+        petal_id = R.group(0)
+        pid = int(R.group(1))
+
+        if has_list and pid not in options.cores:
+            continue
+
+        SN = petal_db[petal_id][0]
+        side = 1 if "back" in fstem else 0
+        mould = int(petal_db[petal_id][1])
+
+        data = None
+        with open(fnam, "r", encoding="utf-8") as fin:
+            data = json.load(fin)
+
+        ofile.write("{}, {}, {}, {}".format(petal_id, SN, side, mould))
+        print("data SN", data["component"])
+        D = data["results"]["METROLOGY"]["REL_POS_DELTA"]["FD01-FD02"]
+        ofile.write(",{:.5f}, {:.5f}".format(D[0], D[1]))
+
+        D = data["results"]["METROLOGY"]["PARALLELISM"]
+        ofile.write(",{:.5f}".format(D))
+
+        for x in data["results"]["METROLOGY"]["FLATNESS_LOCAL"]:
+            ofile.write(", {:.5f}".format(x))
+
+        D = data["results"]["METROLOGY"]["FLATNESS_GLOBAL"]
+        ofile.write(", {:.5f}".format(D))
+
+        ofile.write("\n")
+
+    ofile.close()
+
+
+if __name__ == "__main__":
+    parser = ArgumentParser()
+    parser.add_argument("--input-folder", dest="input", default=".", help="Input folder")
+    parser.add_argument("--out", default="out.csv", help="Output table.")
+    parser.add_argument("--cores", dest="cores", action=RangeListAction, default=[],
+                        help="Create list of cores to analyze. The list is made with  numbers or ranges (ch1:ch2 or ch1:ch2:step) ")
+
+    opts = parser.parse_args()
+    main(opts)

petal_qc/test/getAVStests.py

@@ -4,6 +4,7 @@ import sys
 from pathlib import Path
 import numpy as np
 import matplotlib.pyplot as plt
+from argparse import ArgumentParser
 
 try:
     import petal_qc
@@ -14,7 +15,7 @@ except ImportError:
 
 from itkdb_gtk import ITkDBlogin
 import petal_qc.utils.docx_utils as docx_utils
-
+from petal_qc.utils.ArgParserUtils import RangeListAction
 
 
 def get_value(results, code):
@@ -171,7 +172,7 @@ def plot_weighing(weights, tick_labels, document, show_total=False):
     ax.legend(loc="upper left", ncol=4)
 
 
-def main(session):
+def main(session, options):
     """Entry point"""
     # find all cores
     # Now all the objects
@@ -200,11 +201,16 @@ def main(session):
     for T in core_tests:
         counter[T]=0
 
+    mould_db = {}
     for core in core_list:
         SN = core["serialNumber"]
         altid = core['alternativeIdentifier']
         if "PPC" not in altid:
             continue
+        
+        pID = int(altid[4:])
+        if len(options.cores)>0 and pID not in options.cores:
+            continue
 
         petal_ids.append(altid)
         location = core["currentLocation"]['code']
@@ -243,6 +249,7 @@ def main(session):
         mould_desc = do_manufacturing(good_tests["MANUFACTURING"])
         pos = mould_desc.rfind('.')
         mould_id = mould_desc[pos+1:]
+        mould_db[altid] = (SN, mould_id)
         if mould_id not in Mould_values:
             Mould_values[mould_id] = {}
 
@@ -302,13 +309,24 @@ def main(session):
     plot_weighing(weights, petal_ids, document)
     plot_metrology(M_values, Mould_values, petal_ids, document)
     document.save("AVStests.docx")
+
+    with open("mould_db.csv", "w", encoding="utf-8") as fout:
+        fout.write("PetalID, SerialNo, MouldID\n")
+        for key, val in mould_db.items():
+            fout.write("{}, {}, {}\n".format(key, val[0], val[1]))
+
     plt.show()
 
 if __name__ == "__main__":
     # ITk_PB authentication
+    parser = ArgumentParser()
+    parser.add_argument("--cores", dest="cores", action=RangeListAction, default=[],
+                        help="Create list of cores to analyze. The list is made with  numbers or ranges (ch1:ch2 or ch1:ch2:step) ")
+
+    
     dlg = ITkDBlogin.ITkDBlogin()
     session = dlg.get_client()
 
-    main(session)
+    main(session, options)
 
     dlg.die()

petal_qc/test/prepareDESYfiles.py

@@ -96,7 +96,8 @@ def main(folder, out_folder):
             data = ""
             for data_type, fnam in values.items():
                 if fnam is None:
-                    print("This should not happen.")
+                    print("Missing file name for {} {} [{}]".format(petal_id, side, data_type))
+                    continue
 
                 with open(fnam, "r", encoding="UTF-8") as ifile:
                     data += ifile.read()

petal_qc/test/test_Graphana.py

@@ -19,7 +19,8 @@ from petal_qc.thermal import IRBFile
 
 
 options = IRPetalParam()
-options.files = [Path("~/tmp/thermal/PPC.008.irb").expanduser().resolve()]
+options.institute = "IFIC"
+options.files = [Path("~/tmp/thermal/IFIC-thermal/IRB_files/PPC.008.irb").expanduser().resolve()]
 getter = IRDataGetter.factory(options.institute, options)
 DB = ReadGraphana("localhost")
 irbf = IRBFile.open_file(options.files)

petal_qc/thermal/IRPetalParam.py

@@ -27,6 +27,7 @@ class IRPetalParam(object):
         self.rotate = True        # Rotate to have a vertical petal in mirror image
         self.debug = False        # To debug
         self.report = False       # 
+        self.graphana = None      # Graphana server
 
         if values is not None:
             self.set_values(values)
@@ -61,6 +62,7 @@ class IRPetalParam(object):
                             help="Either IFIC or DESY to treat the different files")
         parser.add_argument("--thrs", type=float, default=P.thrs, help="Temperature threshold")
         parser.add_argument("--tco2", type=float, default=P.tco2, help="CO2 Inlet temperature")
+        parser.add_argument("--graphana", type=str, default=None, help="Graphana server.")
         parser.add_argument("--gauss_size", type=int, default=P.gauss_size, help="Radius of gausian filtering")
         parser.add_argument("--distance", type=float, default=P.distance, help="Distance in contour beteween slices")
         parser.add_argument("--npoints", type=int, default=P.npoints, help="Number of points per segment")

petal_qc/thermal/PipeFit.py

@@ -8,7 +8,6 @@ from pathlib import Path
 import matplotlib.path as mplPath
 import matplotlib.pyplot as plt
 import numpy as np
-import numpy.linalg as linalg
 from scipy.optimize import least_squares
 
 from petal_qc.thermal import contours
@@ -43,6 +42,9 @@ class PipeFit(object):
         self.bounds = [[-math.pi, 0.8, 0.8, -np.inf, -np.inf],
                        [math.pi, 3.0, 3.0, np.inf, np.inf]]
 
+        self.core_center = None
+        self.core_band = None
+
     def set_front(self, is_front=True):
         """Sets for a front image.
 
@@ -119,6 +121,7 @@ class PipeFit(object):
         """
         DB = np.load(pipe_file)
         sensors = []
+        pipe = []
         for key, val in DB.items():
             if key == "pipe":
                 pipe = val
@@ -213,7 +216,7 @@ class PipeFit(object):
             print("Center {:5.3f}, {:5.3f}".format(M[5], M[6]))
 
     @staticmethod
-    def get_data_center(data, fraction=0.5, y0=None):
+    def get_data_center(data, fraction=0.5, y0=None, min_pts=5):
         """Compute the data center.
 
         Assumes it is in the half height, and the X is the average of the points
@@ -221,19 +224,19 @@ class PipeFit(object):
         """
         if y0 is None:
             bounding_box = contours.contour_bounds(data)
-            y0 = fraction*(bounding_box[1]+bounding_box[3])
+            y0 = bounding_box[1] + fraction*(bounding_box[3]-bounding_box[1])
 
         window = 10
         while True:
             stripe = data[np.abs(data[:, 1] - y0) < window]
-            if len(stripe) > 5:
+            if len(stripe) > min_pts:
                 break
 
             window += 1
 
         m0 = np.mean(stripe, axis=0)
         center = np.array([m0[0], y0])
-        return center
+        return center, (np.min(stripe[:, 0]), np.max(stripe[:, 0]))
 
     @staticmethod
     def guess_pipe_type(data, fraction=0.25) -> int:
@@ -256,7 +259,7 @@ class PipeFit(object):
 
         D = data[:, 0:2] - P0
         m0 = np.mean(D, axis=0)
-        center = PipeFit.get_data_center(D, y0=m0[1])
+        center, _ = PipeFit.get_data_center(D, y0=m0[1])
         # fig, ax = plt.subplots(1,1)
         # ax.plot(D[:,0], D[:, 1], 'o', label="Data")
         # ax.plot(center[0], center[1], '*', label="Center")
@@ -273,9 +276,16 @@ class PipeFit(object):
     @staticmethod
     def guess_pipe_angle(data, center):
         """Get an estimation of th epipe angle."""
-        c1 = PipeFit.get_data_center(data, 0.6)
-        delta = c1 - center
-        angle = math.atan(delta[0]/delta[1])
+        c1, _ = PipeFit.get_data_center(data, 0.)
+        delta = np.abs(c1 - center)
+        angle = math.atan(delta[1]/delta[0])
+        while angle<0:
+            angle += 2.0*math.pi
+
+        if angle > 1.5*math.pi:
+            angle -= math.pi/2.0
+
+        angle = 0.5*math.pi - angle
         return angle
 
     @staticmethod
@@ -289,8 +299,11 @@ class PipeFit(object):
 
     def initial_guess(self, data):
         """Make a first guess of the transform."""
-        Mdata = self.get_data_center(data)
+        Mdata, _ = self.get_data_center(data)
         theta = self.guess_pipe_angle(data, Mdata)
+        if self.front:
+            theta = -theta
+
         T = -Mdata
         dxd = np.amax(data, axis=0) - np.amin(data, axis=0)
         dxp = np.amax(self.cpipe, axis=0) - np.amin(self.cpipe, axis=0)
@@ -313,7 +326,11 @@ class PipeFit(object):
     def get_residuals(self, M):
         """Compute intersecting area."""
         out = self.transform_data(self.data, M)
-        use_area = True
+        y_max = np.max(out[:,1])
+        y_min = np.min(out[:,1])
+
+        height = y_max - y_min
+        use_area = False
         if use_area:
             path = mplPath.Path(out)
             ngood = 0.0
@@ -327,20 +344,29 @@ class PipeFit(object):
             real_area = 100*ngood/ntot
 
         else:
-            area = 0.0
-            real_area = 0
+            area = 1.0
+            real_area = 1
 
         npts = len(self.data)
         D = np.zeros([npts, 2])
         ddd = np.zeros(npts)
+        sum_weights = 0.0
+        weights = np.zeros(npts)
         for i in range(npts):
             X = out[i, :]
             dst, P = contours.find_closest_point(X[0], X[1], self.cpipe)
             D[i, :] = P - X
             ddd[i] = dst
+            W = 1 + 1.6*(y_max - X[1])/height
+            weights[i] = W
+            sum_weights += W
 
         # return value
-        res = D.flatten()*area
+        if use_area:
+            res = D.flatten()*area
+
+        else:
+            res = np.dot(ddd, weights)/sum_weights
 
         if self.debug:
             dbg_ax. clear()
@@ -420,14 +446,17 @@ class PipeFit(object):
                 elif val > self.bounds[1][i]:
                     val = 0.99 *  self.bounds[0][i]
                     M[i] = val
-                    
+
             if self.debug:
                 print("\n** Initial guess")
                 self.print_transform(M)
         else:
             M = M0
 
-        self.data = contours.contour_simplify(data, 1.5*factor)
+        self.core_center, self.core_band = self.get_data_center(data)
+
+
+        self.data = contours.contour_simplify(data, 1.25*factor)
         # self.data = data
         verbose = 0
         if self.debug:
@@ -436,7 +465,7 @@ class PipeFit(object):
         res = least_squares(self.get_residuals, M,
                             method='trf',
                             # ftol=5e-16,
-                            # xtol=None,
+                            #xtol=1.0e-10,
                             # max_nfev=10000,
                             # diff_step=[0.05, 0.02, 0.02, 50, 50],
                             bounds=self.bounds,
@@ -551,7 +580,7 @@ def main(data_file, opts):
     ax[0].plot(PF.center[0], PF.center[1], 'o')
     ax[0].set_title("Pipe")
 
-    center = PF.get_data_center(data)
+    center, _ = PF.get_data_center(data)
     ax[1].plot(data[:, 0], data[:, 1])
     ax[1].plot(center[0], center[1], 'o')
     ax[1].set_title("Data on IR")
@@ -560,7 +589,7 @@ def main(data_file, opts):
     plt.figure(fign)
     out = PF.transform_data(PF.data, R)
     aout = PF.transform_data(data, R)
-    center = PF.get_data_center(aout)
+    center, _ = PF.get_data_center(aout)
     ax[2].plot(PF.pipe[:, 0], PF.pipe[:, 1])
     ax[2].plot(out[:, 0], out[:, 1], 'o')
     ax[2].plot(center[0], center[1], 'o')