petal-qc 0.0.17__py3-none-any.whl → 0.0.24__py3-none-any.whl

petal_qc/test/reportFromJSon.py

@@ -6,17 +6,43 @@ from pathlib import Path
 import json
 
 
+try:
+    import petal_qc
+
+except ImportError:
+    cwd = Path(__file__).parent.parent
+    sys.path.append(cwd.as_posix())
+
+from petal_qc.utils.ArgParserUtils import RangeListAction
+from petal_qc.utils.all_files import all_files
+
+
 
 def main(options):
     """main entry."""
     petal_cores = {}
-    for fnam in options.files:
-        ifile = Path(fnam).expanduser().resolve()
-        with open(ifile, "r", encoding="utf-8") as fin:
+    for fpath in options.files:
+        fnam = str(fpath)
+        if "PPC." not in fnam:
+            continue
+
+        ipos = fnam.find("PPC")
+        lpos = fnam[ipos:].find("-")
+        petal_id = fnam[ipos:ipos+lpos]
+        pid = int(petal_id[4:])
+        
+        if len(options.cores)>0 and pid not in options.cores:
+            continue
+
+        with open(fpath, "r", encoding="utf-8") as fin:
             data = json.load(fin)
 
         if not data["passed"]:
-            petalId = data["component"]
+            if data["component"] is None:
+                print("Petal {} has bad Serial number".format(petal_id))
+                continue
+            
+            petalId = "{}-{}".format(petal_id, data["component"])
             if petalId not in petal_cores:
                 petal_cores[petalId] = {"FRONT": [], "BACK": []}
 
@@ -40,12 +66,15 @@ def main(options):
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
     parser.add_argument('files', nargs='*', help="Input files")
+    parser.add_argument("--cores", dest="cores", action=RangeListAction, default=[],
+                        help="Create list of cores to analyze. The list is made with  numbers or ranges (ch1:ch2 or ch1:ch2:step) ")
     opts = parser.parse_args()
     
-    from petal_qc.utils.all_files import all_files
     
+    #folder = Path("/tmp/petal-metrology/results")
+    folder = Path("~/tmp/petal-metrology/Production/Results").expanduser()
     opts.files = []
-    for fnam in all_files(Path("~/tmp/petal-metrology/Production/Results").expanduser(), "*.json"):
+    for fnam in all_files(folder, "*.json"):
         opts.files.append(fnam)
     
     main(opts)

petal_qc/test/testMitutoyo.py

@@ -0,0 +1,10 @@
+#!/usr/bin/env python
+"""Test."""
+
+from pathlib import Path
+from petal_qc.metrology import DataFile
+
+ifile = Path("/tmp/petals/re-measured/test-xx.txt")
+
+data = DataFile.read(ifile, r"Punto(-|(Vision-))\d", "Punto")
+print(data)

petal_qc/test/thermalReportFromJSon.py

@@ -0,0 +1,99 @@
+#!/usr/bin/env python3
+"""List failing cores from JSon files."""
+import sys
+import argparse
+import re
+from pathlib import Path
+import json
+
+from itkdb_gtk import ITkDBlogin
+from itkdb_gtk import ITkDButils
+
+try:
+    import petal_qc
+
+except ImportError:
+    cwd = Path(__file__).parent.parent
+    sys.path.append(cwd.as_posix())
+
+from petal_qc.utils.ArgParserUtils import RangeListAction
+from petal_qc.utils.all_files import all_files
+
+
+match_fnam = re.compile("20US.*Thermal.json", re.DOTALL)
+
+
+def main(client, options):
+    """main entry."""
+    petal_cores = {}
+    for fpath in options.files:
+        fnam = str(fpath.name)
+        M = match_fnam.match(fnam)
+        if M is None:
+            continue
+
+#         ipos = fnam.find("PPC")
+#         lpos = fnam[ipos:].find("-")
+#         petal_id = fnam[ipos:ipos+lpos]
+#         pid = int(petal_id[4:])
+# 
+#         if len(options.cores)>0 and pid not in options.cores:
+#             continue
+
+        with open(fpath, "r", encoding="utf-8") as fin:
+            data = json.load(fin)
+
+        if not data["passed"]:
+            if data["component"] is None:
+                print("Petal {} has bad Serial number".format(petal_id))
+                continue
+
+            core = ITkDButils.get_DB_component(client, data["component"])
+            petal_id = core['alternativeIdentifier']
+            
+            petalId = "{}-{}".format(petal_id, data["component"])
+            print (petalId)
+            for D in data["defects"]:
+                print("{}: {} {}".format(D["name"], D["description"], D["properties"]["msg"]))
+
+
+    keys = sorted(petal_cores.keys())
+    for petalId in keys:
+        print(petalId)
+        for side in ["FRONT","BACK"]:
+            if len(petal_cores[petalId][side])>0:
+                print("+-", side)
+            for D in petal_cores[petalId][side]:
+                print("  ", D)
+
+        print("\n")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument('files', nargs='*', help="Input files")
+    parser.add_argument("--cores", dest="cores", action=RangeListAction, default=[],
+                        help="Create list of cores to analyze. The list is made with  numbers or ranges (ch1:ch2 or ch1:ch2:step) ")
+    opts = parser.parse_args()
+
+
+    dlg = None
+    try:
+        # We use here the Gtk GUI
+        dlg = ITkDBlogin.ITkDBlogin()
+        client = dlg.get_client()
+
+    except Exception:
+        # Login with "standard" if the above fails.
+        client = ITkDButils.create_client()
+
+
+    #folder = Path("/tmp/petal-metrology/results")
+    folder = Path("~/SACO-CSIC/ITk-Strips/Local Supports/thermal/IFIC-thermal/thermal-new-golden/results").expanduser()
+    opts.files = []
+    for fnam in all_files(folder, "*.json"):
+        opts.files.append(fnam)
+
+    main(client, opts)
+    dlg.die()
+    

petal_qc/thermal/IRDataGetter.py

@@ -51,6 +51,7 @@ class IRDataGetter(object):
 
     def fine_tune_params(self, param):
         """Set default values for the parameters."""
+        param.sensor_max = -5
         return
 
     def get_IR_data(self, image, **kargs):
@@ -298,6 +299,7 @@ class IRDataDESY(IRDataGetter):
         """Set default values for the parameters."""
         param.distance = 16
         param.width = 16
+        param.sensor_max = -5
 
     def get_IR_data(self, image, **kargs):
         """Get the data from the image in the proper orientation.

petal_qc/thermal/IRPetal.py

@@ -515,7 +515,7 @@ def find_slice_minimum(T, X):
     return Tmin, Pmin
 
 
-def get_T_profile(data, A, B, npts=10, do_fit=False, npdim=7, debug=False):
+def get_T_profile(data, A, B, npts=10, do_fit=False, npdim=6, debug=False):
     """Get the temperature profile between A and B.
 
     Args:
@@ -569,13 +569,16 @@ def get_T_profile(data, A, B, npts=10, do_fit=False, npdim=7, debug=False):
 
         # Do the polinomialfit.
         if ndof > npdim:
-            c, stats = Polynomial.polyfit(X, T, npdim, full=True)
+            indx = remove_outliers(np.array(T), 6, indx=True)
+            TT = np.array(T)
+            XX = np.array(X)
+            c, stats = Polynomial.polyfit(XX[indx], TT[indx], npdim, full=True)
             pfunc = Polynomial.Polynomial(c)
 
             # Get the polinomial derivative roots and curvature.
             # This is done to catch double minimums
 
-            # valid_roots, valid_values, valid_curv = get_derivatives(T_min, X[1], X[-1], c)
+            valid_roots, valid_values, valid_curv = get_derivatives(T_min, X[1], X[-1], c)
 
             #
             # TODO: temporary remove the search for double minima in the segment.
@@ -585,6 +588,13 @@ def get_T_profile(data, A, B, npts=10, do_fit=False, npdim=7, debug=False):
                 print("--- Multi-minimum segment.")
                 for r, v, crv in zip(valid_roots, valid_values, valid_curv):
                     print("{} -> {} [{}]".format(r, v, crv))
+                    
+                x = np.linspace(x_min, x_max, 3*npts)
+                debug_plot.plot('TProf', X, T, 'o', x, pfunc(x), '-', valid_roots, valid_values, 'o')
+                if len(valid_roots) > 1:
+                    debug_plot.setup_debug('TProfMulti')
+                    debug_plot.plot('TProfMulti', X, T, 'o', x, pfunc(x), '-', valid_roots, valid_values, 'o')
+                    debug_plot.set_title('TProfMulti', "Curvature: {}".format(str(valid_curv)))
 
             # Find the polynomial minimum with `minimize` (redundant, probably)
             X0 = X[np.argmin(T)]
@@ -722,7 +732,7 @@ def get_derivatives(T_min, x_min, x_max, coeff):
     valid_values = pfunc(valid_roots)
     valid_curv = deriv2(valid_roots)
 
-    indx = np.where(valid_curv > 0.1)
+    indx = np.where(valid_curv > 1e-3)
     valid_roots = valid_roots[indx]
     valid_values = valid_values[indx]
     valid_curv = valid_curv[indx]
@@ -983,13 +993,14 @@ def get_IR_pipe(data, params):
     return min_path
 
 
-def remove_outliers(data, cut=2.0, debug=False):
+def remove_outliers(data, cut=2.0, indx=False, debug=False):
     """Remove points far away form the rest.
 
     Args:
     ----
         data : The data
         cut: max allowed distance
+        indx: if True return indices rather than a new array
         debug: be bverbose if True
 
     """
@@ -997,7 +1008,10 @@ def remove_outliers(data, cut=2.0, debug=False):
     d = np.abs(data - np.median(data))
     mdev = np.median(d)
     s = d / (mdev if mdev else 1.)
-    return data[s < cut]
+    if indx:
+        return np.where(s < cut)[0]
+    else:
+        return data[s < cut]
 
 
 def get_T_along_path(irpath, data, width, norm=True):

petal_qc/thermal/IRPetalParam.py

@@ -1,5 +1,5 @@
 """Parameters needed for the thermal analysis of the petal."""
-
+import sys
 
 class IRPetalParam(object):
     """Default values for IR image handling."""
@@ -14,7 +14,7 @@ class IRPetalParam(object):
         """
         self.institute = None     # Either IFIC or DESY to treat the different files
         self.thrs = -22.0         # the threshold
-        self.tco2 = -35.0         # Inlet temperature
+        self.tco2 = -32.0         # Inlet temperature
         self.gauss_size = 15      # Radius of gausian filtering
         self.grad_sigma = 2.5     # Sigma of grading calculation
         self.distance = 5         # Distance in contour between slices
@@ -26,8 +26,13 @@ class IRPetalParam(object):
         self.do_fit = True        # True to fit the segment points.
         self.rotate = True        # Rotate to have a vertical petal in mirror image
         self.debug = False        # To debug
-        self.report = False       # 
+        self.report = False       #
         self.graphana = None      # Graphana server
+        self.save_pipes = False   # If true save pipe path
+        self.legend = True        # if false do not plot legend
+        self.sensor_min = -sys.float_info.max # cut on lower temp for sensor contour average
+        self.sensor_max = sys.float_info.max # cut on higher temp for sensor contour average
+
 
         if values is not None:
             self.set_values(values)
@@ -75,3 +80,7 @@ class IRPetalParam(object):
                             help="Value to smooth contour")
         parser.add_argument("--debug", action="store_true", default=False, help="Show additional information.")
         parser.add_argument("--report", action="store_true", default=False, help="True if figures kept for the report.")
+        parser.add_argument("--save_pipes", default=False, action="store_true", help="SAve pipe path. Output is alias_pipe-i.txt")
+        parser.add_argument("--no-legend", dest="legend", action="store_false", default=True, help="Do not show the legend in plots.")
+        parser.add_argument("--sensor_min", dest="sensor_min", default=P.sensor_min, help="cut on lower temp for sensor contour average")
+        parser.add_argument("--sensor_max", dest="sensor_max", default=P.sensor_max, help="cut on higher temp for sensor contour average")

petal_qc/thermal/Petal_IR_Analysis.py

@@ -88,7 +88,7 @@ def analyze_IR_image(img, pipe, sensors, iside, params) -> AnalysisResult:
     Savg = []
     Sstd = []
     for S in sensors:
-        avg, std = contours.get_average_in_contour(img, S)
+        avg, std = contours.get_average_in_contour(img, S, tmin=params.sensor_min, tmax=params.sensor_max)
         Savg.append(avg)
         Sstd.append(std)
 

petal_qc/thermal/PipeFit.py

@@ -44,6 +44,8 @@ class PipeFit(object):
 
         self.core_center = None
         self.core_band = None
+        self.sample_indx = None
+        self.curv = []
 
     def set_front(self, is_front=True):
         """Sets for a front image.
@@ -357,7 +359,8 @@ class PipeFit(object):
             dst, P = contours.find_closest_point(X[0], X[1], self.cpipe)
             D[i, :] = P - X
             ddd[i] = dst
-            W = 1 + 1.6*(y_max - X[1])/height
+            W = 1 + 2*(y_max - X[1])/height
+            #W = self.curv[i]
             weights[i] = W
             sum_weights += W
 
@@ -369,7 +372,7 @@ class PipeFit(object):
             res = np.dot(ddd, weights)/sum_weights
 
         if self.debug:
-            dbg_ax. clear()
+            dbg_ax.clear()
             dbg_ax.plot(self.cpipe[:, 0], self.cpipe[:, 1])
             dbg_ax.plot(out[:, 0], out[:, 1])
             dbg_ax.set_title("area {:3f} dist {:.3f}".format(real_area, np.sum(ddd)/npts))
@@ -420,7 +423,7 @@ class PipeFit(object):
 
         return res
 
-    def fit(self, data, M0=None, factor=1.0):
+    def fit(self, data, M0=None, factor=1.0, simplify=True):
         """Do the fit.
 
         Args:
@@ -451,12 +454,19 @@ class PipeFit(object):
                 print("\n** Initial guess")
                 self.print_transform(M)
         else:
-            M = M0
+            M = M0[0:5]
 
         self.core_center, self.core_band = self.get_data_center(data)
 
-
-        self.data = contours.contour_simplify(data, 1.25*factor)
+        if simplify:
+            self.sample_indx = contours.contour_simplify(data, 1.25, return_mask=True)
+            self.data = np.array(data[self.sample_indx, :])
+            
+        else:
+            self.data = np.array(data)
+            
+        self.curv = contours.contour_curvature(self.data)
+        
         # self.data = data
         verbose = 0
         if self.debug:
@@ -482,9 +492,9 @@ class PipeFit(object):
         self.R = M
         return M
 
-    def fit_ex(self, data, limit=5e6, factor=1):
+    def fit_ex(self, data, M0=None, limit=5e6, factor=1, simplify=True):
         """Does the regular fit and tries to correct."""
-        R = self.fit(data, factor=factor)
+        R = self.fit(data, factor=factor, M0=M0, simplify=simplify)
 
         # Check for an offset...
         if self.res.cost > limit:
@@ -535,9 +545,13 @@ class PipeFit(object):
 
         return X, Y
 
-    def plot(self, show_fig=True):
+    def plot(self, show_fig=True, ax=None):
         """Plot result of fit and data."""
-        fig, ax = plt.subplots(1, 1, tight_layout=True)
+        if ax is None:
+            fig, ax = plt.subplots(1, 1, tight_layout=True)
+        else:
+            fig = ax.get_figure()
+            
         ax.plot(self.pipe[:, 0], self.pipe[:, 1], label="Pipe")
         if self.R is not None:
             out = self.transform_data(self.data, self.R)

petal_qc/thermal/PipeIterFit.py

@@ -0,0 +1,94 @@
+import sys
+from pathlib import Path
+import numpy as np
+import matplotlib.pyplot as plt
+
+
+
+cwd = Path(__file__).parent
+if cwd.exists():
+    sys.path.insert(0, cwd.as_posix())
+
+from petal_qc.thermal import PipeFit, contours, DebugPlot
+
+# create a global instance of DebugPlot
+debug_plot = DebugPlot.DebugPlot()
+
+
+class PipeIterFit:
+    """makes an iterative fit removing outliers in each iteration."""
+
+    def __init__(self, data):
+        """Initialize class."""
+        self.data = data
+        ptype = PipeFit.PipeFit.guess_pipe_type(data)
+        self.PF = PipeFit.PipeFit(ptype)
+
+    def remove_outsiders(self, data, thrs):
+        """Removes points which are further than thrs from the fit."""
+        D = np.zeros(len(data))
+        out = self.PF.transform_data(data, self.PF.R)
+        i = 0
+        for x, y in out:
+            dst, P = contours.find_closest_point(x, y, self.PF.pipe)
+            D[i] = dst
+            i += 1
+
+        indx = np.where(D < thrs)[0]
+        return np.array(data[indx, :])
+
+    def fit(self, threshold=20, factor=1.0, debug=False):
+        """Do the fit."""
+        global debug_plot
+        if debug:
+            debug_plot.setup_debug('IterFit')
+        
+        
+        total_data = self.data
+        data_size = len(total_data)
+
+        M0 = self.PF.fit_ex(total_data, factor=factor, simplify=True)
+        sample_data = self.remove_outsiders(self.PF.data, threshold)
+        last_size = len(sample_data)
+        if debug:
+            ax = debug_plot.get_ax("IterFit")
+            ax.clear()
+            ax.plot(total_data[:, 0], total_data[:, 1], 'o')
+            ax.plot(sample_data[:, 0], sample_data[:, 1], 'P')
+            plt.draw()
+            plt.pause(0.0001)
+            
+        # Adaptively determining the number of iterations
+        while True:
+            M0 = self.PF.fit_ex(sample_data, M0=M0, factor=factor, simplify=False)
+
+            out = self.PF.transform_data(self.PF.data, M0)
+            D = []
+            for x, y in out:
+                dst, P = contours.find_closest_point(x, y, self.PF.pipe)
+                D.append(dst)
+
+            if debug:
+                ax = debug_plot.get_ax("IterFit")
+                ax.clear()
+                ax.hist(D)
+                plt.draw()
+                plt.pause(0.0001)
+                self.PF.plot(ax=ax)
+            
+            sample_data = self.remove_outsiders(self.PF.data, 20)
+            sample_size = len(sample_data)
+            if sample_size == last_size:
+                break
+
+            last_size = sample_size
+
+        if debug:
+            self.PF.plot(ax=ax)
+            
+        return M0
+
+    def plot(self):
+        """Plot this fit."""
+        global debug_plot
+        self.PF.plot(ax=debug_plot.get_ax("IterFit"))

petal_qc/thermal/contours.py

@@ -8,6 +8,8 @@ import random
 import sys
 
 import matplotlib.path as mplPath
+import matplotlib.pyplot as plt
+
 import numpy as np
 
 from petal_qc.utils.Geometry import Point, remove_outliers_indx
@@ -258,8 +260,46 @@ def in_contour(x, y, C):
     path = mplPath.Path(C)
     return path.contains_point(x, y, C)
 
+def show_contour_values(img, C):
+    """Creates an image of the contour values.
+
+    Args:
+    ----
+        img: The image
+        C: The contour
 
-def get_average_in_contour(img, C, remove_outliers=None):
+
+    """
+    values = []
+    path = mplPath.Path(C)
+    my, mx = img.shape
+    min_x, min_y, max_x, max_y = contour_bounds(C)
+    imin_x = int(min_x)
+    imin_y = int(min_y)
+    chst = np.zeros([int(max_y-min_y+2), int(max_x-min_x+2)])
+    for ix in range(int(min_x), int(max_x+1)):
+        if ix < 0 or ix >= mx:
+            continue
+
+        for iy in range(int(min_y), int(max_y)+1):
+            if iy < 0 or iy >= my:
+                continue
+
+            if path.contains_point([ix, iy]):
+                chst[iy-imin_y, ix-imin_x] = img[iy, ix]
+                values.append(img[iy, ix])
+
+    values = np.array(values)
+    mean = np.mean(values)
+    std = np.std(values)
+
+    fig, ax = plt.subplots(nrows=1,ncols=2)
+    ax[0].hist(values, 25, range=(mean-3*std, mean+3*std))
+    pcm = ax[1].imshow(chst, origin='lower', vmin=mean-3*std, vmax=mean+3*std)
+    fig.colorbar(pcm, ax=ax[1])
+
+
+def get_average_in_contour(img, C, tmin=sys.float_info.min, tmax=sys.float_info.max, remove_outliers=None):
     """Gets average and std of points within contour.
 
     We are assuming here that coordinates are integers, ie,
@@ -269,7 +309,8 @@ def get_average_in_contour(img, C, remove_outliers=None):
     ----
         img: The image
         C: The contour
-        remove_outliers: If an int, 
+        tmin, tmax: only values in range [tmin, tmax] will be considered
+        remove_outliers: If an int,
 
     Returns
     -------
@@ -289,7 +330,9 @@ def get_average_in_contour(img, C, remove_outliers=None):
                 continue
 
             if path.contains_point([ix, iy]):
-                values.append(img[iy, ix])
+                val = img[iy, ix]
+                if val > tmin and val < tmax:
+                    values.append(val)
 
     values = np.array(values)
     if remove_outliers is not None and isinstance(remove_outliers, (int, float)):
@@ -365,6 +408,42 @@ def contour_eval(C, x):
     return value
 
 
+
+def contour_curvature(contour, stride=1):
+    """Computes contour curvature at each point.
+
+    Args:
+        contour: Input contour
+        stride (int, optional): stride to compute curvature. Defaults to 1.
+
+    Returns:
+        curvature: aray of curvatures.
+        
+    """
+    curvature = []
+    assert stride < len(contour), "stride must be shorther than length of contour"
+    
+    npts = len(contour)
+    for i in range(npts):
+        before = i - stride + npts if i - stride < 0 else i - stride
+        after = i + stride - npts if i + stride >= npts else i + stride
+
+        f1x, f1y = (contour[after] - contour[before]) / stride
+        f2x, f2y = (contour[after] - 2 * contour[i] + contour[before]) / stride**2
+        denominator = (f1x**2 + f1y**2) ** 3 + 1e-11
+
+        curvature_at_i = (
+            np.sqrt(4 * (f2y * f1x - f2x * f1y) ** 2 / denominator)
+            if denominator > 1e-12
+            else -1
+        )
+
+        curvature.append(curvature_at_i)
+
+
+    return curvature
+
+
 if __name__ == "__main__":
     cntr = np.loadtxt("long_contour.csv")
 

petal_qc/thermal/coreThermal.py

@@ -258,7 +258,7 @@ class CoreThermal(itkdb_gtk.dbGtkUtils.ITkDBWindow):
             if upper:
                 self.dataBox.remove(self.btnData)
                 self.dataBox.add(self.desyData)
-            
+
             self.param.distance = 16
             self.param.width = 16
 
@@ -268,7 +268,7 @@ class CoreThermal(itkdb_gtk.dbGtkUtils.ITkDBWindow):
             if upper:
                 self.dataBox.remove(self.desyData)
                 self.dataBox.add(self.btnData)
-            
+
             self.param.distance = 5
             self.param.width = 2