petal-qc 0.0.3__py3-none-any.whl → 0.0.5__py3-none-any.whl

petal_qc/metrology/compare_Cores.py

@@ -11,22 +11,55 @@ import matplotlib.pyplot as plt
 
 
 def get_value(data, value_path):
-    """Get the value."""
+    """Get the value from the path given."""
     a = data
     for k in value_path.split('/'):
         a = a[k]
 
     return a
 
+def save_figure(fig, fnam, prefix=None, dpi=192):
+    """Saves the figure
 
-def main(options):
+    Args:
+        fig (Figure): The figure to save
+        fnam (str): The name of the output file
+        dpi (int, optional): The Dots por Inch. Defaults to 300.
+    """
+    if fnam is not None:
+        if prefix:
+            P = Path(fnam).expanduser().resolve()
+            name = prefix + P.name
+            out = P.parent / name
+            fnam = out
+
+        print("out: {}".format(fnam))
+        fig.savefig(fnam, dpi=dpi)
+
+
+def read_data_files(options):
+    """Read the data files.
+
+    It assumes the data file names are in the format:
+
+    <AlternativeID>-<side>.json
+
+    AlternativeID is PPC.nnn side is either front or back.
+
+    Returns the values for front and back petal tests.
+
+    The results are a dictionary with the file label as key and the requested
+    value, which can be a number of an array.
+
+    the file labels are also returned.
+
+    """
     labels = []
     front = {}
     back = {}
-    
+
     for fnam in options.files:
         ifile = Path(fnam).expanduser().resolve()
-        print(ifile.name)
         if not ifile.exists():
             print("File does not exist: ", fnam)
             continue
@@ -43,54 +76,160 @@ def main(options):
         label = tmp[0]
         if not label in labels:
             labels.append(label)
-            
-        is_front = False
-        if "front" in tmp[1].lower():
-            is_front = True
-            
+
         val = get_value(data, options.value)
-        if is_front:
+        if "front" in tmp[1].lower():
             front[label] = val
         else:
             back[label] = val
 
     labels.sort()
-    X = np.arange(0, len(labels))
-    fig, ax = plt.subplots(1, 1, tight_layout=True)
+
+    return front, back, labels
+
+def draw_deltas(data, keys, fnam=None, title="Front"):
+    """Plot the  position deltas."""
+    key_table = {"Bot.": "PL01", "Slot": "PL02", "Top": "PL03",
+                 "Bot-FD01": "PL01-FD01", "Bot-FD02": "PL01-FD02", "FD01-FD02": "FD01-FD02" }
+    nfiles = len(data)
+
+    P = [np.zeros([nfiles, 2]),
+         np.zeros([nfiles, 2]),
+         np.zeros([nfiles, 2])]
+    fig_width = 12.0
+    fig_height = 1.2*fig_width/3.0
+    fig, ax = plt.subplots(nrows=1, ncols=3, tight_layout=True, figsize=(fig_width, fig_height))
+    fig.suptitle(title)
+    for i in range(3):
+        LBL = [[],[],[]]
+        ax[i].set_title(keys[i])
+        ax[i].set_aspect('equal', adjustable='box')
+        ax[i].set_xlim(-100, 100)
+        ax[i].set_ylim(-100, 100)
+        circle = plt.Circle((0,0), 50, color="red", alpha=0.25)
+        ax[i].add_patch(circle)
+        circle = plt.Circle((0,0), 25, color="green", alpha=0.25)
+        ax[i].add_patch(circle)
+
+        ax[i].set_xlabel("X (µm)")
+        ax[i].set_ylabel("Y (µm)")
+        ax[i].grid()
+
+        for j, v in enumerate(data.items()):
+            label, values = v
+            for k in range(3):
+                ky = key_table[keys[k]]
+                P[k][j, :] = 1000*np.array(values[ky])
+                LBL[k].append(label.split('.')[1].lstrip('0'))
+
+        ax[i].scatter(P[i][:,0], P[i][:,1])
+        for j in range(len(LBL[i])):
+            ax[i].text(P[i][j,0], P[i][j,1], LBL[i][j]) #, ha='center', va='top')
+
+    save_figure(fig, fnam, prefix=title)
+
+
+def show_positions(options):
+    """Make position plots."""
+
+    if "LOCATION" in options.value:
+        keys = ["Bot.", "Slot", "Top"]
+    elif "REL_POS" in options.value:
+        keys = ["Bot-FD01", "Bot-FD02", "FD01-FD02"]
+    else:
+        print("Invalid value")
+        return
+
+    front, back, labels = read_data_files(options)
+    val_name = options.value.split('/')[-1]
+    draw_deltas(front, keys, fnam=options.out, title="{} - Front".format(val_name))
+    draw_deltas(back, keys, fnam=options.out, title="{} - Back".format(val_name))
+
+    plt.show()
+
+def show_flatness(options):
+    """Show flatness plots."""
+    tick_labels =  ["R0", "R1", "R2", "R3S0", "R3S1", "R4S0", "R4S1", "R5S0", "R5S1"]
+    cindx = ["Front", "Back"]
+
+    front, back, labels = read_data_files(options)
+    nfiles = len(labels)
+    npts = len(tick_labels)
+    X = np.array([float(x) for x in range(npts)])
+
+
+    fig, ax = plt.subplots(nrows=1, ncols=2, tight_layout=True, figsize=(9., 5.))
     fig.suptitle(options.value)
-    ax.set_xticks(range(len(labels)), labels=labels)
-    ax.grid()
-    
-    vfront = [front[x] for x in labels]
-    vback = [back[x] for x in labels]
-    ax.plot(X, vfront, '*', label="Front")
-    ax.plot(X, vback, 'o', label="Back")
-    ax.legend()
-    
-    if options.out:
-        fig.savefig(options.out, dpi=300)
-    
+
+    P = [ np.zeros([nfiles, npts]), np.zeros([nfiles, npts]) ]
+    y_lim = []
+    for i, V in enumerate([front, back]):
+        ax[i].set_title(cindx[i])
+        ax[i].set_xticks(range(npts), labels=tick_labels)
+        ax[i].grid()
+
+        for lbl in labels:
+            ax[i].plot(X, V[lbl], '-', label=lbl)
+
+        y_lim.append(ax[i].get_ylim())
+
+    for a in ax:
+        a.set_ylim(0, 1.2*max(y_lim[0][1], y_lim[1][1]))
+        x_lim = a.get_xlim()
+        a.fill_between(x_lim, 0, 0.050, facecolor="darkseagreen", alpha=0.1)
+        a.legend(ncol=3, fontsize="x-small")
+
+    save_figure(fig, options.out, prefix=opts.prefix)
+
+def main(options):
+    """Main entry."""
+
+    if "LOCATION" in options.value or "REL_POS" in options.value:
+        show_positions(options)
+
+    elif "FLATNESS_LOCAL" in options.value:
+        show_flatness(options)
+
+    else:
+        front, back, labels = read_data_files(options)
+
+        labels.sort()
+        X = np.arange(0, len(labels))
+        fig, ax = plt.subplots(1, 1, tight_layout=True)
+        fig.suptitle(options.value)
+        ax.set_xticks(range(len(labels)), labels=labels, rotation="vertical")
+        ax.grid()
+
+        vfront = [front[x] for x in labels]
+        vback = [back[x] for x in labels]
+        ax.plot(X, vfront, '*', label="Front")
+        ax.plot(X, vback, 'o', label="Back")
+        ax.legend()
+
+        save_figure(fig, options.out, prefix=options.prefix)
+
     plt.show()
-    
+
 
 if __name__ == "__main__":
     # Argument parser
     parser = argparse.ArgumentParser()
     parser.add_argument('files', nargs='*', help="Input files")
     parser.add_argument("--value", default=None, help="Value to plot")
+    parser.add_argument("--prefix", default=None, help="prefix for out file")
     parser.add_argument("--out", default=None, help="File to store the figure.")
 
-    options = parser.parse_args()
-    if len(options.files) == 0:
+    opts = parser.parse_args()
+    if len(opts.files) == 0:
         print("I need at least one input file")
         sys.exit()
 
-    if len(options.files) == 1:
-        xxx = any(elem in options.files[0] for elem in r"*?")
+    elif len(opts.files) == 1:
+        xxx = any(elem in opts.files[0] for elem in r"*?")
         if xxx:
-            options.files = glob.glob(options.files[0])
+            opts.files = glob.glob(opts.files[0])
 
-    if options.value[0] == '/':
-        options.value = options.value[1:]
-    options.value = "results/METROLOGY/" + options.value
-    main(options)
+    if opts.value[0] == '/':
+        opts.value = opts.value[1:]
+    opts.value = "results/METROLOGY/" + opts.value
+    main(opts)

petal_qc/metrology/convert_smartscope.py

@@ -10,15 +10,15 @@ import matplotlib.pyplot as plt
 
 
 from petal_qc.metrology import DataFile
-from .analyze_locking_points import analyze_locking_point_data
+#from .analyze_locking_points import analyze_locking_point_data
 
 def get_data_chunck(flist, data_type, ofile):
     """Get one data type chunk from file
 
     Args:
-        fname (): input file
-        data_tyle (): data type
-        ofile (): file object for output.
+        fname: input file
+        data_tyle: data type
+        ofile: file object for output.
 
     """
     rgx = "Step Name:\\s+({}).*?=".format(data_type)
@@ -41,7 +41,7 @@ def get_data_chunck(flist, data_type, ofile):
                 ofile.write(txt.group(0)+'\n')
     
 
-def read_smartscope(fnam, ofile, data_type, keep=False,):
+def read_smartscope(fnam, ofile, data_type, keep=False):
     """Convert a SmartScope txt file into CVS
 
     Args:
@@ -58,7 +58,11 @@ def read_smartscope(fnam, ofile, data_type, keep=False,):
     if isinstance(ofile, io.IOBase):
         fout = ofile
     else:
-        fout = open(ofile, 'w')
+        try:
+            fout = open(ofile, 'w', encoding="utf-8")
+        except TypeError:
+            fout = ofile
+
         is_file = True
 
     if keep:
@@ -106,7 +110,84 @@ def read_smartscope(fnam, ofile, data_type, keep=False,):
             values = [ float(x) for x in items[2:5]]
             fout.write("{:6f}, {:6f}, {:6f}\n".format(values[1], values[0], values[2]))
 
-    fout.close()
+    if is_file:
+        fout.close()
+    
+def get_smarscope_locator_positions(fnam, ofile, data_type, keep=False):
+    """REad DESY 2D file.
+
+    Args:
+        fnam: Input file
+        ofile: Output file
+        data_type: Label to search
+        keep (bool, optional): If tre keep label. Defaults to False.
+    """
+    ifile = Path(fnam).expanduser().resolve()
+    
+    is_file = False
+    if isinstance(ofile, io.IOBase):
+        fout = ofile
+    else:
+        try:
+            fout = open(ofile, 'w', encoding="utf-8")
+        except TypeError:
+            fout = ofile
+
+        is_file = True
+
+    if keep:
+        fout.write("X,Y,Z, label\n")
+    else:
+        fout.write("X,Y,Z\n")
+
+    rgx = "^Step Name:\\s+(?P<label>{})".format(data_type)
+    start = re.compile(rgx, re.DOTALL)
+    finder = re.compile("^Finder Name")
+
+    out = {}
+    with open(ifile, 'r', encoding='ISO-8859-1') as fin:
+        while True:
+            line = fin.readline()
+            if not line:
+                break
+
+            r = start.match(line)
+            if not r:
+                continue
+            
+            label = r.group('label').strip()
+            eof = False
+            out[label] = [0. for i in range(3)]
+            for i in range(3):
+                line = fin.readline()
+            
+            while True:
+                line = fin.readline().strip()
+                if len(line)==0:
+                    break
+            
+                values = line.split()
+                val = float(values[3])
+                if values[0] in ["Diame", "Width"]:
+                    out[label][2] = val
+                elif values[0] == "X":
+                    out[label][1] = val
+                elif values[0] == "Y":
+                    out[label][0] = val
+                    
+    lbls = ["BottomPL", "SlotPL", "OversizedPL", "Bottom_Fiducial", "Slot_Fiducial"]
+    for L in lbls:
+        try:
+            V = [float(x) for x in out[L]]
+            fout.write("{:.6f}, {:.6f}, {:.6f}".format(V[0], V[1], V[2]))
+            if keep:
+                fout.write(", {}".format(L))
+            fout.write('\n')
+        except KeyError:
+            continue
+                          
+    if is_file:
+        fout.close()
 
 def do_locking_points(args):
     ifile = args.smartscope_files[0]
@@ -120,14 +201,14 @@ if __name__ == "__main__":
     import argparse
 
     parser = argparse.ArgumentParser()
-    parser.add_argument("smartscope_files", nargs='*',
+    parser.add_argument("files", nargs='*',
                         help="The SmartScope files to parse")
     parser.add_argument("--label", default="\\w+", help="The label to select")
     parser.add_argument("--out", help="Output CSV file", default="out.csv")
     parser.add_argument("--keep_label", dest="keep", default=False, action="store_true", help="Store label in output")
 
     args = parser.parse_args()
-    if len(args.smartscope_files) == 0:
+    if len(args.files) == 0:
         print("I need an input file")
         sys.exit()
 
@@ -135,11 +216,13 @@ if __name__ == "__main__":
 
     # do_locking_points(args)
     
-    fout = open(Path(args.out).expanduser().resolve(), 'w')
-    get_data_chunck(args.smartscope_files, args.label, fout)
-    fout.close()
+    #fout = open(Path(args.out).expanduser().resolve(), 'w')
+    #get_data_chunck(args.smartscope_files, args.label, fout)
+    #fout.close()
     
     
     #read_smartscope(args.smartscope_files[0], args.out, args.label, args.keep)
 
+    get_smarscope_locator_positions(args.files[0], args.out, args.label)
+
     plt.show()

petal_qc/metrology/coreMetrology.py

@@ -16,7 +16,7 @@ import gi
 gi.require_version("Gtk", "3.0")
 from gi.repository import Gtk, GObject, Gio, GLib
 
-
+__HELP__ = "https://petal-qc.docs.cern.ch"
 
 class CommaSeparatedListAction(Action):
     """Create a list from the comma sepparated numbers at imput."""
@@ -41,7 +41,7 @@ class CoreMetrology(itkdb_gtk.dbGtkUtils.ITkDBWindow):
             pannel_size: size of message panel.
 
         """
-        super().__init__(session=session, title=title)
+        super().__init__(session=session, title=title, help=__HELP__)
 
         self.data_file = None
         self.folder = None

petal_qc/metrology/do_Metrology.py

@@ -42,13 +42,13 @@ def do_analysis(fnam, prefix, SN, options):
     """Perform analysis of a file.
 
     Args:
-        fnam (): Input data file
-        prefix (): Prefix telling if it is front or back
-        SN (): Core serial number
-        options (): Options.
-        
+        fnam: Input data file
+        prefix: Prefix telling if it is front or back
+        SN: Core serial number
+        options: Options.
+
     """
-    
+
     is_front = prefix.lower().find("front") >= 0
     print(fnam, prefix)
     options.out = prefix + '.docx'
@@ -63,10 +63,10 @@ def do_analysis(fnam, prefix, SN, options):
     ofile = output_folder(options.folder, prefix + '.json')
     with open(ofile, 'w', encoding='UTF-8') as of:
         json.dump(outDB, of, indent=3)
-    
+
     return outDB
 
-def main(ifile, options):
+def analyze_files(ifile, options):
     """Main entry."""
     with open(ifile, 'r', encoding='UTF-8') as inp:
 
@@ -86,25 +86,25 @@ def main(ifile, options):
 
             do_analysis(fnam, prefix, SN, options)
 
-
-if __name__ == "__main__":
+def main():
+    "Main entry."
     parser = ArgumentParser()
     parser.add_argument('files', nargs='*', help="Input files")
-    parser.add_argument("--prefix", dest='prefix', default=None)
-    parser.add_argument("--SN", dest='SN', default=None)
+    parser.add_argument("--prefix", dest='prefix', default=None, help="prefix telling if it is front or back.")
+    parser.add_argument("--SN", dest='SN', default=None, help="The petal core Serial Number")
     parser.add_argument("--save", dest='save', action="store_true", default=False)
-    parser.add_argument("--desy", dest='desy', action="store_true", default=False)
+    parser.add_argument("--desy", dest='desy', action="store_true", default=False, help="True if data is from DESY's SmartScope")
     parser.add_argument("--out", dest="out", default="petal_flatness.docx",
-                        type=str, help="The output fiel name")
+                        type=str, help="The output file name")
     parser.add_argument("--bottom-lp", dest='bLP', action=CommaSeparatedListAction, default=None,
                         help="Bottom locking point fiducial coordinates")
     parser.add_argument("--upper-lp", dest='uLP', action=CommaSeparatedListAction, default=None,
                         help="upper locking point fiducials coordinates")
 
     parser.add_argument("--title", dest="title", default=None,
-                        type=str, help="Document title")
+                        type=str, help="Report Document title")
     parser.add_argument("--nbins", dest="nbins", default=25,
-                        type=int, help="Number of bins")
+                        type=int, help="Number of bins in the histograms")
     parser.add_argument("--folder", default=None, help="Folder to store output files. Superseeds folder in --out")
     parser.add_argument("--locking_points", action="store_true", default=False)
 
@@ -119,7 +119,10 @@ if __name__ == "__main__":
         sys.exit()
 
     try:
-        main(options.files[0], options)
+        analyze_files(options.files[0], options)
 
     except Exception:
         print(traceback.format_exc())
+
+if __name__ == "__main__":
+    main()

petal_qc/metrology/petal_flatness.py

@@ -162,7 +162,12 @@ def petal_flatness(orig_data, options, document=None):
     flatness_func = flatness_LSPL
     F = {}
     for key, val in sensors.items():
-        F[key] = flatness_func(val)
+        try:
+            F[key] = flatness_func(val)
+        except ValueError as E:
+            print("*** Error: Petal flatnes: key {}\n{}".format(key, E))
+            print(val)
+            return TM, avg, Zmean, [-9999, -9999]
 
     flatness_all_sensor_area = flatness_func(all_data)
 
@@ -182,7 +187,7 @@ def petal_flatness(orig_data, options, document=None):
             outF.extend(values)
             print("{}: {:.3f}  {}: {:.3f}".format(lbls[0], values[0], lbls[1], values[1]))
 
-    print("All sensor area: {:.3f}".format(flatness_all_sensor_area))
+    print("All sensor area: {:.3f}".format(1000*flatness_all_sensor_area))
 
     # Add table in document
     if document:
@@ -213,7 +218,7 @@ def petal_flatness(orig_data, options, document=None):
         # add allsensor area
         cell = table.add_row().cells
         cell[0].text = "All sensor area"
-        cell[1].text = "{:.4f}".format(flatness_all_sensor_area)
+        cell[1].text = "{:.4f}".format(flatness_all_sensor_area*1000)
 
     return TM, avg, Zmean, outF
 

petal_qc/test/prepareDESYfiles.py

@@ -0,0 +1,86 @@
+#!/usr/bin/env python3
+"""Prepara input files from DESY RAW."""
+import os
+import fnmatch
+import re
+from pathlib import Path
+
+def all_files(root, patterns='*', single_level=False, yield_folders=False):
+    """A generator that reruns all files in the given folder.
+
+    Args:
+    ----
+        root (file path): The folder
+        patterns (str, optional): The pattern of the files. Defaults to '*'.
+        single_level (bool, optional): If true, do not go into sub folders. Defaults to False.
+        yield_folders (bool, optional): If True, return folders as well. Defaults to False.
+
+    Yields
+    ------
+        str: file path name
+
+    """
+    patterns = patterns.split(';')
+    for path, subdirs, files in os.walk(root):
+        if yield_folders:
+            files.extend(subdirs)
+
+        files.sort()
+        for name in files:
+            for pattern in patterns:
+                if fnmatch.fnmatch(name, pattern):
+                    yield os.path.join(path, name)
+                    break
+
+        if single_level:
+            break
+
+class PetalCore:
+    def __init__(self, front, back):
+        self.front = {}
+        self.back = {}
+
+def main(folder, out_folder):
+    """Main entry point."""
+    outF = Path(out_folder).expanduser().resolve()
+    if not outF.exists():
+        os.mkdir(outF)
+        
+    rgx = re.compile(r"Project Name: (\w+)side_.*AlternativeID=PPC-(\d+)", re.MULTILINE|re.DOTALL)
+    petal_cores = {}
+    for fnam in all_files(folder, "*.txt"):
+        P = Path(fnam).expanduser().resolve()
+        with open(fnam, "r", encoding="UTF-8") as ff:
+            R = rgx.search(ff.read())
+            if R:
+                petal_id = "PPC.{}".format(R.group(2))
+                side = R.group(1).lower()
+                if "_2D_" in P.name:
+                    test_type = "2D"
+                else:
+                    test_type = "3D"
+
+                if not petal_id in petal_cores:
+                    petal_cores[petal_id] = {
+                                             "back":{"2D":None, "3D":None},
+                                             "front":{"2D":None, "3D":None},
+                                             }
+
+                petal_cores[petal_id][side][test_type] = P
+
+    list_file = open("{}/Desy-cores.txt".format(outF.as_posix()), "w", encoding="UTF-8")
+    for petal_id, sides in petal_cores.items():
+        for side, values in sides.items():
+            oname = "{}/{}-{}.txt".format(outF.as_posix(), petal_id, side)
+            list_file.write("{} {}-{} {}\n".format(oname, petal_id, side, petal_id))
+            data = ""
+            for data_type, fnam in values.items():
+                with open(fnam, "r", encoding="UTF-8") as ifile:
+                    data += ifile.read()
+                    
+            with open(oname, "w", encoding="UTF-8") as ofile:
+                ofile.write(data)
+       
+    list_file.close()
+if __name__ == "__main__":
+    main("/Users/lacasta/Downloads/DESY_Production", "/tmp/desy")

petal_qc/thermal/IRDataGetter.py

@@ -133,7 +133,8 @@ class IRDataIFIC(IRDataGetter):
         super().__init__()
         self.analysis_frame = None
         if HAS_GRAPHANA:
-            self.DB = ReadGraphana("localhost")
+            #self.DB = ReadGraphana("localhost")
+            self.DB = ReadGraphana()
         else:
             self.DB = None
 
@@ -156,9 +157,10 @@ class IRDataIFIC(IRDataGetter):
         """
         irbf.set_concatenate(True)
         frame = self.get_analysis_frame(irbf)
-        i_min = self.indx
+        i_min = self.indx[-1]
         min_T = np.min(frame[0].image)
-        return min_T, i_min, frame
+        values = self.get_IR_data(frame[0])
+        return min_T, i_min, [values, ]
 
         #  if len(args) == 0:
         #      T_min = -22.0

petal_qc/thermal/IRPetal.py

@@ -551,7 +551,7 @@ def get_T_profile(data, A, B, npts=10, do_fit=False, npdim=7, debug=False):
                         fmin.success = True
                         break
 
-            # If the re is aclear minimum within the segment
+            # If there is a clear minimum within the segment
             if fmin.success and fmin.x[0] > 0 and fmin.x[0] < mx_dist:
                 # This should no happen. if minimize returns we should have found
                 # at least a valid root.