petal-qc 0.0.4__py3-none-any.whl → 0.0.5__py3-none-any.whl

petal_qc/metrology/compare_Cores.py

@@ -32,27 +32,27 @@ def save_figure(fig, fnam, prefix=None, dpi=192):
             name = prefix + P.name
             out = P.parent / name
             fnam = out
-            
+
         print("out: {}".format(fnam))
         fig.savefig(fnam, dpi=dpi)
-        
+
 
 def read_data_files(options):
     """Read the data files.
-    
+
     It assumes the data file names are in the format:
-    
+
     <AlternativeID>-<side>.json
-    
+
     AlternativeID is PPC.nnn side is either front or back.
-    
+
     Returns the values for front and back petal tests.
-    
+
     The results are a dictionary with the file label as key and the requested
-    value, which can be a number of an array. 
-    
+    value, which can be a number of an array.
+
     the file labels are also returned.
-    
+
     """
     labels = []
     front = {}
@@ -60,7 +60,6 @@ def read_data_files(options):
 
     for fnam in options.files:
         ifile = Path(fnam).expanduser().resolve()
-        print(ifile.name)
         if not ifile.exists():
             print("File does not exist: ", fnam)
             continue
@@ -83,9 +82,9 @@ def read_data_files(options):
             front[label] = val
         else:
             back[label] = val
-            
+
     labels.sort()
-    
+
     return front, back, labels
 
 def draw_deltas(data, keys, fnam=None, title="Front"):
@@ -97,12 +96,12 @@ def draw_deltas(data, keys, fnam=None, title="Front"):
     P = [np.zeros([nfiles, 2]),
          np.zeros([nfiles, 2]),
          np.zeros([nfiles, 2])]
-
     fig_width = 12.0
     fig_height = 1.2*fig_width/3.0
     fig, ax = plt.subplots(nrows=1, ncols=3, tight_layout=True, figsize=(fig_width, fig_height))
     fig.suptitle(title)
     for i in range(3):
+        LBL = [[],[],[]]
         ax[i].set_title(keys[i])
         ax[i].set_aspect('equal', adjustable='box')
         ax[i].set_xlim(-100, 100)
@@ -121,15 +120,18 @@ def draw_deltas(data, keys, fnam=None, title="Front"):
             for k in range(3):
                 ky = key_table[keys[k]]
                 P[k][j, :] = 1000*np.array(values[ky])
+                LBL[k].append(label.split('.')[1].lstrip('0'))
 
         ax[i].scatter(P[i][:,0], P[i][:,1])
+        for j in range(len(LBL[i])):
+            ax[i].text(P[i][j,0], P[i][j,1], LBL[i][j]) #, ha='center', va='top')
 
     save_figure(fig, fnam, prefix=title)
 
 
 def show_positions(options):
     """Make position plots."""
-    
+
     if "LOCATION" in options.value:
         keys = ["Bot.", "Slot", "Top"]
     elif "REL_POS" in options.value:
@@ -154,58 +156,58 @@ def show_flatness(options):
     nfiles = len(labels)
     npts = len(tick_labels)
     X = np.array([float(x) for x in range(npts)])
-    
-    
+
+
     fig, ax = plt.subplots(nrows=1, ncols=2, tight_layout=True, figsize=(9., 5.))
     fig.suptitle(options.value)
-    
+
     P = [ np.zeros([nfiles, npts]), np.zeros([nfiles, npts]) ]
     y_lim = []
     for i, V in enumerate([front, back]):
         ax[i].set_title(cindx[i])
         ax[i].set_xticks(range(npts), labels=tick_labels)
         ax[i].grid()
-        
+
         for lbl in labels:
             ax[i].plot(X, V[lbl], '-', label=lbl)
-             
+
         y_lim.append(ax[i].get_ylim())
-        
+
     for a in ax:
         a.set_ylim(0, 1.2*max(y_lim[0][1], y_lim[1][1]))
+        x_lim = a.get_xlim()
+        a.fill_between(x_lim, 0, 0.050, facecolor="darkseagreen", alpha=0.1)
         a.legend(ncol=3, fontsize="x-small")
-        
-    save_figure(fig, options.out, prefix="zzz-")
+
+    save_figure(fig, options.out, prefix=opts.prefix)
 
 def main(options):
     """Main entry."""
-    
+
     if "LOCATION" in options.value or "REL_POS" in options.value:
         show_positions(options)
-        return
 
-    if "FLATNESS_LOCAL" in options.value:
+    elif "FLATNESS_LOCAL" in options.value:
         show_flatness(options)
-        return
 
+    else:
+        front, back, labels = read_data_files(options)
 
-    front, back, labels = read_data_files(options)
+        labels.sort()
+        X = np.arange(0, len(labels))
+        fig, ax = plt.subplots(1, 1, tight_layout=True)
+        fig.suptitle(options.value)
+        ax.set_xticks(range(len(labels)), labels=labels, rotation="vertical")
+        ax.grid()
 
-    labels.sort()
-    X = np.arange(0, len(labels))
-    fig, ax = plt.subplots(1, 1, tight_layout=True)
-    fig.suptitle(options.value)
-    ax.set_xticks(range(len(labels)), labels=labels)
-    ax.grid()
+        vfront = [front[x] for x in labels]
+        vback = [back[x] for x in labels]
+        ax.plot(X, vfront, '*', label="Front")
+        ax.plot(X, vback, 'o', label="Back")
+        ax.legend()
 
-    vfront = [front[x] for x in labels]
-    vback = [back[x] for x in labels]
-    ax.plot(X, vfront, '*', label="Front")
-    ax.plot(X, vback, 'o', label="Back")
-    ax.legend()
+        save_figure(fig, options.out, prefix=options.prefix)
 
-    save_figure(fig, options.out)
-    
     plt.show()
 
 
@@ -214,6 +216,7 @@ if __name__ == "__main__":
     parser = argparse.ArgumentParser()
     parser.add_argument('files', nargs='*', help="Input files")
     parser.add_argument("--value", default=None, help="Value to plot")
+    parser.add_argument("--prefix", default=None, help="prefix for out file")
     parser.add_argument("--out", default=None, help="File to store the figure.")
 
     opts = parser.parse_args()
@@ -221,7 +224,7 @@ if __name__ == "__main__":
         print("I need at least one input file")
         sys.exit()
 
-    if len(opts.files) == 1:
+    elif len(opts.files) == 1:
         xxx = any(elem in opts.files[0] for elem in r"*?")
         if xxx:
             opts.files = glob.glob(opts.files[0])

petal_qc/metrology/convert_smartscope.py

@@ -10,15 +10,15 @@ import matplotlib.pyplot as plt
 
 
 from petal_qc.metrology import DataFile
-from .analyze_locking_points import analyze_locking_point_data
+#from .analyze_locking_points import analyze_locking_point_data
 
 def get_data_chunck(flist, data_type, ofile):
     """Get one data type chunk from file
 
     Args:
-        fname (): input file
-        data_tyle (): data type
-        ofile (): file object for output.
+        fname: input file
+        data_tyle: data type
+        ofile: file object for output.
 
     """
     rgx = "Step Name:\\s+({}).*?=".format(data_type)
@@ -41,7 +41,7 @@ def get_data_chunck(flist, data_type, ofile):
                 ofile.write(txt.group(0)+'\n')
     
 
-def read_smartscope(fnam, ofile, data_type, keep=False,):
+def read_smartscope(fnam, ofile, data_type, keep=False):
     """Convert a SmartScope txt file into CVS
 
     Args:
@@ -58,7 +58,11 @@ def read_smartscope(fnam, ofile, data_type, keep=False,):
     if isinstance(ofile, io.IOBase):
         fout = ofile
     else:
-        fout = open(ofile, 'w')
+        try:
+            fout = open(ofile, 'w', encoding="utf-8")
+        except TypeError:
+            fout = ofile
+
         is_file = True
 
     if keep:
@@ -106,7 +110,84 @@ def read_smartscope(fnam, ofile, data_type, keep=False,):
             values = [ float(x) for x in items[2:5]]
             fout.write("{:6f}, {:6f}, {:6f}\n".format(values[1], values[0], values[2]))
 
-    fout.close()
+    if is_file:
+        fout.close()
+    
+def get_smarscope_locator_positions(fnam, ofile, data_type, keep=False):
+    """REad DESY 2D file.
+
+    Args:
+        fnam: Input file
+        ofile: Output file
+        data_type: Label to search
+        keep (bool, optional): If tre keep label. Defaults to False.
+    """
+    ifile = Path(fnam).expanduser().resolve()
+    
+    is_file = False
+    if isinstance(ofile, io.IOBase):
+        fout = ofile
+    else:
+        try:
+            fout = open(ofile, 'w', encoding="utf-8")
+        except TypeError:
+            fout = ofile
+
+        is_file = True
+
+    if keep:
+        fout.write("X,Y,Z, label\n")
+    else:
+        fout.write("X,Y,Z\n")
+
+    rgx = "^Step Name:\\s+(?P<label>{})".format(data_type)
+    start = re.compile(rgx, re.DOTALL)
+    finder = re.compile("^Finder Name")
+
+    out = {}
+    with open(ifile, 'r', encoding='ISO-8859-1') as fin:
+        while True:
+            line = fin.readline()
+            if not line:
+                break
+
+            r = start.match(line)
+            if not r:
+                continue
+            
+            label = r.group('label').strip()
+            eof = False
+            out[label] = [0. for i in range(3)]
+            for i in range(3):
+                line = fin.readline()
+            
+            while True:
+                line = fin.readline().strip()
+                if len(line)==0:
+                    break
+            
+                values = line.split()
+                val = float(values[3])
+                if values[0] in ["Diame", "Width"]:
+                    out[label][2] = val
+                elif values[0] == "X":
+                    out[label][1] = val
+                elif values[0] == "Y":
+                    out[label][0] = val
+                    
+    lbls = ["BottomPL", "SlotPL", "OversizedPL", "Bottom_Fiducial", "Slot_Fiducial"]
+    for L in lbls:
+        try:
+            V = [float(x) for x in out[L]]
+            fout.write("{:.6f}, {:.6f}, {:.6f}".format(V[0], V[1], V[2]))
+            if keep:
+                fout.write(", {}".format(L))
+            fout.write('\n')
+        except KeyError:
+            continue
+                          
+    if is_file:
+        fout.close()
 
 def do_locking_points(args):
     ifile = args.smartscope_files[0]
@@ -120,14 +201,14 @@ if __name__ == "__main__":
     import argparse
 
     parser = argparse.ArgumentParser()
-    parser.add_argument("smartscope_files", nargs='*',
+    parser.add_argument("files", nargs='*',
                         help="The SmartScope files to parse")
     parser.add_argument("--label", default="\\w+", help="The label to select")
     parser.add_argument("--out", help="Output CSV file", default="out.csv")
     parser.add_argument("--keep_label", dest="keep", default=False, action="store_true", help="Store label in output")
 
     args = parser.parse_args()
-    if len(args.smartscope_files) == 0:
+    if len(args.files) == 0:
         print("I need an input file")
         sys.exit()
 
@@ -135,11 +216,13 @@ if __name__ == "__main__":
 
     # do_locking_points(args)
     
-    fout = open(Path(args.out).expanduser().resolve(), 'w')
-    get_data_chunck(args.smartscope_files, args.label, fout)
-    fout.close()
+    #fout = open(Path(args.out).expanduser().resolve(), 'w')
+    #get_data_chunck(args.smartscope_files, args.label, fout)
+    #fout.close()
     
     
     #read_smartscope(args.smartscope_files[0], args.out, args.label, args.keep)
 
+    get_smarscope_locator_positions(args.files[0], args.out, args.label)
+
     plt.show()

petal_qc/metrology/coreMetrology.py

@@ -16,7 +16,7 @@ import gi
 gi.require_version("Gtk", "3.0")
 from gi.repository import Gtk, GObject, Gio, GLib
 
-
+__HELP__ = "https://petal-qc.docs.cern.ch"
 
 class CommaSeparatedListAction(Action):
     """Create a list from the comma sepparated numbers at imput."""
@@ -41,7 +41,7 @@ class CoreMetrology(itkdb_gtk.dbGtkUtils.ITkDBWindow):
             pannel_size: size of message panel.
 
         """
-        super().__init__(session=session, title=title)
+        super().__init__(session=session, title=title, help=__HELP__)
 
         self.data_file = None
         self.folder = None

petal_qc/metrology/do_Metrology.py

@@ -42,13 +42,13 @@ def do_analysis(fnam, prefix, SN, options):
     """Perform analysis of a file.
 
     Args:
-        fnam (): Input data file
-        prefix (): Prefix telling if it is front or back
-        SN (): Core serial number
-        options (): Options.
-        
+        fnam: Input data file
+        prefix: Prefix telling if it is front or back
+        SN: Core serial number
+        options: Options.
+
     """
-    
+
     is_front = prefix.lower().find("front") >= 0
     print(fnam, prefix)
     options.out = prefix + '.docx'
@@ -63,10 +63,10 @@ def do_analysis(fnam, prefix, SN, options):
     ofile = output_folder(options.folder, prefix + '.json')
     with open(ofile, 'w', encoding='UTF-8') as of:
         json.dump(outDB, of, indent=3)
-    
+
     return outDB
 
-def main(ifile, options):
+def analyze_files(ifile, options):
     """Main entry."""
     with open(ifile, 'r', encoding='UTF-8') as inp:
 
@@ -86,25 +86,25 @@ def main(ifile, options):
 
             do_analysis(fnam, prefix, SN, options)
 
-
-if __name__ == "__main__":
+def main():
+    "Main entry."
     parser = ArgumentParser()
     parser.add_argument('files', nargs='*', help="Input files")
-    parser.add_argument("--prefix", dest='prefix', default=None)
-    parser.add_argument("--SN", dest='SN', default=None)
+    parser.add_argument("--prefix", dest='prefix', default=None, help="prefix telling if it is front or back.")
+    parser.add_argument("--SN", dest='SN', default=None, help="The petal core Serial Number")
     parser.add_argument("--save", dest='save', action="store_true", default=False)
-    parser.add_argument("--desy", dest='desy', action="store_true", default=False)
+    parser.add_argument("--desy", dest='desy', action="store_true", default=False, help="True if data is from DESY's SmartScope")
     parser.add_argument("--out", dest="out", default="petal_flatness.docx",
-                        type=str, help="The output fiel name")
+                        type=str, help="The output file name")
     parser.add_argument("--bottom-lp", dest='bLP', action=CommaSeparatedListAction, default=None,
                         help="Bottom locking point fiducial coordinates")
     parser.add_argument("--upper-lp", dest='uLP', action=CommaSeparatedListAction, default=None,
                         help="upper locking point fiducials coordinates")
 
     parser.add_argument("--title", dest="title", default=None,
-                        type=str, help="Document title")
+                        type=str, help="Report Document title")
     parser.add_argument("--nbins", dest="nbins", default=25,
-                        type=int, help="Number of bins")
+                        type=int, help="Number of bins in the histograms")
     parser.add_argument("--folder", default=None, help="Folder to store output files. Superseeds folder in --out")
     parser.add_argument("--locking_points", action="store_true", default=False)
 
@@ -119,7 +119,10 @@ if __name__ == "__main__":
         sys.exit()
 
     try:
-        main(options.files[0], options)
+        analyze_files(options.files[0], options)
 
     except Exception:
         print(traceback.format_exc())
+
+if __name__ == "__main__":
+    main()

petal_qc/metrology/petal_flatness.py

@@ -218,7 +218,7 @@ def petal_flatness(orig_data, options, document=None):
         # add allsensor area
         cell = table.add_row().cells
         cell[0].text = "All sensor area"
-        cell[1].text = "{:.4f}".format(flatness_all_sensor_area)
+        cell[1].text = "{:.4f}".format(flatness_all_sensor_area*1000)
 
     return TM, avg, Zmean, outF
 

petal_qc/test/prepareDESYfiles.py

@@ -0,0 +1,86 @@
+#!/usr/bin/env python3
+"""Prepara input files from DESY RAW."""
+import os
+import fnmatch
+import re
+from pathlib import Path
+
+def all_files(root, patterns='*', single_level=False, yield_folders=False):
+    """A generator that reruns all files in the given folder.
+
+    Args:
+    ----
+        root (file path): The folder
+        patterns (str, optional): The pattern of the files. Defaults to '*'.
+        single_level (bool, optional): If true, do not go into sub folders. Defaults to False.
+        yield_folders (bool, optional): If True, return folders as well. Defaults to False.
+
+    Yields
+    ------
+        str: file path name
+
+    """
+    patterns = patterns.split(';')
+    for path, subdirs, files in os.walk(root):
+        if yield_folders:
+            files.extend(subdirs)
+
+        files.sort()
+        for name in files:
+            for pattern in patterns:
+                if fnmatch.fnmatch(name, pattern):
+                    yield os.path.join(path, name)
+                    break
+
+        if single_level:
+            break
+
+class PetalCore:
+    def __init__(self, front, back):
+        self.front = {}
+        self.back = {}
+
+def main(folder, out_folder):
+    """Main entry point."""
+    outF = Path(out_folder).expanduser().resolve()
+    if not outF.exists():
+        os.mkdir(outF)
+        
+    rgx = re.compile(r"Project Name: (\w+)side_.*AlternativeID=PPC-(\d+)", re.MULTILINE|re.DOTALL)
+    petal_cores = {}
+    for fnam in all_files(folder, "*.txt"):
+        P = Path(fnam).expanduser().resolve()
+        with open(fnam, "r", encoding="UTF-8") as ff:
+            R = rgx.search(ff.read())
+            if R:
+                petal_id = "PPC.{}".format(R.group(2))
+                side = R.group(1).lower()
+                if "_2D_" in P.name:
+                    test_type = "2D"
+                else:
+                    test_type = "3D"
+
+                if not petal_id in petal_cores:
+                    petal_cores[petal_id] = {
+                                             "back":{"2D":None, "3D":None},
+                                             "front":{"2D":None, "3D":None},
+                                             }
+
+                petal_cores[petal_id][side][test_type] = P
+
+    list_file = open("{}/Desy-cores.txt".format(outF.as_posix()), "w", encoding="UTF-8")
+    for petal_id, sides in petal_cores.items():
+        for side, values in sides.items():
+            oname = "{}/{}-{}.txt".format(outF.as_posix(), petal_id, side)
+            list_file.write("{} {}-{} {}\n".format(oname, petal_id, side, petal_id))
+            data = ""
+            for data_type, fnam in values.items():
+                with open(fnam, "r", encoding="UTF-8") as ifile:
+                    data += ifile.read()
+                    
+            with open(oname, "w", encoding="UTF-8") as ofile:
+                ofile.write(data)
+       
+    list_file.close()
+if __name__ == "__main__":
+    main("/Users/lacasta/Downloads/DESY_Production", "/tmp/desy")

petal_qc/thermal/analyze_IRCore.py

@@ -3,6 +3,7 @@
 import json
 import os
 import sys
+import bisect
 from pathlib import Path
 
 import matplotlib.dates as pldates
@@ -19,6 +20,14 @@ from petal_qc.thermal import Petal_IR_Analysis
 
 CO2_Temp_setting = -35
 
+def get_array_as_string(A):
+    """Return a string rerpresentation."""
+    sss="["
+    for v in A:
+        sss += " {:5.3f}".format(v)
+
+    sss+=" ]"
+    return sss
 
 def get_acceptance_band():
     """returns the width of the acceptance band."""
@@ -362,6 +371,9 @@ def compare_golden(core, golden, value):
         golden (AnalysisResult): The golden avrage
         value (str): The name of the variable to compare
 
+    Returns:
+        (pass, max difference, array of location of outsiders)
+
     """
     Y = getattr(core, value)
     gY = getattr(golden, value)
@@ -373,18 +385,23 @@ def compare_golden(core, golden, value):
         X = np.array([float(x) for x in range(10)])
         gX = X
 
-    # convert to golden X
-    spln = CubicSpline(X, Y)
-    gC = np.array([spln(x) for x in gX])
-
     # compute difference
     band = get_acceptance_band()
-    diff = np.abs(gC-gY)
-    outsiders = np.where(diff > band)
-    mxval = np.amax(diff)
+    outsiders = []
+    mxval = -1
+    indx = 1
+    for x, y in zip(X, Y):
+        indx = bisect.bisect_left(gX, x, lo=(indx-1 if indx>1 else 0))
+        val = gY[indx]
+        diff = abs(y-val)
+        if diff > mxval:
+            mxval = diff
+
+        if diff>band:
+            outsiders.append(x)
 
     rc = mxval < band
-    return rc, mxval, len(outsiders)
+    return rc, mxval, outsiders
 
 
 def analyze_petal_cores(files, golden, options):
@@ -456,15 +473,16 @@ def analyze_petal_cores(files, golden, options):
                 if val[0]:
                     continue
 
-                if val[2] < 2:
-                    dbOut["comments"].append(
-                        "{} in side {}: 1 point out of acceptance band.".format(
-                            key, iside)
-                    )
-                else:
+                noutsiders = len(val[2])
+                #if noutsiders < 2:
+                #    dbOut["comments"].append(
+                #        "{} in side {}: 1 point out of acceptance band. {}".format(
+                #            key, iside, val[2])
+                #    )
+                if noutsiders>0:
                     dbOut["defects"].append(
                         {"name": key,
-                            "description": "{} in side {}: {} points out of acceptance band".format(key, iside, val[2]),
+                            "description": "{} in side {}: {} points out of acceptance band. {}".format(key, iside, noutsiders, get_array_as_string(val[2])),
                             "properties": "{} > {}".format(val[1], get_acceptance_band())
                          }
                     )

petal_qc/thermal/coreThermal.py

@@ -11,6 +11,7 @@ import itkdb_gtk.ITkDButils
 import itkdb_gtk.dbGtkUtils
 import itkdb_gtk.UploadTest
 
+__HELP__ = "https://petal-qc.docs.cern.ch"
 
 try:
     import petal_qc
@@ -45,7 +46,7 @@ class CoreThermal(itkdb_gtk.dbGtkUtils.ITkDBWindow):
             title: Window title.
             panel_size: size of message pannel.
         """
-        super().__init__(session=session, title=title)
+        super().__init__(session=session, title=title, help_link=__HELP__ )
 
         self.petal_SN = None
         self.param = params if params else IRPetalParam()