petal-qc 0.0.14.dev1__py3-none-any.whl → 0.0.16__py3-none-any.whl

petal_qc/test/createMetrologyFile.py

@@ -0,0 +1,77 @@
+#!/usr/bin/env python3
+"""Locates all raw data files."""
+
+import os
+import sys
+import fnmatch
+import re
+import shutil
+from pathlib import Path
+import argparse
+
+def all_files(root, patterns='*', single_level=False, yield_folders=False):
+    """A generator that reruns all files in the given folder.
+
+    Args:
+    ----
+        root (file path): The folder
+        patterns (str, optional): The pattern of the files. Defaults to '*'.
+        single_level (bool, optional): If true, do not go into sub folders. Defaults to False.
+        yield_folders (bool, optional): If True, return folders as well. Defaults to False.
+
+    Yields
+    ------
+        str: file path name
+
+    """
+    patterns = patterns.split(';')
+    for path, subdirs, files in os.walk(root):
+        if yield_folders:
+            files.extend(subdirs)
+
+        files.sort()
+        for name in files:
+            for pattern in patterns:
+                if fnmatch.fnmatch(name, pattern):
+                    yield os.path.join(path, name)
+                    break
+
+        if single_level:
+            break
+
+def main(options):
+    """Locate raw data files in input folder and copies them in out folder
+
+    Args:
+        folder_in (Path): Input folder
+        folder_oyt (Path): Output folder
+    """
+    reg = re.compile(r"(PPC\.[0-9]+)-(\w+)")
+    outF = Path(options.output).expanduser().resolve()
+    with open(outF, "w", encoding="utf-8") as fout:
+        for folder_in in options.files:
+            inF =  Path(folder_in).expanduser().resolve()
+            if not inF.exists():
+                print("Input folder does not exist. {}".format(inF))
+                continue
+
+            for fnam in all_files(inF, "PPC*.txt", yield_folders=False):
+                R = reg.search(fnam)
+                if R is None:
+                    continue
+                
+                petal_id = R.group(1)
+                side = R.group(2)
+                fout.write("{ifile} {pid}-{side} {pid}\n".format(ifile=fnam, side=side, pid=petal_id))
+        
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument('files', nargs='*', help="Input files")
+    parser.add_argument("--output", dest="output", help="Output file", default="out.txt")
+    opts = parser.parse_args()
+    if len(opts.files)==0:
+        print("I need at least one input folder")
+        sys.exit(-1)
+
+    main(opts)

petal_qc/test/createMetrologyTable.py

@@ -0,0 +1,87 @@
+#!/usr/bin/env python3
+"""Create  table for metrology analysis."""
+
+import sys
+import re
+import json
+from pathlib import Path
+from argparse import ArgumentParser
+from petal_qc.utils.all_files import all_files
+from petal_qc.utils.ArgParserUtils import RangeListAction
+
+r_petal_id = re.compile("PPC.([0-9]*)")
+
+db_file = Path("/Users/lacasta/cernbox/workspace/Petal-QC/PetalMould.csv")
+
+petal_db = {}
+with open(db_file, "r", encoding="utf-8") as fin:
+    for i, line in enumerate(fin):
+        if i==0:
+            continue
+
+        values = [x.strip() for x in line.split(",")]
+        petal_db[values[0]] = values[1:]
+
+def main(options):
+    """Main entry."""
+    ifolder = Path(options.input)
+    if not ifolder.exists():
+        print("Input folder does not exist.\n{}".format(ifolder))
+        return
+
+    has_list = len(options.cores) != 0
+
+    ofile = open(options.out, "w", encoding="utf-8")
+    ofile.write("PetalID,SN,side,mould,fd_dx,fd_dy,paralelism,R0,R1,R2,R3S0,R3S1,R4S0,R4S1,R5S0,R5S1,flatness\n")
+
+    for fnam in all_files(ifolder.as_posix(), "*.json"):
+        fstem = fnam.stem
+        if "PPC." not in fstem:
+            continue
+
+        R = r_petal_id.search(fstem)
+        if R is None:
+            continue
+
+        petal_id = R.group(0)
+        pid = int(R.group(1))
+
+        if has_list and pid not in options.cores:
+            continue
+
+        SN = petal_db[petal_id][0]
+        side = 1 if "back" in fstem else 0
+        mould = int(petal_db[petal_id][1])
+
+        data = None
+        with open(fnam, "r", encoding="utf-8") as fin:
+            data = json.load(fin)
+
+        ofile.write("{}, {}, {}, {}".format(petal_id, SN, side, mould))
+        print("data SN", data["component"])
+        D = data["results"]["METROLOGY"]["REL_POS_DELTA"]["FD01-FD02"]
+        ofile.write(",{:.5f}, {:.5f}".format(D[0], D[1]))
+
+        D = data["results"]["METROLOGY"]["PARALLELISM"]
+        ofile.write(",{:.5f}".format(D))
+
+        for x in data["results"]["METROLOGY"]["FLATNESS_LOCAL"]:
+            ofile.write(", {:.5f}".format(x))
+
+        D = data["results"]["METROLOGY"]["FLATNESS_GLOBAL"]
+        ofile.write(", {:.5f}".format(D))
+
+        ofile.write("\n")
+
+    ofile.close()
+
+
+if __name__ == "__main__":
+    parser = ArgumentParser()
+    parser.add_argument("--input-folder", dest="input", default=".", help="Input folder")
+    parser.add_argument("--out", default="out.csv", help="Output table.")
+    parser.add_argument("--cores", dest="cores", action=RangeListAction, default=[],
+                        help="Create list of cores to analyze. The list is made with  numbers or ranges (ch1:ch2 or ch1:ch2:step) ")
+
+    opts = parser.parse_args()
+    main(opts)

petal_qc/test/desyModuleBow.py

@@ -0,0 +1,126 @@
+#!/usr/bin/env python3
+"""Get module bow from desy metrology files"""
+import sys
+import re
+from pathlib import Path
+import matplotlib.pyplot as plt
+import numpy as np
+import numpy.linalg as linalg
+
+
+from petal_qc.utils.all_files import all_files
+from petal_qc.metrology import DataFile
+from petal_qc.metrology import DataFile
+from petal_qc.utils.Geometry import fit_plane
+from petal_qc.utils.Geometry import project_to_plane
+from petal_qc.utils.Geometry import remove_outliers_indx
+
+
+rgx = "serialNo =\\s+([A-Z0-9]+)"
+serial_no = re.compile(rgx)
+
+mtype = re.compile("Project Name:\\s+([A-Za-z0-9]+)_bow")
+
+
+def get_array_range(A):
+    """Gets mean and range of given array."""
+    avg = np.mean(A)
+    imin = np.argmin(A)
+    vmin = A[imin]
+    imax = np.argmax(A)
+    vmax = A[imax]
+    stdv = np.std(A)
+    return avg, stdv, [vmin, vmax], [imin, imax]
+
+
+
+def create_bow_figure(SN, mtype, pout, width) -> plt.Figure:
+    """Create the sensor bow figure.
+
+    Args:
+        options: Program options.
+        pout: set of points
+        width: the actual bow.
+
+    Returns:
+        plt.Figure: The bow figure.
+    """
+    fig_bow, ax = plt.subplots(subplot_kw={'projection': '3d'})
+    fig_bow.suptitle(r"{}_{} - Sensor bow {:.1f} $\mu$m".format(SN, mtype, width))
+
+    zplt = 1000*pout[:, 2]
+    # surf = ax.plot_trisurf(pout[:, 0], pout[:, 1], zplt, cmap=plt.cm.jet, edgecolor="black", linewidths=0.2)
+    surf = ax.scatter(pout[:, 0], pout[:, 1], zplt, c=zplt, marker="o", edgecolor='none')
+    ax.set_xlabel("X")
+    ax.set_ylabel("Y")
+    ax.set_zlabel("Z")
+    cbar = fig_bow.colorbar(surf, shrink=0.5, aspect=5, location="left")
+    cbar.set_label(r"Z ($\mu$m)")
+    return fig_bow
+
+
+def module_bow(ifile):
+    """Compute module bow."""
+    SN = None
+    mod_type = None
+    print(Path(ifile).name)
+    with open(ifile, "r", encoding="utf-8") as fin:
+        ss = fin.read()
+        mtx = serial_no.search(ss)
+        if mtx:
+            SN = mtx.group(1)
+            
+        mtp = mtype.search(ss)
+        if mtp:
+            mod_type = mtp.group(1)
+
+    if SN is None:
+        return None, None, None
+    
+    print("{} - {}".format(SN, mod_type))
+    
+    data = DataFile.read(ifile, "(Bow|Sensor)")
+    if len(data) == 0:
+        return None, None, None
+    
+    # Try to remove locator points and get the plane
+    indx = remove_outliers_indx(data[:, 2])
+    M, TM, *_ = fit_plane(data[indx], use_average=False)
+
+    # project all data to the plane
+    Zmean = np.mean(M[:, 2])
+    M[:, 2] -= Zmean
+    
+    mean, stdev, rng, irng = get_array_range(M[:, 2])
+    width = 1000*(rng[1]-rng[0])
+    
+    # Check which one is closest to the center
+    min2center = linalg.norm(M[irng[0], 0:2])
+    max2center = linalg.norm(M[irng[1], 0:2])
+    # If center above the top, bow negative
+    if max2center < min2center:
+        width = -width
+
+    print("Sensor bow\nAverage: {:.1f} min {:.1f} max {:.1f} rng {:.1f}\n".format(
+        mean, 1000*rng[0], 1000*rng[1], width))
+
+    fig_bow = create_bow_figure(SN, mod_type, M, width)
+    fig_bow.savefig("{}-{}-bow.png".format(SN, mod_type), dpi=300)
+    plt.close(fig_bow)
+    del fig_bow
+    return SN, mod_type, width
+
+
+def main(folder):
+    """Main entry"""
+    fout = open("module-bow.txt", "w", encoding="utf-8")
+    for fnam in all_files(folder, "*.txt"):
+        SN, mod_type, width = module_bow(fnam)
+        if SN:
+            fout.write("{}-{} : {:.3f}\n".format(SN, mod_type, width))
+            
+    fout.close()
+
+if __name__ == "__main__":
+    main("/Users/lacasta/Downloads/InterposerPetal")
+    #main("/Users/lacasta/Downloads/kkdvk")

petal_qc/test/findRawData.py

@@ -0,0 +1,92 @@
+#!/usr/bin/env python3
+"""Locates all raw data files."""
+
+import os
+import sys
+import fnmatch
+import re
+import shutil
+from pathlib import Path
+import argparse
+
+def all_files(root, patterns='*', single_level=False, yield_folders=False):
+    """A generator that reruns all files in the given folder.
+
+    Args:
+    ----
+        root (file path): The folder
+        patterns (str, optional): The pattern of the files. Defaults to '*'.
+        single_level (bool, optional): If true, do not go into sub folders. Defaults to False.
+        yield_folders (bool, optional): If True, return folders as well. Defaults to False.
+
+    Yields
+    ------
+        str: file path name
+
+    """
+    patterns = patterns.split(';')
+    for path, subdirs, files in os.walk(root):
+        if yield_folders:
+            files.extend(subdirs)
+
+        files.sort()
+        for name in files:
+            for pattern in patterns:
+                if fnmatch.fnmatch(name, pattern):
+                    yield os.path.join(path, name)
+                    break
+
+        if single_level:
+            break
+
+def main(folder_in, folder_out):
+    """Locate raw data files in input folder and copies them in out folder
+
+    Args:
+        folder_in (Path): Input folder
+        folder_oyt (Path): Output folder
+    """
+    inF =  Path(folder_in).expanduser().resolve()
+    if not inF.exists():
+        print("Input folder does not exist. {}".format(inF))
+        return
+    
+    reg = re.compile(r"(PPC\.[0-9]+).*metr")
+    
+    outF = Path(folder_out).expanduser().resolve()
+    if not outF.exists():
+        os.makedirs(outF.as_posix())
+        
+    list_file = open("{}/raw-data-metrology-cores.txt".format(outF.as_posix()), "w", encoding="UTF-8")
+    for fnam in all_files(inF, "PPC*.txt", yield_folders=False):
+        R = reg.search(fnam)
+        if R is None:
+            continue
+        
+        petal_id = R.group(1)
+        if "back" in fnam:
+            side = "{}-back".format(petal_id)
+            ofile = "{}-back.txt".format(petal_id)
+        elif "front" in fnam:
+            side = "{}-front".format(petal_id)
+            ofile = "{}-front.txt".format(petal_id)
+        else:
+            print("Invalid file {}".format(fnam))
+            continue
+        
+        out_name = outF / ofile
+        list_file.write("{} {} {}\n".format(out_name.as_posix(), side, petal_id))
+        shutil.copy(fnam, out_name)
+        
+    list_file.close()
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--input-folder", dest="input", default=None, help="Input folder")
+    parser.add_argument("--output-folder", dest="output", help="Outout fodler", default=None)
+    opts = parser.parse_args()
+    if opts.input is None or opts.output is None:
+        print("I need both an input and an output folder.")
+        sys.exit(-1)
+
+    main(opts.input, opts.output)

petal_qc/test/getAVStests.py

@@ -200,6 +200,7 @@ def main(session):
     for T in core_tests:
         counter[T]=0
 
+    mould_db = {}
     for core in core_list:
         SN = core["serialNumber"]
         altid = core['alternativeIdentifier']
@@ -243,6 +244,7 @@ def main(session):
         mould_desc = do_manufacturing(good_tests["MANUFACTURING"])
         pos = mould_desc.rfind('.')
         mould_id = mould_desc[pos+1:]
+        mould_db[altid] = (SN, mould_id)
         if mould_id not in Mould_values:
             Mould_values[mould_id] = {}
 
@@ -302,6 +304,12 @@ def main(session):
     plot_weighing(weights, petal_ids, document)
     plot_metrology(M_values, Mould_values, petal_ids, document)
     document.save("AVStests.docx")
+
+    with open("mould_db.csv", "w", encoding="utf-8") as fout:
+        fout.write("PetalID, SerialNo, MouldID\n")
+        for key, val in mould_db.items():
+            fout.write("{}, {}, {}\n".format(key, val[0], val[1]))
+
     plt.show()
 
 if __name__ == "__main__":

petal_qc/test/prepareDESYfiles.py

@@ -4,6 +4,7 @@ import os
 import sys
 import fnmatch
 import re
+import shutil
 from pathlib import Path
 import argparse
 
@@ -52,14 +53,23 @@ def main(folder, out_folder):
         folder: the input folder where to find the original files.
         out_folder: the folder where the new fiels will be stored.
     """
+    inF =  Path(folder).expanduser().resolve()
+    if not inF.exists():
+        print("Input folder does not exist. {}".format(inF))
+        return
+    
+    print("Reading folder: {}".format(inF))
+    print("output in {}".format(out_folder))
     outF = Path(out_folder).expanduser().resolve()
     if not outF.exists():
+        print("creating {}".format(outF))
         os.mkdir(outF)
 
     rgx = re.compile(r"Project Name: (\w+)side_.*AlternativeID=PPC[-_](\d+)", re.MULTILINE|re.DOTALL)
     petal_cores = {}
-    for fnam in all_files(folder, "*.txt"):
+    for fnam in all_files(inF, "*.txt"):
         P = Path(fnam).expanduser().resolve()
+        print(P.name)
         with open(fnam, "r", encoding="UTF-8") as ff:
             R = rgx.search(ff.read())
             if R:
@@ -86,7 +96,8 @@ def main(folder, out_folder):
             data = ""
             for data_type, fnam in values.items():
                 if fnam is None:
-                    print("This should not happen.")
+                    print("Missing file name for {} {} [{}]".format(petal_id, side, data_type))
+                    continue
 
                 with open(fnam, "r", encoding="UTF-8") as ifile:
                     data += ifile.read()

petal_qc/test/reportFromJSon.py

@@ -0,0 +1,51 @@
+#!/usr/bin/env python3
+"""List failing cores from JSon files."""
+import sys
+import argparse
+from pathlib import Path
+import json
+
+
+
+def main(options):
+    """main entry."""
+    petal_cores = {}
+    for fnam in options.files:
+        ifile = Path(fnam).expanduser().resolve()
+        with open(ifile, "r", encoding="utf-8") as fin:
+            data = json.load(fin)
+
+        if not data["passed"]:
+            petalId = data["component"]
+            if petalId not in petal_cores:
+                petal_cores[petalId] = {"FRONT": [], "BACK": []}
+
+            side = "FRONT" if "FRONT" in data["testType"] else "BACK"
+            for D in data["defects"]:
+                petal_cores[petalId][side].append("{}: {}".format(D["name"], D["description"]))
+
+
+    keys = sorted(petal_cores.keys())
+    for petalId in keys:
+        print(petalId)
+        for side in ["FRONT","BACK"]:
+            if len(petal_cores[petalId][side])>0:
+                print("+-", side)
+            for D in petal_cores[petalId][side]:
+                print("  ", D)
+
+        print("\n")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument('files', nargs='*', help="Input files")
+    opts = parser.parse_args()
+    
+    from petal_qc.utils.all_files import all_files
+    
+    opts.files = []
+    for fnam in all_files(Path("~/tmp/petal-metrology/Production/Results").expanduser(), "*.json"):
+        opts.files.append(fnam)
+    
+    main(opts)

petal_qc/thermal/show_IR_petal.py

@@ -24,21 +24,12 @@ from petal_qc.thermal import contours
 from petal_qc.thermal import IRBFile
 from petal_qc.thermal import IRPetal
 from petal_qc.thermal import PipeFit
-from petal_qc.thermal.Petal_IR_Analysis import AnalysisResult
 from petal_qc.thermal.Petal_IR_Analysis import show_2D_image
 from petal_qc.thermal.PetalColorMaps import HighContrast
 
 import petal_qc.utils.Progress as Progress
 import petal_qc.utils.utils as utils
-
-
-class CommaSeparatedListAction(Action):
-    """Create a list from the comma sepparated numbers at imput."""
-
-    def __call__(self, parser, namespace, values, option_string=None):
-        """The actual action."""
-        setattr(namespace, self.dest, list(map(int, values.split(','))))
-
+from petal_qc.utils.ArgParserUtils import CommaSeparatedListAction
 
 def get_min_max(values, step=1.0):
     """Return min and max.

petal_qc/utils/ArgParserUtils.py

@@ -0,0 +1,41 @@
+#!/usr/bin/env python3
+"""A number of utilities for argument parser."""
+
+from argparse import Action
+
+class CommaSeparatedListAction(Action):
+    """Create a list from the comma sepparated numbers at imput."""
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        """The actual action."""
+        setattr(namespace, self.dest, list(map(int, values.split(','))))
+
+
+class RangeListAction(Action):
+    """Create a list from a range expresion at input.
+
+    The list is made with  numbers or ranges (ch1:ch2 or ch1:ch2:step)
+    """
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        """The actual action."""
+        value = []
+        for V in values.split(','):
+            try:
+                value.append(int(V))
+            except ValueError:
+                if ':' not in V:
+                    continue
+
+                items = V.split(':')
+                if len(items)==1:
+                    continue
+
+                ival = list(map(int, items))
+                ival[1] += 1
+                for x in range(*ival):
+                    value.append(int(x))
+
+
+        setattr(namespace, self.dest, value)
+

{petal_qc-0.0.14.dev1.dist-info → petal_qc-0.0.16.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: petal_qc
-Version: 0.0.14.dev1
+Version: 0.0.16
 Summary: A collection of scripts for Petal CORE QC.
 Author-email: Carlos Lacasta <carlos.lacasta@cern.ch>
 Project-URL: Homepage, https://gitlab.cern.ch/atlas-itk/sw/db/itk-pdb-gtk-gui-utils
@@ -10,14 +10,14 @@ Classifier: Operating System :: OS Independent
 Requires-Python: >=3.7
 Description-Content-Type: text/markdown
 Requires-Dist: itkdb
-Requires-Dist: itkdb-gtk >=0.10.10
+Requires-Dist: itkdb_gtk>=0.10.10
 Requires-Dist: numpy
 Requires-Dist: matplotlib
 Requires-Dist: lmfit
 Requires-Dist: openpyxl
 Requires-Dist: pandas
-Requires-Dist: python-dateutil
-Requires-Dist: python-docx
+Requires-Dist: python_dateutil
+Requires-Dist: python_docx
 Requires-Dist: scipy
 Requires-Dist: scikit-image