petal-qc 0.0.9__py3-none-any.whl → 0.0.11__py3-none-any.whl

petal_qc/test/prepareDESYfiles.py

@@ -1,9 +1,10 @@
 #!/usr/bin/env python3
-"""Prepara input files from DESY RAW."""
+"""Prepare raw data input files from DESY so that we can use them with this code."""
 import os
 import fnmatch
 import re
 from pathlib import Path
+import argparse
 
 def all_files(root, patterns='*', single_level=False, yield_folders=False):
     """A generator that reruns all files in the given folder.
@@ -41,18 +42,26 @@ class PetalCore:
         self.back = {}
 
 def main(folder, out_folder):
-    """Main entry point."""
+    """Main entry point.
+
+    It takes DESY raw data files and combines them into  a single file for front
+    and back metrology.
+
+    Args:
+        folder: the input folder where to find the original files.
+        out_folder: the folder where the new fiels will be stored.
+    """
     outF = Path(out_folder).expanduser().resolve()
     if not outF.exists():
         os.mkdir(outF)
-        
-    rgx = re.compile(r"Project Name: (\w+)side_.*AlternativeID=PPC-(\d+)", re.MULTILINE|re.DOTALL)
+
+    rgx = re.compile(r"Project Name: (\w+)side_.*AlternativeID=PPC[-_](\d+)", re.MULTILINE|re.DOTALL)
     petal_cores = {}
     for fnam in all_files(folder, "*.txt"):
         P = Path(fnam).expanduser().resolve()
         with open(fnam, "r", encoding="UTF-8") as ff:
             R = rgx.search(ff.read())
-            if R:
+            if R: 
                 petal_id = "PPC.{}".format(R.group(2))
                 side = R.group(1).lower()
                 if "_2D_" in P.name:
@@ -75,12 +84,20 @@ def main(folder, out_folder):
             list_file.write("{} {}-{} {}\n".format(oname, petal_id, side, petal_id))
             data = ""
             for data_type, fnam in values.items():
+                if fnam is None:
+                    print("This should not happen.")
+                    
                 with open(fnam, "r", encoding="UTF-8") as ifile:
                     data += ifile.read()
-                    
+
             with open(oname, "w", encoding="UTF-8") as ofile:
                 ofile.write(data)
-       
+
     list_file.close()
+    
 if __name__ == "__main__":
-    main("/Users/lacasta/Downloads/DESY_Production", "/tmp/desy")
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--input-folder", default=None, help="Input folder")
+    parser.add_argument("--output-folder", help="Outout fodler", default=None)
+    
+    main("/Users/lacasta/Downloads/Report_QC-desy_Sep2024", "/tmp/desy")

petal_qc/thermal/DESYdata.py

@@ -0,0 +1,58 @@
+"""Dialog to get DESY IRB files."""
+from pathlib import Path
+import itkdb_gtk
+import itkdb_gtk.dbGtkUtils
+
+import gi
+gi.require_version("Gtk", "3.0")
+from gi.repository import Gtk, Gio
+
+
+class DesyData(Gtk.Dialog):
+    """To get DESY data"""
+
+    def __init__(self, files=None):
+        super().__init__(title="DESY IRB files")
+
+        self.front = None
+        self.back = None
+
+        self.add_buttons(Gtk.STOCK_CANCEL, Gtk.ResponseType.CANCEL,
+                         Gtk.STOCK_OK, Gtk.ResponseType.OK)
+
+        area = self.get_content_area()
+        box = Gtk.Box(orientation=Gtk.Orientation.VERTICAL)
+        area.pack_start(box, True, True, 10)
+
+        lbl = Gtk.Label(label="Choose the IRB files.")
+        box.pack_start(lbl, True, True, 10)
+
+        grid = Gtk.Grid(column_spacing=10, row_spacing=5)
+        box.pack_start(grid, True, True, 10)
+        grid.attach(Gtk.Label(label="Front"), 0, 0, 1, 1)
+        grid.attach(Gtk.Label(label="Back"), 0, 1, 1, 1)
+
+        fback = Gtk.FileChooserButton()
+        fback.connect("file-set", self.on_file_set, 0)
+        ffront = Gtk.FileChooserButton()
+        ffront.connect("file-set", self.on_file_set, 1)
+        grid.attach(ffront, 1, 0, 1, 1)
+        grid.attach(fback, 1, 1, 1, 1)
+        
+        if files is not None and len(files)==2:
+            ffront.set_file(Gio.File.new_for_path(files[0]))
+            fback.set_file(Gio.File.new_for_path(files[1]))
+
+    def on_file_set(self, *args):
+        """Get back side file."""
+        fnam = args[0].get_filename()
+        if fnam is None or not Path(fnam).exists():
+            itkdb_gtk.dbGtkUtils.complain("Could not find Data File", fnam, parent=self)
+            return
+
+        if args[1] == 0:
+            self.back = fnam
+        elif args[1] == 1:
+            self.front = fnam
+        else:
+            itkdb_gtk.dbGtkUtils.complain("This should not happen.", fnam, parent=self)

petal_qc/thermal/IRBFile.py

@@ -166,11 +166,32 @@ class HeaderBlock(object):
         self.word8 = ifile.getInt()
         self.end = ifile.tell()
 
+        if self.end - self.where != 32:
+            raise RuntimeError("byte counting error in parsing of HeaderBlock")
+
     def get_name(self):
         """Return the header type as string."""
         return HeaderBlock.header_name[self.type]
 
 
+class IRBPreview(object):
+    """A preview in the IRB file."""
+
+    def __init__(self, block):
+        """Initialize.
+
+        Args:
+            block: the block with the information.
+
+        """
+        self.block = block
+        old_position = block.ifile.tell()
+
+
+
+        block.ifile.seek(old_position)
+
+
 class IRBImage(object):
     """An image in the IRBis file."""
 
@@ -181,7 +202,6 @@ class IRBImage(object):
         """Initialize.
 
         Args:
-        ----
             ifile: The input file
             block: The block with the information.
 
@@ -233,9 +253,8 @@ class IRBImage(object):
         zero = ifile.getShort()
 
         if self.width > 10000 or self.height > 10000:
-            print("Image size out of range ({}x{})".format(
+            raise RuntimeError("Image size out of range ({}x{})".format(
                 self.width, self.height))
-            return
 
         flagsPosition = block.offset + IRBImage.FLAGS_OFFSET
         self.readImageFlags(flagsPosition)
@@ -314,13 +333,18 @@ class IRBImage(object):
                 if v2_count < 1:
                     v2_count = v2_data[v2_pos]
                     v2 = v2_data[v2_pos+1]
-                    P1 = self.palette[v2+1]
+                    if v2 < 0:
+                        v2 += 256
+                    P1 = self.palette[(v2+1) % 256]
                     P2 = self.palette[v2]
                     v2_pos += 2
 
                 v2_count -= 1
 
                 v1 = v1_data[i]
+                if v1 < 0:
+                    v1 += 256
+
                 f = v1/256.0
 
                 # linear interpolation betweeen eighboring palette entries
@@ -423,6 +447,7 @@ class IRBFile(object):
         """
         self.concatenate = concatenate
         self.imgList = []
+        self.previewList = []
         self.blocks = []
         self.ifiles = []
         self.file_images = []
@@ -493,6 +518,9 @@ class IRBFile(object):
             if block.type == HeaderBlock.IMAGE:
                 image = IRBImage(block)
                 self.imgList.append(image)
+            elif block.type == HeaderBlock.PREVIEW:
+                image = IRBPreview(block)
+                self.previewList.append(image)
 
         if self.is_sequence:
             offset = -1
@@ -549,6 +577,10 @@ class IRBFile(object):
         self.current_file = i
         self.imgList = self.file_images[i]
 
+    def getImage_timestamp(self, i):
+        """Return timestamp of i-th image."""
+        return self.imgList[i].timestamp
+
     def getImage(self, i):
         """Get i-th image in current file.
 
@@ -682,7 +714,7 @@ class IRBFile(object):
         return out
 
 
-def open_file(fname):
+def open_file(fname, concat=False):
     """Opens a file an returns a IRBFile instance.
 
     Args:
@@ -694,7 +726,19 @@ def open_file(fname):
     if is_iterable(fname):
         # we assume it is a list of IRB files...
         try:
-            irbf = IRBFile(fname)
+            if concat:
+                irbf = IRBFile(fname)
+                return irbf
+            else:
+                out = []
+                for f in fname:
+                    irbf = IRBFile(f)
+                    out.append(irbf)
+                
+                if len(out) == 1:
+                    out = out[0]
+                    
+                return out
 
         except FileNotFoundError as eee:
             print(eee)
@@ -769,4 +813,4 @@ def main():
     plt.show()
 
 if __name__ == "__main__":
-    main()
+    main()

petal_qc/thermal/IRCore.py

@@ -41,7 +41,7 @@ class IRCore(object):
         self.files = []
         self.results = [] if results is None else results  # list of AnalysisResults. One per side
         self.golden = []                                    # list of Golden results. One per side.
-        self.inlet = -9999
+        self.inlet = params.tco2
 
     def set_files(self, files):
         """Set the input files."""

petal_qc/thermal/IRDataGetter.py

@@ -19,10 +19,10 @@ HAS_GRAPHANA = False
 try:
     from petal_qc.utils.readGraphana import ReadGraphana
     HAS_GRAPHANA = True
-    
+
 except ImportError:
     HAS_GRAPHANA = False
-    
+
 
 
 class IRDataGetter(object):
@@ -62,7 +62,7 @@ class IRDataGetter(object):
             kargs: keyword arguments to pass.
 
         """
-        pass
+        return
 
     def find_reference_image(self, irbf, *args, **kargs):
         """Find first image in sequence with T < T_min.
@@ -114,7 +114,7 @@ class IRDataGetter(object):
             An array of AnalysisResult objects.
 
         """
-        pass
+        return
 
     def get_analysis_frame(self, irbf):
         """Get the frame where we want to perform the analysis."""
@@ -166,12 +166,12 @@ class IRDataIFIC(IRDataGetter):
         #      T_min = -22.0
         #  else:
         #      T_min = args[0]
-#  
+        #
         #  irbf.set_concatenate(True)
         #  min_T, i_min, ref_img = IRPetal.find_reference_image(irbf, T_min)
         #  values = self.get_IR_data(ref_img)
         #  self.factor = values.shape[0]/640
-#  
+        #
         #  return min_T, i_min, [values, ]
 
     def get_IR_data(self, image, **kargs):
@@ -281,10 +281,10 @@ class IRDataIFIC(IRDataGetter):
             img = self.analysis_frame[0]
             X, T = self.DB.get_temperature(img.timestamp, 1)
             return np.min(T)
-        
+
         else:
             return -9999
-            
+
 
 
 class IRDataDESY(IRDataGetter):
@@ -343,14 +343,28 @@ class IRDataDESY(IRDataGetter):
             nframes = -1
 
         min_T = 1e50
-        avg_img = irbf.append_average(nframes=nframes)
-        for img in avg_img:
-            val = np.min(img.image)
-            if val < min_T:
-                min_T = val
+        i_min = -1
+        try:
+            avg_img = irbf.append_average(nframes=nframes)
+            for img in avg_img:
+                val = np.min(img.image)
+                if val < min_T:
+                    min_T = val
 
-        i_min = irbf.nimages-1
-        values = self.get_IR_data(avg_img)
+            i_min = irbf.nimages-1
+            values = self.get_IR_data(avg_img)
+
+        except AttributeError:
+            # We have 2 files, one front y the second back
+            values = []
+            factors = []
+            for ifile in irbf:
+                val = self.get_IR_data(ifile.getImage(0))
+                Tmin = np.min(val)
+                if Tmin < min_T:
+                    min_T = Tmin
+                    i_min = 0
+                values.append(val)
 
         factors = []
         for img in values:
@@ -359,25 +373,26 @@ class IRDataDESY(IRDataGetter):
 
         return min_T, i_min, values
 
-    def extract_pipe_path(self, images, params) -> list:
+    def extract_pipe_path(self, image, params) -> list:
         """Extract the "pipe path"  in a petal IR image.
 
         Args:
-        ----
-            images(list(ndarray)): The array of 2D arrays containing the images
+            image(list(ndarray)): The array of 2D arrays containing the images
             params: IRPetalPam object with options.
 
         Returns
-        -------
             pipe: the list of pipe contours or paths.
 
         """
         pipes = []
-        for img in images:
+        points_3d = []
+        for img in image:
             pipe = IRPetal.extract_pipe_path(img, params)
+            points_3d.append(IRPetal.get_all_3d_points())
             pipes.append(pipe)
 
-        IRPetal.set_images(images)
+        IRPetal.set_all_3d_points(points_3d)
+        IRPetal.set_images(image)
         return pipes
 
     def analyze_IR_image(self, img, pipes, sensors, iside, params):
@@ -401,7 +416,11 @@ class IRDataDESY(IRDataGetter):
 
         DESY gets the average of all frames.
         """
-        if not irbf.has_average:
-            irbf.append_average(nframes=5)
+        try:
+            if not irbf.has_average:
+                irbf.append_average(nframes=5)
+
+            return [v[-1] for v in irbf.file_images]
 
-        return [v[-1] for v in irbf.file_images]
+        except AttributeError:
+            return [irbf[0].getImage(0), irbf[1].getImage(0)]

petal_qc/thermal/IRPetal.py

@@ -35,7 +35,7 @@ debug_plot = DebugPlot.DebugPlot()
 the_segments = None  # global variable to store the last segments found.
 the_contours = None  # Global variable to store the last contours found
 the_images = None
-
+the_3d_points = None
 
 class DebugTempProfile(object):
     """Stores Debug data."""
@@ -68,6 +68,16 @@ class DebugTempProfile(object):
 T_profile = None      # Global to store the temp. profile.
 
 
+def get_all_3d_points():
+    """Return all pipe path 3D points."""
+    return the_3d_points
+
+def set_all_3d_points(all_3d_points):
+    """Set all pipe path 3D points."""
+    global the_3d_points
+    the_3d_points = all_3d_points
+
+
 def get_last_segments():
     """Return the last segments found."""
     global the_segments
@@ -145,6 +155,39 @@ class Segment(object):
 
         return points
 
+    @staticmethod
+    def get_3d_points_in_list(segments):
+        """Return a list with all the 3D points (x,y,T) in the list of segments.
+
+        Args:
+        ----
+            segments (list[Segment]): The list of segments
+
+        Returns
+        -------
+            np.array: array of points.
+
+        """
+        # Count the number of points
+        npts = 0
+        for S in segments:
+            if S.Pmin is not None:
+                npts += len(S.Pmin)
+
+        points = np.zeros([npts, 3])
+        ipoint = 0
+        for S in segments:
+            if S.Pmin is None:
+                continue
+
+            for i, Pmin in enumerate(S.Pmin):
+                points[ipoint, 0] = Pmin.x
+                points[ipoint, 1] = Pmin.y
+                points[ipoint, 2] = S.Tmin[i]
+                ipoint += 1
+
+        return points
+
     @staticmethod
     def get_spread_in_list(segments):
         """Return an array with spread values."""
@@ -323,7 +366,7 @@ def extract_mirrored_pipe_path(values, params):
         (left_pipe, right_pipe): The 2 paths.
 
     """
-    global the_images
+    global the_images, the_3d_points
     if params.rotate:
         rotated = rotate_full_image(values)
     else:
@@ -331,11 +374,14 @@ def extract_mirrored_pipe_path(values, params):
 
     imgs = split_IR_image(rotated)
     pipes = []
+    points_3d = []
     for img in imgs:
         pipe = extract_pipe_path(img, params)
         pipes.append(pipe)
+        points_3d.append(get_all_3d_points())
 
     the_images = imgs
+    the_3d_points = points_3d
     return list(pipes)
 
 
@@ -684,7 +730,7 @@ def get_derivatives(T_min, x_min, x_max, coeff):
     return valid_roots, valid_values, valid_curv
 
 
-def show_profile(debug_plot):
+def show_profile(debug_plot, title=None):
     """Plot a profile."""
     debug_plot.setup_debug("Slices", is_3d=True)
     if T_profile:
@@ -693,6 +739,12 @@ def show_profile(debug_plot):
         if T_profile._pF is not None:
             debug_plot.plotx("Slices", T_profile._pF[:, 0], T_profile._pF[:, 1], T_profile._Fn, 'b-')
 
+        if title:
+            debug_plot.set_title("Slices", title)
+
+
+__call_id__ = 0
+__img_num__ = 0
 
 def slice_contours(data, inner, outer, distance=50, npoints=15, do_fit=False, show=False) -> list[Segment]:
     """Make slices between contours.
@@ -712,8 +764,11 @@ def slice_contours(data, inner, outer, distance=50, npoints=15, do_fit=False, sh
         A list of Segments.
 
     """
-    global debug_plot
+    global debug_plot, __call_id__, __img_num__
 
+    __call_id__ += 1
+    __img_num__ = 0
+    #show=True
     segments = []
     dist = 0
     npts = len(outer)
@@ -726,7 +781,15 @@ def slice_contours(data, inner, outer, distance=50, npoints=15, do_fit=False, sh
     U = contours.find_closest(x0, y0, inner)
     Tmin, Pmin, spread = get_T_profile(data, Point(x0, y0), Point(U[0], U[1]), npts=npoints, do_fit=do_fit, debug=False)
     if show:
-        show_profile(debug_plot)
+        show_profile(debug_plot, "Dist {:.3f}, Tmin {:.2f} ({:.1f}, {:.1f})".format(dist, Tmin[0], Pmin[0].x, Pmin[0].y))
+        debug_plot.savefig("Slices", "/tmp/img_{}_{}.png".format(__call_id__, __img_num__))
+        __img_num__ += 1
+        try:
+            ax = debug_plot.get_ax("Contours")
+            ax.plot([x0, U[0]], [y0, U[1]], '-')
+            ax.plot(Pmin[0].x, Pmin[0].y, 'o')
+        except KeyError:
+            pass
 
     # store results: distance along segment and temperature
     T = []
@@ -764,7 +827,16 @@ def slice_contours(data, inner, outer, distance=50, npoints=15, do_fit=False, sh
 
                 segments.append(Segment((x0, y0), V, Pmin, path_length, Tmin, spread))
                 if show:
-                    show_profile(debug_plot)
+                    show_profile(debug_plot, "Dist {:.3f} Tmin {:.2f} ({:.1f}, {:.1f})".format(path_length, Tmin[0], Pmin[0].x, Pmin[0].y))
+                    debug_plot.savefig("Slices", "/tmp/img_{}_{}.png".format(__call_id__, __img_num__))
+                    __img_num__ += 1
+                    
+                    try:
+                        ax = debug_plot.get_ax("Contours")
+                        ax.plot([x0, V[0]], [y0, V[1]], '-')
+                        ax.plot(Pmin[0].x, Pmin[0].y, 'o')
+                    except KeyError:
+                        pass
 
             dist = 0
 
@@ -772,6 +844,7 @@ def slice_contours(data, inner, outer, distance=50, npoints=15, do_fit=False, sh
     if show:
         debug_plot.setup_debug("SlicePipe")
         debug_plot.plot("SlicePipe", D, T, '-o')
+        debug_plot.savefig("SlicePipe", "/tmp/slice_pipe_{}.png".format(__call_id__))
 
     return segments
 
@@ -829,10 +902,12 @@ def find_image_contours(data, params):
             print("contour area: {:.3f}".format(area))
 
     if params.debug:
+        debug_plot.setup_debug("Contours", 1, 1)
         colors = ["#ff9b54", "#ff7f51"]  # ["#540b0e", "#9e2a2b"]
         print("Number of countours:", len(out))
 
-        fig, ax = plt.subplots(1, 1, tight_layout=True)
+        ax = debug_plot.get_ax("Contours")
+        ax.clear()
         ax.imshow(data, origin='lower', cmap='jet')
         for i, cnt in enumerate(out):
             ax.plot(cnt[:, 0], cnt[:, 1], linewidth=3, color=colors[i % 2])
@@ -870,6 +945,7 @@ def get_segments(data, params) -> list[Segment]:
         list of segments. See `slice_contours`
 
     """
+    global the_3d_points
     cntrs = find_image_contours(data, params)
     if cntrs is None:
         return None
@@ -881,6 +957,7 @@ def get_segments(data, params) -> list[Segment]:
                                   do_fit=params.do_fit,
                                   show=params.debug)
 
+    the_3d_points = Segment.get_3d_points_in_list(the_segments)
     return the_segments
 
 

petal_qc/thermal/IRPetalParam.py

@@ -12,7 +12,7 @@ class IRPetalParam(object):
             values: ArgParser or dict with user values-
 
         """
-        self.institute = 'IFIC'   # Either IFIC or DESY to treat the different files
+        self.institute = None     # Either IFIC or DESY to treat the different files
         self.thrs = -22.0         # the threshold
         self.tco2 = -35.0         # Inlet temperature
         self.gauss_size = 15      # Radius of gausian filtering

petal_qc/thermal/Petal_IR_Analysis.py

@@ -30,7 +30,7 @@ class AnalysisResult(object):
                 continue
 
 
-def show_2D_image(img, title=None):
+def show_2D_image(img, title=None, show_fig=True):
     """Show a 2D image."""
     if isinstance(img, list):
         fig, ax = plt.subplots(1, len(img))
@@ -51,8 +51,9 @@ def show_2D_image(img, title=None):
         pcm = ax.imshow(img, origin='lower', cmap=HighContrast.reversed())  # cmap="jet")
         fig.colorbar(pcm, ax=ax)
 
-    plt.draw()
-    plt.pause(0.001)
+    if show_fig:
+        plt.draw()
+        plt.pause(0.001)
 
     return fig, ax
 

petal_qc/thermal/PipeFit.py

@@ -413,6 +413,14 @@ class PipeFit(object):
         """
         if M0 is None:
             M = self.initial_guess(data)
+            for i, val in enumerate(M):
+                if val < self.bounds[0][i]:
+                    val = 1.01 *  self.bounds[0][i]
+                    M[i] = val
+                elif val > self.bounds[1][i]:
+                    val = 0.99 *  self.bounds[0][i]
+                    M[i] = val
+                    
             if self.debug:
                 print("\n** Initial guess")
                 self.print_transform(M)
@@ -498,7 +506,7 @@ class PipeFit(object):
 
         return X, Y
 
-    def plot(self):
+    def plot(self, show_fig=True):
         """Plot result of fit and data."""
         fig, ax = plt.subplots(1, 1, tight_layout=True)
         ax.plot(self.pipe[:, 0], self.pipe[:, 1], label="Pipe")
@@ -509,8 +517,9 @@ class PipeFit(object):
 
         ax.plot(out[:, 0], out[:, 1], 'o', label="Data")
         ax.legend()
-        plt.draw()
-        plt.pause(0.0001)
+        if show_fig:
+            plt.draw()
+            plt.pause(0.0001)
         return fig, ax