pypff 1.0.0__py3-none-any.whl

pypff/__init__.py

@@ -0,0 +1,8 @@
+from .io import datapff
+from .io2 import PanosetiRun, PFFSequence, hkpff, qconfig
+from .models import QuaboHeader, ModuleHeader, PFFHeader, FrameConfig
+from . import pixelmap
+from .pixelmap_maroc2phys_bga import maroc2phys_bga
+from .pixelmap_maroc2phys_qfp import maroc2phys_qfp
+from .pixelmap_phys2maroc_bga import phys2maroc_bga
+from .pixelmap_phys2maroc_qfp import phys2maroc_qfp

pypff/_cli/profile.py

@@ -0,0 +1,54 @@
+import typer
+from pathlib import Path
+from typing import Annotated, Optional
+from pypff.profiling import Profiler
+from rich.console import Console
+
+app = typer.Typer()
+
+@app.command()
+def run(
+    run_dir: Annotated[Path, typer.Argument(help="Path to the .pffd run directory.")],
+    products: Annotated[Optional[list[str]], typer.Option("--product", "-p", help="Specific products to test. Defaults to all.")] = None,
+    n_frames: Annotated[int, typer.Option("--frames", "-n", help="Number of frames to test per product.")] = 1000,
+    step: Annotated[int, typer.Option("--step", "-s", help="Step size for strided read test.")] = 10,
+    random_samples: Annotated[int, typer.Option("--random", "-r", help="Number of random access samples.")] = 100,
+):
+    """Run performance benchmarks on a PanoSETI run."""
+    console = Console()
+    
+    if not run_dir.exists():
+        console.print(f"[red]❌ Run directory {run_dir} does not exist.[/]")
+        raise typer.Exit(1)
+    
+    profiler = Profiler(run_dir)
+    target_products = products or profiler.run.list_products()
+    
+    if not target_products:
+        console.print("[yellow]⚠️ No data products found in the directory.[/]")
+        return
+
+    with console.status("[bold green]Running benchmarks...[/]"):
+        for prod in target_products:
+            try:
+                # 1. Sequential Read
+                profiler.profile_sequential_read(prod, n_frames=n_frames)
+                
+                # 2. Bulk Read
+                profiler.profile_bulk_read(prod, n_frames=n_frames)
+                
+                # 3. Strided Read
+                profiler.profile_strided_read(prod, step=step, n_frames=n_frames // step)
+                
+                # 4. Random Access
+                profiler.profile_random_access(prod, n_samples=random_samples)
+                
+                # 5. Metadata
+                profiler.profile_metadata_extraction(prod)
+            except Exception as e:
+                console.print(f"[red]Error profiling {prod}: {e}[/]")
+
+    profiler.display_results()
+
+if __name__ == "__main__":
+    app()

pypff/_cli/root.py

@@ -0,0 +1,19 @@
+import typer
+from pathlib import Path
+from typing import Annotated
+from pypff import PanosetiRun
+
+app = typer.Typer()
+
+@app.command()
+def show(
+    run_dir: Annotated[Path, typer.Argument(help="Path to the .pffd run directory.")],
+    details: Annotated[bool, typer.Option("--details", "-d", help="Show individual PFF files.")] = False,
+):
+    """Explore the structure of a PanoSETI run."""
+    if not run_dir.exists():
+        print(f"❌ Run directory {run_dir} does not exist.")
+        raise typer.Exit(1)
+    
+    run = PanosetiRun(run_dir)
+    run.show(details=details)

pypff/_cli/test.py

@@ -0,0 +1,57 @@
+import typer
+import subprocess
+import sys
+from typing import Annotated
+
+app = typer.Typer(help="Run pypff test suite.")
+
+@app.command()
+def all(
+    lint: Annotated[bool, typer.Option("--lint", help="Run linters (Ruff/MyPy)")] = False,
+    cov: Annotated[bool, typer.Option("--cov", help="Enable coverage reporting")] = False,
+):
+    """Run all pypff tests (Unit + Logic + Legacy)."""
+    if lint:
+        run_lint()
+
+    cov_args = ["--cov=src/pypff", "--cov-report=term-missing"] if cov else []
+
+    print("Running Tier 1 (Unit) tests...")
+    subprocess.run([sys.executable, "-m", "pytest"] + cov_args + ["src/ci/tier1_unit"], check=True)
+    
+    if cov:
+        cov_args.append("--cov-append")
+
+    print("Running Tier 2 (Logic) tests...")
+    subprocess.run([sys.executable, "-m", "pytest"] + cov_args + ["src/ci/tier2_logic"], check=True)
+    
+    print("Running Legacy Integration tests...")
+    subprocess.run([sys.executable, "-m", "pytest"] + cov_args + ["src/ci/legacy_tests"], check=True)
+
+@app.command()
+def unit():
+    """Run Tier 1 (Unit) tests."""
+    print("Running Tier 1 (Unit) tests...")
+    subprocess.run([sys.executable, "-m", "pytest", "src/ci/tier1_unit"], check=True)
+
+@app.command()
+def logic():
+    """Run Tier 2 (Logic) tests."""
+    print("Running Tier 2 (Logic) tests...")
+    subprocess.run([sys.executable, "-m", "pytest", "src/ci/tier2_logic"], check=True)
+
+@app.command()
+def legacy():
+    """Run Legacy Integration tests."""
+    print("Running Legacy Integration tests...")
+    subprocess.run([sys.executable, "-m", "pytest", "src/ci/legacy_tests"], check=True)
+
+@app.command()
+def lint():
+    """Run linters (Ruff/MyPy)."""
+    run_lint()
+
+def run_lint():
+    print("Running linters...")
+    subprocess.run([sys.executable, "-m", "ruff", "check", "."], check=True)
+    subprocess.run([sys.executable, "-m", "mypy", "src"], check=True)

pypff/_cli/zarr.py

@@ -0,0 +1,46 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Annotated
+
+import typer
+
+app = typer.Typer(help="PFF → Zarr v3 conversion tools.")
+
+
+@app.command()
+def convert(
+    obs_dir: Annotated[Path, typer.Argument(help="Input .pffd observation directory")],
+    out_dir: Annotated[Path, typer.Argument(help="Output directory for .zarr stores")],
+    codec: Annotated[str, typer.Option(help="Compression codec: zstd, blosc-lz4, gzip, none")] = "zstd",
+    level: Annotated[int, typer.Option(help="Compression level (codec-specific)")] = 3,
+    time_chunk: Annotated[
+        int, typer.Option(help="Frames per time chunk (0 = auto-size to ~8 MB)")
+    ] = 0,
+) -> None:
+    """Convert all data products in a .pffd observation directory to Zarr v3 stores."""
+    from pypff.io2 import PanosetiRun
+    from pypff.zarr import convert_run
+
+    if not obs_dir.exists():
+        typer.echo(f"Error: {obs_dir} does not exist", err=True)
+        raise typer.Exit(1)
+
+    run = PanosetiRun(obs_dir)
+    products = run.list_products()
+    if not products:
+        typer.echo(f"No data products found in {obs_dir}", err=True)
+        raise typer.Exit(1)
+
+    typer.echo(f"Found {len(products)} product(s) in {obs_dir.name}:")
+    for p in products:
+        seq = run.get_product(p)
+        typer.echo(f"  {p}: {len(seq):,} frames")
+
+    chunk = time_chunk if time_chunk > 0 else None
+    stores = convert_run(run, out_dir, codec=codec, level=level, time_chunk=chunk)
+
+    typer.echo(f"\nWrote {len(stores)} Zarr store(s) to {out_dir}:")
+    for s in stores:
+        size_mb = sum(f.stat().st_size for f in s.rglob("*") if f.is_file()) / 1024**2
+        typer.echo(f"  {s.name}  ({size_mb:.1f} MB)")

pypff/cli.py

@@ -0,0 +1,55 @@
+from __future__ import annotations
+
+import os
+from typing import Annotated, Any
+
+import typer
+
+# Local Imports
+from .util.cli import BaseLazyGroup, display_tree_callback
+
+
+class PypffLazyGroup(BaseLazyGroup):
+    """
+    Custom Click Group that lazy-loads commands from other modules.
+    Ensures that heavy dependencies (like NumPy or Rich) aren't loaded
+    until a specific command is actually executed.
+    """
+
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
+        lazy_mapping = {
+            "show": ("pypff._cli.root", "show", "Explore the structure of a PanoSETI run."),
+            "test": ("pypff._cli.test", "app", "Run pypff test suite."),
+            "profile": ("pypff._cli.profile", "app", "Run performance benchmarks on a PanoSETI run."),
+            "zarr": ("pypff._cli.zarr", "app", "PFF → Zarr v3 conversion tools."),
+        }
+        super().__init__(*args, lazy_mapping=lazy_mapping, **kwargs)
+
+
+app = typer.Typer(
+    cls=PypffLazyGroup,
+    help="Pypff Management CLI",
+    context_settings={"help_option_names": ["-h", "--help"]},
+)
+
+
+@app.callback(invoke_without_command=True)
+def main(
+    ctx: typer.Context,
+    tree: Annotated[
+        bool,
+        typer.Option(
+            "--tree",
+            "-t",
+            help="Display the command tree and exit.",
+            callback=display_tree_callback,
+        ),
+    ] = False,
+) -> None:
+    """PYPFF I/O CLI."""
+    if ctx.invoked_subcommand is None and not tree:
+        print(ctx.get_help())
+
+
+if __name__ == "__main__":
+    app()

pypff/io.py

@@ -0,0 +1,336 @@
+'''
+This module provides methods to reading pff data file, including img16, img8, ph256, ph1024 and hk.pff
+'''
+import json
+import datetime
+import numpy as np
+import mmap
+from glob import glob
+from . import pixelmap
+
+MOBO_DIM = 16
+QUABO_DIM = 32
+
+# The metadata loc in the data is hard-coded here.
+loc_arr = np.zeros(2, dtype=object)
+# metadata loc for ph256
+# metadata example
+'''
+b'{ "quabo_num": 0, "pkt_num":      32280, "pkt_tai":  398, "pkt_nsec": 723300414, "tv_sec": 1690934633, "tv_usec": 720082}\n'
+'''
+loc_arr[0] = {
+    'quabo_num' : [14, 16],
+    'pkt_num'   : [28, 39],
+    'pkt_tai'   : [51, 56],
+    'pkt_nsec'  : [69, 79],
+    'tv_sec'    : [90, 101],
+    'tv_usec'   : [113, 120]
+}
+# metadata loc for ph1024, img16 and img8
+# metadata example
+'''
+'{\n   
+    "quabo_0": { "pkt_num":      23855, "pkt_tai":  906, "pkt_nsec": 774507484, "tv_sec": 1691048805, "tv_usec": 778782}, \n   
+    "quabo_1": { "pkt_num":      16262, "pkt_tai":  906, "pkt_nsec": 774507492, "tv_sec": 1691048805, "tv_usec": 778789}, \n   
+    "quabo_2": { "pkt_num":       9069, "pkt_tai":  906, "pkt_nsec": 774507484, "tv_sec": 1691048805, "tv_usec": 778800}, \n   
+    "quabo_3": { "pkt_num":       1234, "pkt_tai":  906, "pkt_nsec": 774507484, "tv_sec": 1691048805, "tv_usec": 778804}
+    \n}\n'
+'''
+loc_arr[1] = {
+    'quabo_0':
+    {
+        'pkt_num': [28 ,39],
+        'pkt_tai': [51 ,56],
+        'pkt_nsec': [69, 79],
+        'tv_sec': [90, 101],
+        'tv_usec': [113, 120]
+    },
+    'quabo_1':
+    {
+        'pkt_num': [150 ,161],
+        'pkt_tai': [173 ,178],
+        'pkt_nsec': [191, 201],
+        'tv_sec': [212, 223],
+        'tv_usec': [235, 242]
+    },
+    'quabo_2':
+    {
+        'pkt_num': [272 ,283],
+        'pkt_tai': [295 ,300],
+        'pkt_nsec': [313, 323],
+        'tv_sec': [334, 345],
+        'tv_usec': [357, 364]
+    },
+    'quabo_3':
+    {
+        'pkt_num': [394 ,405],
+        'pkt_tai': [417 ,422],
+        'pkt_nsec': [435, 445],
+        'tv_sec': [456, 467],
+        'tv_usec': [479, 486]
+    }
+}  
+md_loc = {
+    'ph256': loc_arr[0],
+    'ph1024': loc_arr[1],
+    'img16': loc_arr[1],
+    'img8': loc_arr[1]
+}
+# generate dict template
+#
+def _gen_dict_template(d):
+    template = {}
+    for k in d:
+        # chagne TEMP1 to DET_TEMP, and change TEMP1 to FPGA_TEMP
+        if k == 'TEMP1':
+            k = 'DET_TEMP'
+        if k == 'TEMP2':
+            k = 'FPGA_TEMP'
+        template[k] = []
+    return template
+
+class hkpff(object):
+    '''
+    Description:
+        The hkpff class reads hk.pff, and returns a dict, including housekeeping of quabo, wrs, wps and gps
+    '''
+    def __init__(self,fn='hk.pff'):
+        '''
+        Description:
+            Create a hkpff object based on the filename.
+        Input:
+            -- fn(str): file name of a hk.pff 
+        '''
+        self.fn = fn
+        self.hk_info = {}
+                
+    def readhk(self):
+        '''
+        Description:
+            Read hk.pff, and convert the info to a dict.
+        Output:
+            -- hk_info(dict): a dict contains all of the hk info.
+        '''
+        with open(self.fn, 'rb') as f:
+            hk_lines = f.readlines()
+        for hk_str in hk_lines:
+            try:
+                hk = json.loads(hk_str)
+            except:
+                continue
+            key, = hk.keys()
+            # check if the key is already in the hk_info
+            if(not key in self.hk_info):
+                template = _gen_dict_template(hk[key])
+                self.hk_info[key] = template
+            for k,v in hk[key].items():
+                # chagne TEMP1 to DET_TEMP, and change TEMP1 to FPGA_TEMP
+                if k == 'TEMP1':
+                    k = 'DET_TEMP'
+                if k == 'TEMP2':
+                    k = 'FPGA_TEMP'
+                try:
+                    # if the type of value is int
+                    self.hk_info[key][k].append(int(v))
+                except:
+                    try:
+                        # if the type of value is float
+                        self.hk_info[key][k].append(float(v))
+                    except:
+                        self.hk_info[key][k].append(v)
+        return self.hk_info
+
+
+class datapff(object):
+    '''
+    Description:
+        The datapff class reads all kinds of data files, including img16, img8, ph256, ph1024.
+    '''
+
+    def __init__(self, fn):
+        '''
+        Description:
+            Read data from a data pff file.
+        Input:
+            -- fn(str): pff file name.
+        '''
+        self.fn = fn
+        fn_str = fn.split('/')[-1]
+        info = fn_str.split('.')
+        stringIndex = 0
+        if len(info[0]) != 0:
+            stringIndex = 0
+        else:
+            stringIndex = 1
+        startdt_str = info[stringIndex].split('_')[1]
+        stringIndex += 1
+        # It looks like we have two formats of file name
+        try:
+            self.startdt = datetime.datetime.strptime(startdt_str, '%Y-%m-%dT%H:%M:%SZ')
+        except:
+            # macos
+            self.startdt = datetime.datetime.strptime(startdt_str, '%Y-%m-%dT%H-%M-%SZ')
+        self.dp = info[stringIndex].split('_')[1]
+        stringIndex += 1
+        self.bpp = int(info[stringIndex].split('_')[1])
+        stringIndex += 1
+        self.module = int(info[stringIndex].split('_')[1])
+        stringIndex += 1
+        self.seqno = int(info[stringIndex].split('_')[1])
+        if self.dp == 'ph256':
+            self._md_size = 124
+            self._pixels = 256
+            self._d_size = self._pixels * self.bpp
+            self.datasize = self._md_size + self._d_size
+        else:
+            # TODO: check if the metadata size for ph1024/img16/img8 is the same
+            self._md_size = 492
+            self._pixels = 1024
+            self._d_size = self._pixels * self.bpp
+            self.datasize = self._md_size + self._d_size
+        if self.dp == 'ph256' or self.dp == 'ph1024':
+            self.dtype = np.int16
+        elif self.dp == 'img16':
+            self.dtype = np.uint16
+        else:
+            self.dtype = np.uint8
+        self.metadata = {}
+
+    def readpff(self, samples=-1, skip = 0, pixel = -1, ver='qfb', metadata=False, mode='mmap'):
+        '''
+        Description:
+            Read data from a data pff file.
+        Inputs:
+            -- samples(int): The sample number to be read out.
+                             If it's -1, all of the data will be read out.
+                             Default = -1
+            -- skip(int): Skip the number of smaples.
+                          Default = 0
+            -- pixel(int): select the pixel.
+                          If it's -1, we will get the data of all the channels.
+                          Default = -1
+            -- quabo(int): It specifies the quabo number on the mobo.
+                          Default = 0
+            -- ver(str): quabo version.
+                        Default = 'qfp'
+            -- metadata(bool): If True, read Medatadata out.
+                        Default = False
+            -- mode(str): reading mode. 'mmap' or 'read'.
+                        Default = 'mmap'
+
+        Outputs:
+            -- metadata(dict): a dict contains the metadata from each sample.
+            -- data(np.array): data array.
+        '''
+        # get metadata location, which is hard-coded above
+        metadata_loc = md_loc[self.dp]
+        # read data out from a ph256, img16 or ph1024 file
+        with open(self.fn,'rb') as f:
+            if mode == 'mmap':
+                mm = mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ)
+                if samples == -1:
+                    tmp = np.frombuffer(mm,dtype = self.dtype)
+                else:
+                    tmp = np.frombuffer(mm, dtype=self.dtype, count=samples*int(self.datasize/self.bpp), offset=0)
+            elif mode == 'read':
+                if samples == -1:
+                    tmp = np.frombuffer(f.read(),dtype = self.dtype)
+                else:
+                    tmp = np.frombuffer(f.read(samples*int(self.datasize/self.bpp)), dtype=self.dtype)
+            else:
+                raise ValueError(f"mode({mode} is not supported.)")
+        # reshape the data
+        tmp.shape = (-1, int(self.datasize/self.bpp))
+        # get data
+        self.data = tmp[:, int(self._md_size/self.bpp):]
+        if metadata==True and tmp.shape[0] != 0:
+            # we need to skip the '* ', which are 2 bytes
+            if self.bpp == 1:
+                metadataraw = tmp[:,0: int(self._md_size/self.bpp) - 2]
+            else:
+                metadataraw = tmp[:,0: int(self._md_size/self.bpp) - 1]
+            metadataraw = metadataraw.tobytes()
+            # convert byte to int8
+            metadataraw = np.frombuffer(metadataraw, dtype=np.int8)
+            metadataraw.shape = (-1, self._md_size - 2) 
+            # create metadata template
+            md_json = json.loads(metadataraw[0].tobytes().decode('utf-8')) 
+            if self.dp == 'ph1024' or self.dp == 'img16' or self.dp == 'img8':
+                # ph1024, img16 and img8 data has two stages of metadata
+                template = _gen_dict_template(md_json)
+                for key in template.keys():
+                    subtemplate = _gen_dict_template(md_json[key])
+                    template[key] = subtemplate
+                self.metadata = template
+                for k in metadata_loc.keys():
+                    for subk in metadata_loc[k].keys():
+                        # get the start row and end row from the metadata_loc
+                        r0 = metadata_loc[k][subk][0]
+                        r1 = metadata_loc[k][subk][1]
+                        tmp = metadataraw[:, r0:r1]
+                        # covert int8 to string
+                        tmp = tmp.view(f'S{r1-r0}')
+                        self.metadata[k][subk] = tmp.astype(np.uint64)
+            elif self.dp == 'ph256':
+                template = _gen_dict_template(md_json)
+                # ph256 data has one stage of metadata
+                self.metadata = template
+                for k in metadata_loc.keys():
+                    # get the start row and end row from the metadata_loc
+                    r0 = metadata_loc[k][0]
+                    r1 = metadata_loc[k][1]
+                    tmp = metadataraw[:, r0:r1]
+                    # covert int8 to string
+                    tmp = tmp.view(f'S{r1-r0}')
+                    self.metadata[k] = tmp.astype(np.uint64)
+            else:
+                raise Exception('Data type is not supproted: %s'%(self.dp))
+        if self.dp == 'ph256':
+            for k in self.metadata.keys():
+                self.metadata[k] = np.array(self.metadata[k].flat)
+        elif self.dp == 'img16' or self.dp == 'img8' or self.dp == 'ph1024':
+            for k in self.metadata.keys():
+                for kk in self.metadata[k].keys():
+                    self.metadata[k][kk] = np.array(self.metadata[k][kk].flat)
+        if pixel != -1:
+            self.data = self.data[:,pixel]
+        return self.data, self.metadata
+
+
+
+class qconfig(object):
+    '''
+    Description:
+        This class is used for reading config json files, including obs_config, daq_config, data_config, quabo_config...
+    '''
+    def __init__(self, fn):
+            self.config = {}
+            jfiles = glob(fn)
+            if len(jfiles) == 0:
+                raise Exception("The config file(%s) can not be found!"%(fn))
+            for file in jfiles:
+                key = file.split('/')[-1][:-5]
+                with open(file,'rb') as f:
+                    config = json.load(f)
+                self.config[key] = {}
+                for k, v in config.items():
+                    # if it's quabo_config*, we need to convert the str to int
+                    if(key.startswith('quabo_config')):
+                        try:
+                            tmp = v.split(',')
+                        except:
+                            tmp = []
+                        if len(tmp) == 4:
+                            self.config[key][k] = []
+                            for vv in tmp:
+                                if(vv.startswith('0x')):
+                                    self.config[key][k].append(int(vv,16))
+                                else:
+                                    self.config[key][k].append(int(vv,10))
+                        else:
+                            if(v.startswith('0x')):
+                                self.config[key][k] = int(v,16)
+                            else:
+                                self.config[key][k] = int(v,10)
+                    else:
+                        self.config[key] = config