PycWB 0.22.1__tar.gz → 0.23.0__tar.gz

{pycwb-0.22.1 → pycwb-0.23.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: PycWB
-Version: 0.22.1
+Version: 0.23.0
 Summary: A Python package for coherent gravitational wave burst analysis
 Home-page: https://git.ligo.org/yumeng.xu/pycwb
 Author-email: The PycWB team <yumeng.xu@ligo.org>

{pycwb-0.22.1 → pycwb-0.23.0}/PycWB.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: PycWB
-Version: 0.22.1
+Version: 0.23.0
 Summary: A Python package for coherent gravitational wave burst analysis
 Home-page: https://git.ligo.org/yumeng.xu/pycwb
 Author-email: The PycWB team <yumeng.xu@ligo.org>

{pycwb-0.22.1 → pycwb-0.23.0}/PycWB.egg-info/SOURCES.txt

@@ -231,10 +231,11 @@ prototypes/performance/numba_test.py
 prototypes/performance/taichi_test.py
 pycwb/__init__.py
 pycwb/_version.py
-pycwb/search.py
 pycwb/cli/__init__.py
-pycwb/cli/batch.py
+pycwb/cli/batch_runner.py
+pycwb/cli/batch_setup.py
 pycwb/cli/flow.py
+pycwb/cli/merge_catalog.py
 pycwb/cli/run.py
 pycwb/cli/server.py
 pycwb/cli/xtalk.py
@@ -269,6 +270,8 @@ pycwb/modules/data_conditioning/data_conditioning.py
 pycwb/modules/data_conditioning/module.yaml
 pycwb/modules/data_conditioning/regression.py
 pycwb/modules/data_conditioning/whitening.py
+pycwb/modules/energy_threshold/__init__.py
+pycwb/modules/energy_threshold/threshold.py
 pycwb/modules/job_segment/__init__.py
 pycwb/modules/job_segment/dq_segment.py
 pycwb/modules/job_segment/frame.py
@@ -326,6 +329,8 @@ pycwb/modules/super_cluster/sub_net_cut.py
 pycwb/modules/super_cluster/super_cluster.py
 pycwb/modules/super_cluster/supercluster.py
 pycwb/modules/super_cluster/utils.py
+pycwb/modules/supercluster_naive/__init__.py
+pycwb/modules/supercluster_naive/supercluster2.py
 pycwb/modules/superlag/__init__.py
 pycwb/modules/superlag/superlag.py
 pycwb/modules/web_viewer/__init__.py
@@ -336,6 +341,7 @@ pycwb/modules/workflow_utils/job_setup.py
 pycwb/modules/xtalk/__init__.py
 pycwb/modules/xtalk/monster.py
 pycwb/modules/xtalk/monster_old.py
+pycwb/modules/xtalk/xtalk_data.py
 pycwb/prefect_flow/__init__.py
 pycwb/prefect_flow/pycwb_flow.py
 pycwb/prefect_flow/tasks/__init__.py
@@ -360,6 +366,7 @@ pycwb/utils/dep_check.py
 pycwb/utils/image.py
 pycwb/utils/module.py
 pycwb/utils/network.py
+pycwb/utils/parser.py
 pycwb/utils/skymap_coord.py
 pycwb/utils/yaml_helper.py
 pycwb/utils/conversions/__init__.py
@@ -374,7 +381,10 @@ pycwb/vendor/web_viewer/styles.css
 pycwb/vendor/web_viewer/viewer.html
 pycwb/workflow/__init__.py
 pycwb/workflow/batch.py
+pycwb/workflow/merge.py
+pycwb/workflow/run.py
 pycwb/workflow/subflow/__init__.py
 pycwb/workflow/subflow/postprocess_and_plots.py
 pycwb/workflow/subflow/prepare_job_runs.py
+pycwb/workflow/subflow/process_job_segment.py
 pycwb/workflow/subflow/supercluster_and_likelihood.py

{pycwb-0.22.1 → pycwb-0.23.0}/bin/pycwb

@@ -7,9 +7,10 @@ import argparse
 import pycwb.cli.server as server
 import pycwb.cli.run as run
 import pycwb.cli.flow as flow
-import pycwb.cli.batch as batch
+import pycwb.cli.batch_setup as batch_setup
+import pycwb.cli.batch_runner as batch_runner
 import pycwb.cli.xtalk as xtalk
-
+import pycwb.cli.merge_catalog as merge_catalog
 
 def main():
     parser = argparse.ArgumentParser(prog='pycwb')
@@ -29,14 +30,22 @@ def main():
     flow.init_parser(flow_parser)
     flow_parser.set_defaults(func=flow.command)
 
-    batch_parser = subparsers.add_parser('batch', help='Run search')
-    batch.init_parser(batch_parser)
-    batch_parser.set_defaults(func=batch.command)
+    batch_parser = subparsers.add_parser('batch-setup', help='Setup batch run')
+    batch_setup.init_parser(batch_parser)
+    batch_parser.set_defaults(func=batch_setup.command)
+
+    batch_runner_parser = subparsers.add_parser('batch-runner', help='Runner for batch run')
+    batch_runner.init_parser(batch_runner_parser)
+    batch_runner_parser.set_defaults(func=batch_runner.command)
 
     xtalk_parser = subparsers.add_parser('xtalk', help='Convert xtalk file')
     xtalk.init_parser(xtalk_parser)
     xtalk_parser.set_defaults(func=xtalk.command)
 
+    merge_catalog_parser = subparsers.add_parser('merge-catalog', help='Merge catalog files')
+    merge_catalog.init_parser(merge_catalog_parser)
+    merge_catalog_parser.set_defaults(func=merge_catalog.command)
+
     args = parser.parse_args()
     if hasattr(args, 'func'):
         args.func(args)

{pycwb-0.22.1 → pycwb-0.23.0}/pycwb/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.22.1'
-__version_tuple__ = version_tuple = (0, 22, 1)
+__version__ = version = '0.23.0'
+__version_tuple__ = version_tuple = (0, 23, 0)

pycwb-0.23.0/pycwb/cli/batch_runner.py

@@ -0,0 +1,48 @@
+import os
+
+
+def init_parser(parser):
+    # Add the arguments
+    parser.add_argument('user_parameter_file',
+                        metavar='file_path',
+                        type=str,
+                        help='the path to the user parameter file')
+
+    # working dir
+    parser.add_argument('--work-dir',
+                        '-d',
+                        metavar='work_dir',
+                        type=str,
+                        default='.',
+                        help='the working directory')
+
+    # threads
+    parser.add_argument('--n-proc',
+                        '-n',
+                        metavar='n_proc',
+                        type=int,
+                        default=1,
+                        help='the number of cpu to use, default to 1. If it set to 0, '
+                             'it will use the value from the user parameter file.')
+
+    # jobs
+    parser.add_argument('--jobs',
+                        '-j',
+                        metavar='jobs',
+                        type=str,
+                        default=None,
+                        help='the range of jobs to run, e.g., 0-9')
+
+    # compress json
+    parser.add_argument('--compress_json',
+                        action='store_true',
+                        default=True,
+                        help='compress the json files, by default True')
+
+
+def command(args):
+    from pycwb.workflow.batch import batch_run
+
+    # Run the search function with the specified user parameter file
+    batch_run(args.user_parameter_file, working_dir=args.work_dir,
+              jobs=args.jobs, compress_json=args.compress_json, n_proc=args.n_proc)

pycwb-0.22.1/pycwb/cli/batch.py → pycwb-0.23.0/pycwb/cli/batch_setup.py

@@ -8,13 +8,20 @@ def init_parser(parser):
                         type=str,
                         help='the path to the user parameter file')
 
-    parser.add_argument('--submit',
-                        '-s',
+    parser.add_argument('--cluster',
+                        '-c',
                         metavar='job_submission_system',
                         type=str,
                         choices=['condor', 'slurm'],
+                        default='condor',
                         help='the submit option, the available options are condor and slurm')
 
+    parser.add_argument('--submit',
+                        '-s',
+                        action='store_true',
+                        default=False,
+                        help='submit the jobs to the job submission system')
+
     # working dir
     parser.add_argument('--work-dir',
                         '-d',
@@ -38,12 +45,18 @@ def init_parser(parser):
                         default=os.environ.get('CONDA_DEFAULT_ENV'),
                         help='the conda environment')
 
+    # additional init
+    parser.add_argument('--additional-init',
+                        '-a',
+                        metavar='additional_init',
+                        type=str,
+                        help='additional initialization commands')
+
     # accounting_group
     parser.add_argument('--accounting-group',
                         '-g',
                         metavar='accounting_group',
                         type=str,
-                        default='ligo.sim.o4.burst.allsky.cwboffline',
                         help='the condor accounting group')
 
     # threads
@@ -51,28 +64,23 @@ def init_parser(parser):
                         '-n',
                         metavar='n_proc',
                         type=int,
-                        default=0,
-                        help='the number of cpu to use, if it set to 0, '
+                        default=1,
+                        help='the number of cpu to use, default to 1. If it set to 0, '
                              'it will use the value from the user parameter file.')
 
-    # generate plot
-    parser.add_argument('--plot',
-                        action='store_true',
-                        default=False,
-                        help='generate the plot, by default False')
+    # job_per_worker
+    parser.add_argument('--job-per-worker',
+                        '-j',
+                        metavar='job_per_worker',
+                        type=int,
+                        default=5,
+                        help='the number of jobs per worker')
 
     # compress json
     parser.add_argument('--compress_json',
                         action='store_true',
-                        default=False,
-                        help='compress the json files, by default False')
-
-    # serve name
-    parser.add_argument('--name',
-                        metavar='name',
-                        type=str,
-                        default='pycwb',
-                        help='the name of the serve')
+                        default=True,
+                        help='compress the json files, by default True')
 
     # list number of jobs
     parser.add_argument('--list-n-jobs',
@@ -88,10 +96,10 @@ def init_parser(parser):
 
 
 def command(args):
-    from pycwb.workflow.batch import search
+    from pycwb.workflow.batch import batch_setup
 
     if args.list_n_jobs or args.list_jobs:
-        jobs = search(args.user_parameter_file, working_dir=args.work_dir, n_proc=1, dry_run=True)
+        jobs = batch_setup(args.user_parameter_file, working_dir=args.work_dir, n_proc=1, dry_run=True)
 
         print(f"Number of jobs: {len(jobs)}")
 
@@ -105,6 +113,18 @@ def command(args):
 
         return 0
 
+    if args.cluster is 'slurm':
+        print("Slurm is not supported yet.")
+        return 1
+
     # Run the search function with the specified user parameter file
-    search(args.user_parameter_file, working_dir=args.work_dir, n_proc=args.n_proc, submit=args.submit,
-           overwrite=args.force_overwrite, plot=args.plot, compress_json=args.compress_json)
+    batch_setup(args.user_parameter_file, working_dir=args.work_dir,
+                compress_json=args.compress_json,
+                cluster=args.cluster,
+                conda_env=args.conda_env,
+                additional_init=args.additional_init,
+                n_proc=args.n_proc,
+                accounting_group=args.accounting_group,
+                job_per_worker=args.job_per_worker,
+                submit=args.submit,
+                overwrite=args.force_overwrite, )

pycwb-0.23.0/pycwb/cli/merge_catalog.py

@@ -0,0 +1,18 @@
+import os
+
+
+def init_parser(parser):
+    # working dir
+    parser.add_argument('--work-dir',
+                        '-d',
+                        metavar='work_dir',
+                        type=str,
+                        default='.',
+                        help='the working directory')
+
+
+def command(args):
+    from pycwb.workflow.merge import merge_catalog
+
+    # Run the search function with the specified user parameter file
+    merge_catalog(working_dir=args.work_dir)

{pycwb-0.22.1 → pycwb-0.23.0}/pycwb/cli/run.py

@@ -35,7 +35,7 @@ def init_parser(parser):
     # generate plot
     parser.add_argument('--plot',
                         action='store_true',
-                        default=False,
+                        default=None,
                         help='generate the plot, by default False')
 
     # compress json
@@ -65,7 +65,7 @@ def init_parser(parser):
 
 
 def command(args):
-    from pycwb.workflow.batch import search
+    from pycwb.workflow.run import search
 
     if args.list_n_jobs or args.list_jobs:
         jobs = search(args.user_parameter_file, working_dir=args.work_dir, n_proc=1, dry_run=True)
@@ -83,5 +83,5 @@ def command(args):
         return 0
 
     # Run the search function with the specified user parameter file
-    search(args.user_parameter_file, working_dir=args.work_dir, n_proc=args.n_proc, submit=args.submit,
+    search(args.user_parameter_file, working_dir=args.work_dir, n_proc=args.n_proc,
            overwrite=args.force_overwrite, plot=args.plot, compress_json=args.compress_json)

{pycwb-0.22.1 → pycwb-0.23.0}/pycwb/cli/xtalk.py

@@ -14,6 +14,7 @@ def init_parser(parser):
                         '-o',
                         metavar='output_dir',
                         type=str,
+                        default='.',
                         help='the output directory')
 
 

{pycwb-0.22.1 → pycwb-0.23.0}/pycwb/config/config.py

@@ -5,7 +5,9 @@ pycwb.constants.user_parameters_schema.
 """
 import os.path
 import logging
+import pathlib
 
+from ..modules.xtalk.xtalk_data import check_and_download_xtalk_data
 from ..types.wdm_xtalk import WDMXTalkCatalog
 from ..types.data_quality_file import DQFile
 from ..utils.network import max_delay
@@ -67,9 +69,8 @@ class Config:
             setattr(self, key, params[key])
 
         self.add_derived_key()
-        self.check_file(self.MRAcatalog)
-        if self.MRAcatalog:
-            self.check_MRA_catalog()
+        self.check_xtalk_file(self.MRAcatalog)
+        self.check_MRA_catalog()
         self.check_lagStep()
 
     def add_derived_key(self):
@@ -90,7 +91,10 @@ class Config:
 
         # load WAT filter directory and set MRAcatalog
         if not self.filter_dir:
-            self.filter_dir = os.environ['HOME_WAT_FILTERS']
+            if os.environ.get('HOME_WAT_FILTERS') is None:
+                self.filter_dir = os.path.abspath(".")
+            else:
+                self.filter_dir = os.environ['HOME_WAT_FILTERS']
 
         self.MRAcatalog = f"{self.filter_dir}/{self.wdmXTalk}"
 
@@ -142,6 +146,33 @@ class Config:
         if not os.path.isfile(file_name):
             raise FileNotFoundError(f"File {file_name} does not exist")
 
+    @staticmethod
+    def check_xtalk_file(file_name: str) -> bool:
+        """
+        Helper function to check if xtalk file exists and download it if it does not exist
+
+        Parameters
+        ----------
+        file_name : str
+            Path to the xtalk file
+
+        Returns
+        -------
+        bool
+            True if the file exists or has been downloaded successfully, False otherwise
+
+        Raises
+        ------
+        FileNotFoundError
+            If the file does not exist and cannot be downloaded
+        """
+        if not os.path.isfile(file_name):
+            if check_and_download_xtalk_data(str(pathlib.Path(file_name).name), str(pathlib.Path(file_name).parent)):
+                return True
+            else:
+                raise FileNotFoundError(f"File {file_name} does not exist")
+        return True
+
     def check_lagStep(self):
         """
         Check if lagStep compatible with WDM parity
@@ -190,3 +221,6 @@ class Config:
             logger.info(f"MRA catalog has tag {wdm_MRA.tag}, updating beta order and precision from MRA catalog")
             self.WDM_beta_order, self.WDM_precision = int(wdm_MRA.beta_order), int(wdm_MRA.precision)
 
+    @staticmethod
+    def get_precision(cluster_size_threshold, healpix_order):
+        return cluster_size_threshold+65536*healpix_order

pycwb-0.23.0/pycwb/constants/project_constants.py

@@ -0,0 +1,3 @@
+ROUNDED_DIGITS = 50
+XTALK_FILE_LIST_URL = "https://raw.githubusercontent.com/PycWB/xtalk-data/main/file_list.txt"
+XTALK_DATA_URL = "https://github.com/PycWB/xtalk-data/raw/main"

{pycwb-0.22.1 → pycwb-0.23.0}/pycwb/constants/user_parameters_schema.py

@@ -51,6 +51,18 @@ schema = {
             "default": False,
             "cwb": False
         },
+        "save_injection": {
+            "type": "boolean",
+            "description": "save injected waveform",
+            "default": False,
+            "cwb": False
+        },
+        "plot_injection": {
+            "type": "boolean",
+            "description": "plot injected waveform",
+            "default": False,
+            "cwb": False
+        },
         "save_sky_map": {
             "type": "boolean",
             "description": "save sky maps to json",
@@ -69,6 +81,12 @@ schema = {
             "default": False,
             "cwb": False
         },
+        "use_root_supercluster": {
+            "type": "boolean",
+            "description": "use cwb supercluster, if false use python version",
+            "default": True,
+            "cwb": False
+        },
         "nproc": {
             "type": "integer",
             "description": "number of processes",
@@ -301,9 +319,8 @@ schema = {
         },
         "netRHO": {
             "type": "number",
-            "description": "[>4.0] coherent network SNR (supercluster, likelihood)",
+            "description": "[>4.0] coherent network SNR (supercluster, likelihood), if <=0 then use the new statistics for XGBoost",
             "default": 4.0,
-            "minimum": 4.0,
         },
         "netCC": {
             "type": "number",

{pycwb-0.22.1 → pycwb-0.23.0}/pycwb/modules/coherence/coherence.py

@@ -123,6 +123,9 @@ def coherence_single_res(i, config, tf_maps, nRMS_list, up_n=None, net=None):
 
     # produce TF maps with max over the sky energy
     alp = 0.0
+
+    # FIXME: max time delay is different to pycbc
+    config.max_delay = net.get_max_delay()
     for n in range(len(config.ifo)):
         ts = convert_to_wavearray(tf_maps[n])
         ts.Edge = config.segEdge

pycwb-0.23.0/pycwb/modules/energy_threshold/threshold.py

@@ -0,0 +1,48 @@
+import math
+import numba
+
+
+@numba.jit(nopython=True)
+def threshold(tf_maps, nIFO, Edge, p, shape):
+    """
+    Calculate WaveBurst energy threshold for a given black pixel probability p
+    and single detector Gamma distribution shape. TF data should contain pixel energy.
+    """
+
+    # The WSeries, wavearray and iGamma should be replaced with equivalent
+    # Python implementation or library
+
+    N = nIFO
+    pw = tf_maps[0]
+    M = pw.maxLayer() + 1
+    nL = int(Edge * pw.w_rate * M)
+    nR = len(pw.data) - nL - 1
+    w = pw.copy()
+
+    for i in range(1, N):
+        w += tf_maps[i]
+
+    amp, avr, bbb, alp = 0, 0, 0, 0
+    nn = 0
+
+    for i in range(nL, nR):
+        amp = w.data[i]
+        if amp > N * 100:
+            amp = N * 100
+        if amp > 0.001:
+            avr += amp
+            bbb += math.log(amp)
+            nn += 1
+
+    avr /= nn
+    alp = math.log(avr) - bbb / nn
+    alp = (3 - alp + math.sqrt((alp - 3) * (alp - 3) + 24 * alp)) / (12 * alp)
+
+    bbb = p * alp / shape
+
+    # Assuming iGamma is defined elsewhere or imported from some library/module
+    return avr * iGamma(alp, bbb) / alp / 2
+
+# Call the function
+# result = threshold(tf_maps, nIFO, Edge, p, shape)
+# print(result)

{pycwb-0.22.1 → pycwb-0.23.0}/pycwb/modules/job_segment/job_segment.py

@@ -12,20 +12,20 @@ logger = logging.getLogger(__name__)
 
 
 def create_job_segment_from_config(config):
+    logger.info("-" * 80)
+    logger.info("Initializing job segments")
+
     if not config.simulation:
-        logger.info("-" * 80)
-        logger.info("Initializing job segments")
         job_segments = select_job_segment(config.dq_files, config.ifo, config.frFiles,
                                           config.segLen, config.segMLS, config.segEdge, config.segOverlap,
                                           config.rateANA, config.l_high)
-
-        # log number of segments
-        logger.info(f"Number of segments: {len(job_segments)}")
-        logger.info("-" * 80)
     else:
         job_segments = create_job_segment_from_injection(config.ifo, config.simulation, config.injection)
-        for job_seg in job_segments:
-            logger.info(job_seg)
+        # for job_seg in job_segments:
+        #     logger.info(job_seg)
+        # log number of segments
+    logger.info(f"Number of segments: {len(job_segments)}")
+    logger.info("-" * 80)
     return job_segments
 
 

{pycwb-0.22.1 → pycwb-0.23.0}/pycwb/modules/likelihood/likelihood.py

@@ -4,13 +4,13 @@ import logging
 from pycwb.config import Config
 from pycwb.modules.cwb_conversions import convert_fragment_clusters_to_netcluster, \
     convert_netcluster_to_fragment_clusters
-from pycwb.types.network_cluster import FragmentCluster
+from pycwb.types.network_cluster import FragmentCluster, Cluster
 from pycwb.types.network_event import Event
 
 logger = logging.getLogger(__name__)
 
 
-def likelihood(config, network, fragment_clusters):
+def likelihood(config, network, fragment_clusters) -> tuple[list[Event], list[Cluster], list[dict]]:
     """
     calculate likelihood
 
@@ -107,7 +107,8 @@ def _likelihood(config, network, lag, cluster_id, fragment_cluster):
     wdm_list = network.get_wdm_list()
     for wdm in wdm_list:
         wdm.setTDFilter(config.TDSize, config.upTDF)
-
+    # FIXME: precision setup in config
+    # network.net.precision = config.get_precision(100, 5)
     network.set_delay_index(config.TDRate)
 
     # sparse_table_list = sparse_table_from_fragment_clusters(config, tf_maps, [fragment_cluster])

{pycwb-0.22.1 → pycwb-0.23.0}/pycwb/modules/plot/cluster_statistics.py

@@ -5,13 +5,15 @@ import logging
 logger = logging.getLogger(__name__)
 
 
-def plot_statistics(cluster, key='likelihood', filename=None):
+def plot_statistics(cluster, key='likelihood', gps_shift=0, filename=None):
     """Plot the statistics of the event
 
     :param cluster: cluster
     :type cluster: Cluster
     :param key: key of the statistics, defaults to 'likelihood'
     :type key: str, optional
+    :param gps_shift: gps time shift, defaults to 0
+    :type gps_shift: int, optional
     :param filename: path to save the plot, defaults to None
     :type filename: str, optional
     """
@@ -20,7 +22,7 @@ def plot_statistics(cluster, key='likelihood', filename=None):
 
     merged_map, start, dt, df = cluster.get_sparse_map(key)
 
-    plt = Spectrogram(merged_map, t0=start, dt=dt, f0=0, df=df).plot()
+    plt = Spectrogram(merged_map, t0=start + gps_shift, dt=dt, f0=0, df=df).plot()
     plt.colorbar()
     # save to png
     if filename is not None:

{pycwb-0.22.1 → pycwb-0.23.0}/pycwb/modules/read_data/data_check.py

@@ -55,11 +55,15 @@ def check_and_resample(data, config, ifo_index):
     # DC correction
     if config.dcCal[ifo_index] > 0 and config.dcCal[ifo_index] != 1.0:
         data.data *= config.dcCal[config.ifo.indexof(ifo_index)]
+        logger.info(f"DC correction: {config.dcCal[ifo_index]}")
 
     # resampling
     if config.fResample > 0:
         logger.info(f"Resampling data from {data.sample_rate} to {config.fResample}")
-        data = data.resample(1.0 / config.fResample)
+        # data = data.resample(1.0 / config.fResample)
+        w = convert_to_wavearray(data)
+        w.Resample(config.fResample)
+        data = convert_wavearray_to_pycbc_timeseries(w)
 
     new_sample_rate = data.sample_rate / (1 << config.levelR)
     if new_sample_rate != config.inRate: