PycWB 0.19.0__tar.gz → 0.20.0__tar.gz

{PycWB-0.19.0 → PycWB-0.20.0}/.gitignore

@@ -22,4 +22,6 @@ messenger-web-interface/node_modules
 messenger-web-interface/dist
 
 cwb_wrapper/_version.py
-pycwb/_version.py
+pycwb/_version.py
+
+benchmark/*/*.pkl

{PycWB-0.19.0 → PycWB-0.20.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: PycWB
-Version: 0.19.0
+Version: 0.20.0
 Summary: A Python package for coherent gravitational wave burst analysis
 Home-page: https://git.ligo.org/yumeng.xu/pycwb
 Author-email: The PycWB team <yumeng.xu@ligo.org>

{PycWB-0.19.0 → PycWB-0.20.0}/PycWB.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: PycWB
-Version: 0.19.0
+Version: 0.20.0
 Summary: A Python package for coherent gravitational wave burst analysis
 Home-page: https://git.ligo.org/yumeng.xu/pycwb
 Author-email: The PycWB team <yumeng.xu@ligo.org>

{PycWB-0.19.0 → PycWB-0.20.0}/PycWB.egg-info/SOURCES.txt

@@ -11,6 +11,16 @@ PycWB.egg-info/SOURCES.txt
 PycWB.egg-info/dependency_links.txt
 PycWB.egg-info/requires.txt
 PycWB.egg-info/top_level.txt
+benchmark/README.md
+benchmark/likelihood/README.md
+benchmark/likelihood/generate_data_for_likelihood.py
+benchmark/likelihood/performance_test_dpf.py
+benchmark/likelihood/performance_test_opt_sky.py
+benchmark/likelihood/user_parameters_injection.yaml
+benchmark/supercluster/README.md
+benchmark/supercluster/generate_data.py
+benchmark/supercluster/user_parameters_injection.yaml
+bin/pycwb
 bin/pycwb_search
 bin/pycwb_show
 cwb-core/Biorthogonal.cc
@@ -179,6 +189,7 @@ prototypes/cwb_core/coherence.hh
 prototypes/cwb_core/likelihood.cc
 prototypes/cwb_core/likelihood.hh
 prototypes/dag/dag.py
+prototypes/dag/prefect_test.py
 prototypes/messenger/__init__.py
 prototypes/messenger/server.py
 prototypes/messenger/visualizition.py
@@ -218,6 +229,9 @@ prototypes/performance/taichi_test.py
 pycwb/__init__.py
 pycwb/_version.py
 pycwb/search.py
+pycwb/cli/__init__.py
+pycwb/cli/run.py
+pycwb/cli/server.py
 pycwb/config/__init__.py
 pycwb/config/config.py
 pycwb/constants/__init__.py
@@ -261,6 +275,12 @@ pycwb/modules/job_segment/super_lag.py
 pycwb/modules/likelihood/__init__.py
 pycwb/modules/likelihood/likelihood.py
 pycwb/modules/likelihood/module.yaml
+pycwb/modules/likelihoodWP/__init__.py
+pycwb/modules/likelihoodWP/dpf.py
+pycwb/modules/likelihoodWP/dpf_cython.pyx
+pycwb/modules/likelihoodWP/likelihood.py
+pycwb/modules/likelihoodWP/sky_stat.py
+pycwb/modules/likelihoodWP/utils.py
 pycwb/modules/logger/__init__.py
 pycwb/modules/logger/logger.py
 pycwb/modules/logger/module.yaml
@@ -281,6 +301,7 @@ pycwb/modules/plot_map/module.yaml
 pycwb/modules/plot_map/world_map.py
 pycwb/modules/read_data/__init__.py
 pycwb/modules/read_data/data_check.py
+pycwb/modules/read_data/data_find.py
 pycwb/modules/read_data/mdc.py
 pycwb/modules/read_data/module.yaml
 pycwb/modules/read_data/read_data.py
@@ -290,12 +311,19 @@ pycwb/modules/reconstruction/module.yaml
 pycwb/modules/sparse_series/__init__.py
 pycwb/modules/sparse_series/module.yaml
 pycwb/modules/sparse_series/sparse_table.py
+pycwb/modules/statistics/__init__.py
+pycwb/modules/statistics/eff.py
 pycwb/modules/super_cluster/__init__.py
 pycwb/modules/super_cluster/module.yaml
 pycwb/modules/super_cluster/supercluster.py
+pycwb/modules/superlag/__init__.py
+pycwb/modules/superlag/superlag.py
 pycwb/modules/web_viewer/__init__.py
 pycwb/modules/web_viewer/create.py
 pycwb/modules/web_viewer/module.yaml
+pycwb/modules/xtalk/__init__.py
+pycwb/modules/xtalk/monster.py
+pycwb/modules/xtalk/monster_old.py
 pycwb/types/__init__.py
 pycwb/types/data_quality_file.py
 pycwb/types/detector.py

PycWB-0.20.0/benchmark/README.md

@@ -0,0 +1,10 @@
+# Benchmark for the Python implementation of the algorithms
+
+
+## Supercluster Benchmark
+
+Please refer to the subdirectory `benchmark/supercluster` for more details.
+
+## Likelihood Benchmark
+
+Please refer to the subdirectory `benchmark/likelihood` for more details.

PycWB-0.20.0/benchmark/likelihood/README.md

@@ -0,0 +1,21 @@
+# Likelihood Benchmark
+
+This benchmark is designed to test the performance of the likelihood in Python
+
+## Prepare the test data for the script
+
+Run the script `generate_data_for_likelihood.py` to generate the cluster data for the likelihood benchmark.
+
+```bash
+python generate_data_for_likelihood.py
+```
+
+A `test_data.pkl` file will be generated for all the Python variables required for likelihood benchmark.
+
+## Run the benchmark
+
+Run the script `performance_test_opt_sky.py` to run the benchmark.
+
+```bash
+python performance_test_opt_sky.py
+```

PycWB-0.20.0/benchmark/likelihood/generate_data_for_likelihood.py

@@ -0,0 +1,105 @@
+import os
+
+from pycwb.config import Config
+from pycwb.modules.logger import logger_init
+
+if not os.environ.get('HOME_WAT_FILTERS'):
+    print('Please set HOME_WAT_FILTERS to the directory of WAT filters')
+    exit(1)
+
+logger_init()
+
+config = Config('./user_parameters_injection.yaml')
+
+#%% md
+## generate injected data for each detector with given parameters in config
+#%%
+from pycwb.modules.read_data import generate_injection
+from pycwb.modules.job_segment import create_job_segment_from_injection
+
+job_segments = create_job_segment_from_injection(config.ifo, config.simulation, config.injection)
+
+data = generate_injection(config, job_segments[0])
+
+#%% md
+## apply data conditioning to the data
+#%%
+from pycwb.modules.data_conditioning import data_conditioning
+
+strains, nRMS = data_conditioning(config, data)
+
+#%% md
+## calculate coherence
+#%%
+from pycwb.modules.coherence import coherence
+
+# calculate coherence
+fragment_clusters = coherence(config, strains, nRMS)
+
+#%% md
+## supercluster
+#%%
+from pycwb.modules.super_cluster import supercluster
+from pycwb.types.network import Network
+
+network = Network(config, strains, nRMS)
+
+pwc_list = supercluster(config, network, fragment_clusters, strains)
+
+#%% md
+## dump data
+
+#%%
+# only dump the first supercluster
+d = pwc_list[0]
+
+from pycwb.modules.cwb_conversions import convert_fragment_clusters_to_netcluster, convert_netcluster_to_fragment_clusters
+
+pwc = network.get_cluster(0)
+wdm_list = network.get_wdm_list()
+for wdm in wdm_list:
+    wdm.setTDFilter(config.TDSize, config.upTDF)
+
+# load delay index
+network.set_delay_index(config.TDRate)
+
+# load time delay data
+pwc.cpf(convert_fragment_clusters_to_netcluster(d.dump_cluster(0)), False)
+pwc.setcore(False, 1)
+pwc.loadTDampSSE(network.net, 'a', config.BATCH, config.BATCH)
+
+
+from pycwb.modules.likelihoodWP.likelihood import load_data_from_ifo
+import numpy as np
+
+acor = network.net.acor
+network_energy_threshold = 2 * acor * acor * config.nIFO
+gamma_regulator = network.net.gamma * network.net.gamma * 2 / 3
+delta_regulator = abs(network.net.delta) if abs(network.net.delta) < 1 else 1
+REG = [delta_regulator * np.sqrt(2), 0, 0]
+netEC_threshold = network.net.netRHO * network.net.netRHO * 2
+
+n_sky = network.net.index.size()
+
+ml, FP, FX = load_data_from_ifo(network, config.nIFO)
+
+cluster_test = convert_netcluster_to_fragment_clusters(pwc)
+pixels = cluster_test.clusters[0].pixels
+
+# save FP, FX, rms, n_sky, gamma_regulator, network_energy_threshold to pickle
+test_data = {
+    'FP': FP,
+    'FX': FX,
+    'pixels': pixels,
+    'n_ifo': config.nIFO,
+    'ml': ml,
+    'n_sky': n_sky,
+    'delta_regulator': delta_regulator,
+    'gamma_regulator': gamma_regulator,
+    'netEC_threshold': netEC_threshold,
+    'network_energy_threshold': network_energy_threshold,
+    'netCC': network.net.netCC,
+}
+import pickle
+with open('test_data.pkl', 'wb') as f:
+    pickle.dump(test_data, f)

PycWB-0.20.0/benchmark/likelihood/performance_test_dpf.py

@@ -0,0 +1,273 @@
+import pickle
+import numpy as np
+
+from pycwb.modules.likelihoodWP.likelihood import load_data_from_pixels
+
+# load FP, FX, rms, n_sky, gamma_regulator, network_energy_threshold from pickle
+with open('test_data.pkl', 'rb') as f:
+    test_data = pickle.load(f)
+
+FP = test_data['FP']
+FX = test_data['FX']
+pixels = test_data['pixels']
+n_ifo = test_data['n_ifo']
+n_sky = test_data['n_sky']
+gamma_regulator = test_data['gamma_regulator']
+network_energy_threshold = test_data['network_energy_threshold']
+rms, td00, td90, td_energy = load_data_from_pixels(pixels, n_ifo)
+
+from numba import njit, prange, vectorize, guvectorize, float32, uint32
+
+import numpy as np
+from math import sqrt
+
+
+@njit(cache=True)
+def dpf_np(Fp0, Fx0, rms):
+    """
+    :param Fp0: Fp0 is a NIFO x 1 array
+    :param Fx0: Fx0 is a NIFO x 1 array
+    :param rms: rms is a NIFO x NPIX array
+    :return:
+    """
+    # sign = np.sign(np.dot(Fp0, Fx0))
+
+    f = rms * Fp0
+    F = rms * Fx0
+
+    _ff = np.sum(f * f, axis=1)
+    _FF = np.sum(F * F, axis=1)
+    _fF = np.sum(f * F, axis=1)
+
+    _si = 2.0 * _fF
+    _co = _ff - _FF
+    _AP = _ff + _FF
+    _nn = np.sqrt(_co * _co + _si * _si)
+    _cc = _co / (_nn + 0.0001)
+    fp = (_AP + _nn) / 2.0
+
+    si = np.sqrt((1.0 - _cc) / 2.0)
+    co = np.where(_si > 0.0, np.sqrt((1.0 + _cc) / 2.0), -np.sqrt((1.0 + _cc) / 2.0))
+
+    f, F = f * co[:, np.newaxis] + F * si[:, np.newaxis], F * co[:, np.newaxis] - f * si[:, np.newaxis]
+    fF_new = np.sum(f * F, axis=1) / (fp + 0.0001)
+
+    F -= f * fF_new[:, np.newaxis]
+    fx = np.sum(F * F, axis=1)
+    ni = np.sum(f ** 4, axis=1) / ((fp * fp) + 0.0001)
+
+    NI = np.sum(fx / (ni + 0.0001))
+    NN = np.sum(fp > 0.0)
+
+    return sqrt(NI / (NN + 0.01)), fp, fx, si, co, ni
+
+
+@vectorize([float32(float32, float32)])
+def mul_vec(a, b):
+    return a * b
+
+
+@vectorize([float32(float32, float32)])
+def div_vec(a, b):
+    _o = float32(0.0001)
+    return a / (b + _o)
+
+
+@vectorize([float32(float32, float32)])
+def add_vec(a, b):
+    return a + b
+
+
+@vectorize([float32(float32, float32)])
+def sub_vec(a, b):
+    return a - b
+
+
+@vectorize([float32(float32, float32)])
+def norm_vec(a, b):
+    return sqrt(a * a + b * b)
+
+
+@vectorize([float32(float32, float32)])
+def avg_vec(a, b):
+    return (a + b) / float32(2.)
+
+
+@vectorize([float32(float32)])
+def sin_from_cc(a):
+    return sqrt((float32(1.) - a) / float32(2.))
+
+
+@vectorize([float32(float32, float32)])
+def cos_from_cc(a, si):
+    return sqrt((float32(1.) + a) / float32(2.)) if si > float32(0.) else - sqrt((float32(1.) + a) / float32(2.))
+
+
+@vectorize([uint32(float32)])
+def pos_sign_vec(a):
+    return uint32(1) if a > float32(0.) else uint32(0)
+
+
+@vectorize([float32(float32, float32, float32, float32)])
+def rotate_fp_vec(fp, fx, si, co):
+    return fp * co + fx * si
+
+@vectorize([float32(float32, float32, float32, float32)])
+def rotate_fx_vec(fp, fx, si, co):
+    return fx * co - fp * si
+
+@vectorize([float32(float32)])
+def quad_vec(a):
+    return a * a * a * a
+
+@guvectorize([(float32[:], float32[:], float32[:])], '(n),(n)->()')
+def sum_vec(a, b, res):
+    s = float32(0.)
+    for i in range(a.shape[0]):
+        s += a[i] * b[i]
+    res[0] = s
+
+
+@njit(cache=True)
+def dpf_np_loops_vec(Fp0, Fx0, rms):
+    NPIX, NIFO = rms.shape
+    NPIX = uint32(NPIX)
+    NIFO = uint32(NIFO)
+
+    # variables for return
+    f = np.empty((NPIX, NIFO), dtype=np.float32)
+    F = np.empty((NPIX, NIFO), dtype=np.float32)
+    si = np.empty(NPIX, dtype=np.float32)
+    co = np.empty(NPIX, dtype=np.float32)
+    fp = np.empty(NPIX, dtype=np.float32)
+    fx = np.zeros(NPIX, dtype=np.float32)
+    ni = np.zeros(NPIX, dtype=np.float32)
+
+    _o = float32(0.0001)
+
+    # Prepare constants
+    # NI = np.float32(0.0)
+    # NN = np.uint32(0)
+
+    # Compute f and F
+    for j in range(NIFO):
+        for i in range(NPIX):
+            f[i, j] = mul_vec(rms[i, j], Fp0[j])
+            F[i, j] = mul_vec(rms[i, j], Fx0[j])
+
+    # Compute ff, FF, and fF
+    for i in range(NPIX):
+        _ff = float32(0.)
+        _FF = float32(0.)
+        _fF = float32(0.)
+
+        for j in range(NIFO):
+            _ff += f[i, j] * f[i, j]
+            _FF += F[i, j] * F[i, j]
+            _fF += F[i, j] * f[i, j]
+
+        # Compute si, co, AP, nn, fp, and cc
+        _si = mul_vec(float32(2.), _fF)  # 2. * _fF
+        _co = sub_vec(_ff, _FF)  # _ff - _FF
+        _AP = add_vec(_ff, _FF)  # _ff + _FF
+        _nn = norm_vec(_co, _si)  # np.sqrt(_co * _co + _si * _si)
+        _cc = div_vec(_co, _nn)  # _co / (_nn + 0.0001)
+        fp[i] = avg_vec(_AP, _nn)  # (_AP + _nn) / 2.
+        si[i] = sin_from_cc(_cc)   # sqrt((1. - _cc) / 2.)
+        co[i] = cos_from_cc(_cc,_si) # (sqrt((1. + _cc) / 2.) if _si > 0.0 else - sqrt((1. + _cc) / 2.))
+
+    # Compute f_new, F_new, fF_new, F_new, fx, ni
+    for i in range(NPIX):
+        for j in range(NIFO):
+            f[i, j], F[i, j] = f[i, j] * co[i] + F[i, j] * si[i], F[i, j] * co[i] - f[i, j] * si[i]
+            # f[i, j] = rotate_fp_vec(f[i, j], F[i, j], si[i], co[i])
+            # F[i, j] = rotate_fx_vec(f[i, j], F[i, j], si[i], co[i])
+
+        fF_new = float32(0.)
+        for j in range(NIFO):
+            fF_new += f[i, j] * F[i, j]
+        # fF_new /= (fp[i] + _o)
+        fF_new = div_vec(fF_new, fp[i])
+
+        for j in range(NIFO):
+            F[i, j] -= f[i, j] * fF_new
+            fx[i] += F[i, j] * F[i, j]
+            ni[i] += f[i, j] ** 4
+            # ni[i] += quad_vec(f[i, j])
+
+    NI, NN = float32(0.0), uint32(0)
+
+    # Compute NI and NN
+    for i in range(NPIX):
+        # ni[i] /= (fp[i] * fp[i] + _o)
+        ni[i] = div_vec(ni[i], mul_vec(fp[i], fp[i]))
+        # NI += fx[i] / (ni[i] + _o)
+        NI += div_vec(fx[i], ni[i])
+        # if fp[i] > float32(0.0):
+        NN += pos_sign_vec(fp[i])
+        # NN += 1 if fp[i] > 0.0 else 0
+
+    return sqrt(NI / (NN + 0.01)), fp, fx, si, co, ni
+
+
+@njit(cache=True)
+def calculate_dpf(FP, FX, rms, n_sky: int, gamma_regulator: float, network_energy_threshold: float):
+    FP = FP.astype(np.float32)
+    FX = FX.astype(np.float32)
+    rms = rms.astype(np.float32)
+
+    MM = np.zeros(n_sky, dtype=np.uint8)
+    mm = np.zeros(n_sky, dtype=np.uint8)
+
+    for i in range(n_sky):
+        # todo:          if(!mm[l]) continue;           // skip delay configurations
+        # if(bBB && !BB[l]) continue;                    // skip delay configurations : big clusters
+        MM[i] = 1
+        # FF += 1
+        aa, fp, fx, si, co, ni = dpf_np_loops_vec(FP[i], FX[i], rms)
+
+        mm[i] = 1 if aa > gamma_regulator else 0
+    FF = MM.sum()
+    ff = mm.sum()
+
+    return (FF ** 2 / (ff ** 2 + 1.e-9) - 1) * network_energy_threshold
+
+
+calculate_dpf(FP.T, FX.T, rms.T,
+              int(n_sky),
+              gamma_regulator, network_energy_threshold)
+
+print(dpf_np_loops_vec(FP.T[1000].astype(np.float32), FX.T[1000].astype(np.float32), rms.T.astype(np.float32))[0])
+
+import time
+
+total_time = 0
+# # convert FP, FX, rms to float32
+for i in range(10):
+    start = time.time()
+    calculate_dpf(FP.T, FX.T, rms.T,
+                  int(n_sky),
+                  gamma_regulator, network_energy_threshold)
+    end = time.time()
+    total_time += end - start
+    print(end - start)
+print(f"Average time: {total_time / 10:.4f} s")
+
+from subprocess import Popen
+from contextlib import contextmanager
+from os import getpid
+from time import sleep
+from signal import SIGINT
+
+# @contextmanager
+# def perf_stat():
+#     p = Popen(["perf", "stat", "-p", str(getpid())])
+#     sleep(0.5)
+#     yield
+#     p.send_signal(SIGINT)
+#
+#
+# with perf_stat():
+#     calculate_dpf(FP.T, FX.T, rms.T,
+#                   int(n_sky),
+#                   gamma_regulator, network_energy_threshold)

PycWB-0.20.0/benchmark/likelihood/performance_test_opt_sky.py

@@ -0,0 +1,110 @@
+# add .. to path[0]
+import sys, os
+sys.path.insert(0, '..')
+
+import pickle
+from math import sqrt
+import numpy as np
+from pycwb.modules.likelihoodWP.likelihood import calculate_dpf, find_optimal_sky_localization, calculate_sky_statistics
+from pycwb.modules.likelihoodWP.likelihood import load_data_from_pixels
+from pycwb.modules.xtalk.monster import load_catalog, getXTalk_pixels
+import time
+
+#################################################################################
+# load FP, FX, rms, n_sky, gamma_regulator, network_energy_threshold from pickle
+#################################################################################
+with open('test_data.pkl', 'rb') as f:
+    test_data = pickle.load(f)
+
+FP = test_data['FP']
+FX = test_data['FX']
+ml = test_data['ml']
+n_sky = test_data['n_sky']
+pixels = test_data['pixels']
+gamma_regulator = test_data['gamma_regulator']
+delta_regulator = test_data['delta_regulator']
+network_energy_threshold = test_data['network_energy_threshold']
+netCC = test_data['netCC']
+n_ifo = test_data['n_ifo']
+REG = np.array([delta_regulator * sqrt(2), 0., 0.])
+
+rms, td00, td90, td_energy = load_data_from_pixels(pixels, n_ifo)
+
+n_pix = rms.shape[1]
+#####################
+# Load xtalk catalog
+#####################
+if not os.environ.get('HOME_WAT_FILTERS'):
+    print('Please set HOME_WAT_FILTERS to the directory of WAT filters')
+    exit(1)
+
+fn = f"{os.environ.get('HOME_WAT_FILTERS')}/wdmXTalk/OverlapCatalog16-1024.bin"
+
+start_time = time.time()
+xtalk_coeff, xtalk_lookup_table, layers, nRes = load_catalog(fn)
+print(f"Time for load_catalog: {time.time() - start_time} s")
+
+#######################
+# Get xtalk for pixels
+#######################
+cluster_xtalk_lookup, cluster_xtalk = getXTalk_pixels(pixels, True, layers, xtalk_coeff, xtalk_lookup_table)
+
+#############################################
+# Convert FP, FX, rms, td00, td90 to float32
+#############################################
+td00 = np.transpose(td00.astype(np.float32), (2, 0, 1))  # (ndelay, nifo, npix)
+td90 = np.transpose(td90.astype(np.float32), (2, 0, 1))  # (ndelay, nifo, npix)
+FP = FP.T.astype(np.float32)
+FX = FX.T.astype(np.float32)
+rms = rms.T.astype(np.float32)
+
+#############################################
+# Calculate DPF and the optimal sky location
+#############################################
+REG[1] = calculate_dpf(FP, FX, rms, n_sky, n_ifo, gamma_regulator, network_energy_threshold)
+
+l_max = find_optimal_sky_localization(n_ifo, n_pix, n_sky, FP, FX, rms, td00, td90, ml, REG, netCC, delta_regulator,
+                                      network_energy_threshold)
+print(f"l_max: {l_max}")
+
+#############################################
+# Calculate sky statistics
+#############################################
+calculate_sky_statistics(l_max, n_ifo, n_pix, FP, FX, rms, td00, td90, ml, REG, network_energy_threshold, cluster_xtalk,
+                         cluster_xtalk_lookup)
+
+#############################################
+# Performance test
+#############################################
+
+total_time = 0
+# # convert FP, FX, rms to float32
+for i in range(10):
+    start = time.time()
+    l_max = find_optimal_sky_localization(n_ifo, n_pix, n_sky, FP, FX, rms, td00, td90, ml, REG, netCC, delta_regulator,
+                                          network_energy_threshold)
+    end = time.time()
+    total_time += end - start
+    # print(end - start)
+print(f"Average time for find_optimal_sky_localization: {total_time / 10} s")
+
+total_time = 0
+for i in range(10):
+    start = time.time()
+    calculate_sky_statistics(l_max, n_ifo, n_pix, FP, FX, rms, td00, td90, ml, REG, network_energy_threshold,
+                             cluster_xtalk, cluster_xtalk_lookup)
+    end = time.time()
+    total_time += end - start
+    # print(end - start)
+print(f"Average time for calculate_sky_statistics: {total_time / 10} s")
+
+total_time = 0
+
+for i in range(10):
+    start = time.time()
+    cluster_xtalk_lookup, cluster_xtalk = getXTalk_pixels(pixels, True, layers, xtalk_coeff, xtalk_lookup_table)
+    end = time.time()
+    # print(end - start)
+    total_time += end - start
+
+print(f"Average time for getXTalk_pixels: {total_time / 10} s")