PycWB 0.23.2__tar.gz → 0.25.0__tar.gz

{pycwb-0.23.2 → pycwb-0.25.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: PycWB
-Version: 0.23.2
+Version: 0.25.0
 Summary: A Python package for coherent gravitational wave burst analysis
 Home-page: https://git.ligo.org/yumeng.xu/pycwb
 Author-email: The PycWB team <yumeng.xu@ligo.org>
@@ -31,6 +31,7 @@ Requires-Dist: prefect-dask
 Requires-Dist: dask
 Requires-Dist: dask_jobqueue
 Requires-Dist: htcondor
+Requires-Dist: psutil
 
 # PycWB
 

{pycwb-0.23.2 → pycwb-0.25.0}/PycWB.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: PycWB
-Version: 0.23.2
+Version: 0.25.0
 Summary: A Python package for coherent gravitational wave burst analysis
 Home-page: https://git.ligo.org/yumeng.xu/pycwb
 Author-email: The PycWB team <yumeng.xu@ligo.org>
@@ -31,6 +31,7 @@ Requires-Dist: prefect-dask
 Requires-Dist: dask
 Requires-Dist: dask_jobqueue
 Requires-Dist: htcondor
+Requires-Dist: psutil
 
 # PycWB
 

{pycwb-0.23.2 → pycwb-0.25.0}/PycWB.egg-info/SOURCES.txt

@@ -274,6 +274,10 @@ pycwb/modules/data_conditioning/regression.py
 pycwb/modules/data_conditioning/whitening.py
 pycwb/modules/energy_threshold/__init__.py
 pycwb/modules/energy_threshold/threshold.py
+pycwb/modules/gracedb/__init__.py
+pycwb/modules/gracedb/gracedb.py
+pycwb/modules/gracedb/module.yaml
+pycwb/modules/gracedb/test_gracedb.py
 pycwb/modules/job_segment/__init__.py
 pycwb/modules/job_segment/dq_segment.py
 pycwb/modules/job_segment/frame.py
@@ -281,6 +285,7 @@ pycwb/modules/job_segment/job_segment.py
 pycwb/modules/job_segment/module.yaml
 pycwb/modules/job_segment/plots.py
 pycwb/modules/job_segment/super_lag.py
+pycwb/modules/job_segment/test_job_segment.py
 pycwb/modules/likelihood/__init__.py
 pycwb/modules/likelihood/likelihood.py
 pycwb/modules/likelihood/module.yaml
@@ -334,6 +339,7 @@ pycwb/modules/super_cluster/utils.py
 pycwb/modules/supercluster_naive/__init__.py
 pycwb/modules/supercluster_naive/supercluster2.py
 pycwb/modules/superlag/__init__.py
+pycwb/modules/superlag/module.yaml
 pycwb/modules/superlag/superlag.py
 pycwb/modules/web_viewer/__init__.py
 pycwb/modules/web_viewer/create.py
@@ -365,6 +371,7 @@ pycwb/utils/async_write.py
 pycwb/utils/check_ROOT.py
 pycwb/utils/dataclass_object_io.py
 pycwb/utils/dep_check.py
+pycwb/utils/generate_params_table.py
 pycwb/utils/image.py
 pycwb/utils/module.py
 pycwb/utils/network.py
@@ -388,5 +395,4 @@ pycwb/workflow/run.py
 pycwb/workflow/subflow/__init__.py
 pycwb/workflow/subflow/postprocess_and_plots.py
 pycwb/workflow/subflow/prepare_job_runs.py
-pycwb/workflow/subflow/process_job_segment.py
-pycwb/workflow/subflow/supercluster_and_likelihood.py
+pycwb/workflow/subflow/process_job_segment.py

{pycwb-0.23.2 → pycwb-0.25.0}/PycWB.egg-info/requires.txt

@@ -21,3 +21,4 @@ prefect-dask
 dask
 dask_jobqueue
 htcondor
+psutil

{pycwb-0.23.2 → pycwb-0.25.0}/docs/source/conf.py

@@ -27,7 +27,7 @@ exclude_patterns = ["pycwb/vendor/*", ".DS_Store"]
 # -- Options for HTML output -------------------------------------------------
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output
 
-html_theme = 'sphinx_rtd_theme'
+html_theme = "furo"
 html_static_path = ['_static']
 
 # -- external links ----------------------------------------------------------

pycwb-0.25.0/docs/source/schema.rst

@@ -0,0 +1,15 @@
+.. _schema:
+
+User Parameters
+----------------
+
+The default schema for user parameters is:
+
+.. exec::
+    import json
+    from pycwb.constants import user_parameters_schema
+    from pycwb.utils.generate_params_table import generate_rst_table, parse_description, parse_type_or_enum
+
+    print(generate_rst_table(user_parameters_schema["properties"]))
+    # json_obj = json.dumps(user_parameters_schema, sort_keys=True, indent=4)
+    # print('.. code-block:: JavaScript\n\n    %s\n\n' % json_obj)

{pycwb-0.23.2 → pycwb-0.25.0}/envs/environment.yml

@@ -12,4 +12,5 @@ dependencies:
   - python=3.10
   - root=6.26.10
   - setuptools_scm
-  - sphinx_rtd_theme
+  - sphinx_rtd_theme
+  - furo

{pycwb-0.23.2 → pycwb-0.25.0}/pycwb/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.23.2'
-__version_tuple__ = version_tuple = (0, 23, 2)
+__version__ = version = '0.25.0'
+__version_tuple__ = version_tuple = (0, 25, 0)

{pycwb-0.23.2 → pycwb-0.25.0}/pycwb/cli/batch_setup.py

@@ -73,9 +73,25 @@ def init_parser(parser):
                         '-j',
                         metavar='job_per_worker',
                         type=int,
-                        default=5,
+                        default=1,
                         help='the number of jobs per worker')
 
+    # memory
+    parser.add_argument('--memory',
+                        '-m',
+                        metavar='memory',
+                        type=str,
+                        default='6GB',
+                        help='the memory for each job')
+
+    # disk
+    parser.add_argument('--disk',
+                        '-k',
+                        metavar='disk',
+                        type=str,
+                        default='4GB',
+                        help='the disk space for each job')
+
     # compress json
     parser.add_argument('--compress_json',
                         action='store_true',
@@ -124,6 +140,8 @@ def command(args):
                 conda_env=args.conda_env,
                 additional_init=args.additional_init,
                 n_proc=args.n_proc,
+                memory=args.memory,
+                disk=args.disk,
                 accounting_group=args.accounting_group,
                 job_per_worker=args.job_per_worker,
                 submit=args.submit,

{pycwb-0.23.2 → pycwb-0.25.0}/pycwb/constants/user_parameters_schema.py

@@ -99,6 +99,46 @@ schema = {
             "default": {},
             "cwb": False
         },
+        "gps_start": {
+            "type": "number",
+            "description": "start gps time, used together with gps_end for generating job segments",
+            "cwb": False
+        },
+        "gps_end": {
+            "type": "number",
+            "description": "end gps time, used together with gps_start for generating job segments",
+            "cwb": False
+        },
+        "gps_center": {
+            "type": "number",
+            "description": "center gps time, used together with time_left and time_right for generating job segments",
+            "cwb": False
+        },
+        "time_left": {
+            "type": "number",
+            "description": "left time offset from gps_center, used together with gps_center and time_right for generating job segments",
+            "cwb": False
+        },
+        "time_right": {
+            "type": "number",
+            "description": "right time offset from gps_center, used together with gps_center and time_left for generating job segments",
+            "cwb": False
+        },
+        "superevent": {
+            "type": "string",
+            "description": "superevent name, used together with time_left and time_right for generating job segments",
+            "cwb": False
+        },
+        "gwdatafind": {
+            "type": "object",
+            "description": "If the gwdatafind is set, the framefiles can be pulled from the LIGO data server, "
+                           "the support keys are: site, frametype, host. Example: ",
+                           "     site: ['L','H']"
+                           "     frametype: ['L1_HOFT_C00','H1_HOFT_C00']"
+                           "     host: 'datafind.ldas.cit:80'"
+            "default": {},
+            "cwb": False
+        },
         "WDM_beta_order": {
             "type": "integer",
             "description": "WDM default parameters: beta function order for Meyer",

{pycwb-0.23.2 → pycwb-0.25.0}/pycwb/modules/catalog/catalog.py

@@ -1,7 +1,7 @@
 import os
 import logging
 from filelock import SoftFileLock
-import json
+import orjson
 import pycwb
 from pycwb.config import Config
 from pycwb.types.job import WaveSegment
@@ -32,13 +32,13 @@ def create_catalog(filename: str, config: Config, jobs: list[WaveSegment]) -> No
     output = {
         "config": config.__dict__,
         "version": pycwb.__version__,
-        "jobs": [job.to_dict() for job in jobs],
+        "jobs": jobs,
         "events": []
     }
 
     with SoftFileLock(filename + ".lock", timeout=10):
-        with open(filename, 'w') as f:
-            json.dump(output, f, default=vars)
+        with open(filename, 'wb') as f:
+            f.write(orjson.dumps(output, option=orjson.OPT_SERIALIZE_NUMPY))
 
 
 def add_events_to_catalog(filename: str, events: list[Event]) -> None:
@@ -64,13 +64,13 @@ def add_events_to_catalog(filename: str, events: list[Event]) -> None:
     if os.path.exists(filename):
         with SoftFileLock(filename + ".lock", timeout=10):
             # read the json file
-            with open(filename, 'r+') as f:
-                catalog = json.load(f)
+            with open(filename, 'rb+') as f:
+                catalog = orjson.loads(f.read())
                 # append events
                 catalog["events"].extend(events)
                 # write the json file
                 f.seek(0)
-                json.dump(catalog, f)
+                f.write(orjson.dumps(catalog, option=orjson.OPT_SERIALIZE_NUMPY))
     else:
         logger.warning("Catalog file does not exist. Event will not be saved to catalog.")
 

{pycwb-0.23.2 → pycwb-0.25.0}/pycwb/modules/data_conditioning/whitening.py

@@ -24,6 +24,7 @@ def whitening(config, h):
     layers_white = 2 ** config.l_white if config.l_white > 0 else 2 ** config.l_high
     wdm_white = WDM(layers_white, layers_white, config.WDM_beta_order, config.WDM_precision)
 
+    # TODO: check the length of data and white parameters to prevent freezing
     # check if whitening WDM filter lenght is less than cwb scratch
     wdmFlen = wdm_white.m_H / config.rateANA
     if wdmFlen > config.segEdge + 0.001:

pycwb-0.25.0/pycwb/modules/gracedb/__init__.py

@@ -0,0 +1 @@
+from .gracedb import *

pycwb-0.25.0/pycwb/modules/gracedb/gracedb.py

@@ -0,0 +1,40 @@
+from ligo.gracedb.rest import GraceDb
+from ligo.gracedb.exceptions import HTTPError
+
+
+def get_superevent(event_name):
+    """
+    Get the super event for a given event name.
+
+    :param event_name: The name of the event.
+    :type event_name: str
+    :return: The super event.
+    :rtype: dict
+    """
+    client = GraceDb()
+    try:
+        response = client.superevent(event_name).json()
+    except HTTPError as e:
+        if e.response.status_code == 404:
+            print('No super event found for event {}'.format(event_name))
+            return None
+        else:
+            raise
+
+    return response
+
+
+def get_superevent_t0(event_name):
+    """
+    Get the GPS time of the super event for a given event name.
+
+    :param event_name: The name of the event.
+    :type event_name: str
+    :return: The GPS time of the super event.
+    :rtype: float
+    """
+    super_event = get_superevent(event_name)
+    if super_event is None:
+        return None
+
+    return super_event['t_0']

pycwb-0.25.0/pycwb/modules/gracedb/module.yaml

@@ -0,0 +1,4 @@
+name: gracedb
+author: pycWB
+description: A interface for the LIGO/Virgo GraceDB database
+dependencies: ["ligo-proxy-utils", "ligo-gracedb"]

pycwb-0.25.0/pycwb/modules/gracedb/test_gracedb.py

@@ -0,0 +1,27 @@
+import unittest
+from .gracedb import get_superevent_t0, get_superevent
+
+class TestGraceDB(unittest.TestCase):
+
+    def test_get_superevent_t0(self):
+        event_name = 'S241011k'
+        t0 = get_superevent_t0(event_name)
+        self.assertAlmostEqual(t0, 1412725132.96, places=1)
+
+    def test_get_superevent_t0_no_event(self):
+        event_name = 'S000000a'
+        t0 = get_superevent_t0(event_name)
+        self.assertIsNone(t0)
+
+    def test_get_superevent(self):
+        event_name = 'S241011k'
+        super_event = get_superevent(event_name)
+        self.assertEqual(super_event['superevent_id'], event_name)
+
+    def test_get_superevent_no_event(self):
+        event_name = 'S000000a'
+        super_event = get_superevent(event_name)
+        self.assertIsNone(super_event)
+
+if __name__ == '__main__':
+    unittest.main()

{pycwb-0.23.2 → pycwb-0.25.0}/pycwb/modules/job_segment/dq_segment.py

@@ -6,7 +6,7 @@ from pycwb.types.job import WaveSegment
 logger = logging.getLogger(__name__)
 
 
-def read_seg_list(dq_file_list, dq_cat):
+def read_seg_list(dq_file_list=None, dq_cat='CWB_CAT1', periods=None):
     """Load the files contains segment list from the data quality files below the data quality category (dq_file.load_file)).
     Then merge the segment list (merge_seg_list) and return the merged segment list.
 
@@ -14,14 +14,22 @@ def read_seg_list(dq_file_list, dq_cat):
     :type dq_file_list: list[DQFile]
     :param dq_cat: The data quality category.
     :type dq_cat: str
+    :param periods: Given start and stop periods.
+    :type periods: tuple[list[int], list[int]]
     :return: A list of segments.
     :rtype: list[WaveSegment]
     """
     seg_list = []
-    for dq_file in dq_file_list:
-        if dq_file.dq_cat <= dq_cat:
-            logger.info(f"Loading data quality file {dq_file.file}")
-            seg_list.append(dq_file.get_periods())
+    if dq_file_list is None or len(dq_file_list) == 0:
+        seg_list.append(([0], [np.inf]))
+    else:
+        for dq_file in dq_file_list:
+            if dq_file.dq_cat <= dq_cat:
+                logger.info(f"Loading data quality file {dq_file.file}")
+                seg_list.append(dq_file.get_periods())
+
+    if periods is not None:
+        seg_list.append(periods)
 
     if len(seg_list) == 0:
         logger.error("No CWB_CAT=%s files in the list", dq_cat)
@@ -93,7 +101,7 @@ def get_seg_list(ifo, dq_list, seg_len, seg_mls, seg_edge):
     return []
 
 
-def get_job_list(ifos, dq_list, seg_len, seg_mls, seg_edge):
+def get_job_list(ifos, dq_list, seg_len, seg_mls, seg_edge, sample_rate, shift=None, index_start=0):
     """
     Build the job segment list.
 
@@ -108,6 +116,8 @@ def get_job_list(ifos, dq_list, seg_len, seg_mls, seg_edge):
     output segA : xxx---------xxx \n
     output segB :             xxx----------xxx \n
 
+    :param ifos:  list of interferometers
+    :type ifos:  list[str]
     :param dq_list:  number of detectors
     :type dq_list:  list[DQFile]
     :param seg_len:  Segment length [sec]
@@ -116,6 +126,12 @@ def get_job_list(ifos, dq_list, seg_len, seg_mls, seg_edge):
     :type seg_mls:  int
     :param seg_edge:  wavelet boundary offset [sec]
     :type seg_edge:  int
+    :param sample_rate:  sample rate [Hz]
+    :type sample_rate:  int
+    :param shift:  list of shifts for each interferometer, used for superlags
+    :type shift:  list[float]
+    :param index_start:  index of the first segment
+    :type index_start:  int
     :return:  Return the job segment list
     :rtype:  list[WaveSegment]
     """
@@ -126,7 +142,7 @@ def get_job_list(ifos, dq_list, seg_len, seg_mls, seg_edge):
     lostlivetime = 0
     job_list = []
 
-    seg_index = 0
+    seg_index = index_start
 
     for i in range(len(dq_list[0])):
         start = math.ceil(dq_list[0][i]) + seg_edge
@@ -141,7 +157,8 @@ def get_job_list(ifos, dq_list, seg_len, seg_mls, seg_edge):
                 lostlivetime += length
                 continue
             seg_index += 1
-            job_list.append(WaveSegment(seg_index, ifos, start, stop))
+            job_list.append(WaveSegment(seg_index, ifos, start, stop,
+                                        seg_edge=seg_edge, sample_rate=sample_rate, shift=shift))
             continue
         if n == 1:
             if length > seg_len:
@@ -149,32 +166,40 @@ def get_job_list(ifos, dq_list, seg_len, seg_mls, seg_edge):
                 half = int(remainder / 2)
                 if half >= seg_mls:
                     seg_index += 1
-                    job_list.append(WaveSegment(seg_index, ifos, start, start + half))
+                    job_list.append(WaveSegment(seg_index, ifos, start, start + half,
+                                                seg_edge=seg_edge, sample_rate=sample_rate, shift=shift))
 
                     seg_index += 1
-                    job_list.append(WaveSegment(seg_index, ifos, start + half, stop))
+                    job_list.append(WaveSegment(seg_index, ifos, start + half, stop,
+                                                seg_edge=seg_edge, sample_rate=sample_rate, shift=shift))
                 else:
                     seg_index += 1
-                    job_list.append(WaveSegment(seg_index, ifos, start, start + seg_len))
+                    job_list.append(WaveSegment(seg_index, ifos, start, start + seg_len,
+                                                seg_edge=seg_edge, sample_rate=sample_rate, shift=shift))
             else:
                 seg_index += 1
-                job_list.append(WaveSegment(i, ifos, start, stop))
+                job_list.append(WaveSegment(i, ifos, start, stop,
+                                            seg_edge=seg_edge, sample_rate=sample_rate, shift=shift))
             continue
 
         for j in range(n - 1):
             seg_index += 1
-            job_list.append(WaveSegment(seg_index, ifos, seg_len * j + start, seg_len * j + start + seg_len))
+            job_list.append(WaveSegment(seg_index, ifos, seg_len * j + start, seg_len * j + start + seg_len,
+                                        seg_edge=seg_edge, sample_rate=sample_rate, shift=shift))
 
         remainder = stop - job_list[-1].end_time
         half = int(remainder / 2)
         if half >= seg_mls:
             seg_index += 1
-            job_list.append(WaveSegment(seg_index, ifos, job_list[-1].end_time, job_list[-1].end_time + half))
+            job_list.append(WaveSegment(seg_index, ifos, job_list[-1].end_time, job_list[-1].end_time + half,
+                                        seg_edge=seg_edge, sample_rate=sample_rate, shift=shift))
             seg_index += 1
-            job_list.append(WaveSegment(seg_index, ifos, job_list[-1].end_time, stop))
+            job_list.append(WaveSegment(seg_index, ifos, job_list[-1].end_time, stop,
+                                        seg_edge=seg_edge, sample_rate=sample_rate, shift=shift))
         else:
             seg_index += 1
-            job_list.append(WaveSegment(seg_index, ifos, job_list[-1].end_time, job_list[-1].end_time + seg_len))
+            job_list.append(WaveSegment(seg_index, ifos, job_list[-1].end_time, job_list[-1].end_time + seg_len,
+                                        seg_edge=seg_edge, sample_rate=sample_rate, shift=shift))
 
     logger.info('lost livetime after building of the standard job list = %d sec' % lostlivetime)
 

{pycwb-0.23.2 → pycwb-0.25.0}/pycwb/modules/job_segment/frame.py

@@ -65,9 +65,9 @@ def select_frame_list(frame_list, start, stop, seg_edge):
     :param frame_list: list of frame metadata
     :type frame_list: list[FrameFile]
     :param start: start time of the segment
-    :type start: int
+    :type start: int or float
     :param stop: stop time of the segment
-    :type stop: int
+    :type stop: int or float
     :param seg_edge: buffer of the segment
     :type seg_edge: int or float
     :return: list of frame
@@ -81,3 +81,48 @@ def select_frame_list(frame_list, start, stop, seg_edge):
     return frame_list
 
 
+def get_frame_files_from_gwdatafind(ifo, site, frametype, start, stop, seg_edge, host=None):
+    """
+    Use gwdatafind to get the frame files for the job segments
+
+    :param ifo: name of the interferometer
+    :type ifo: str
+    :param site: site name of the interferometer
+    :type site: str
+    :param frametype: frame type
+    :type frametype: str
+    :param start: start time of the segment
+    :type start: int or float
+    :param stop: stop time of the segment
+    :type stop: int or float
+    :param seg_edge: buffer of the segment
+    :type seg_edge: int or float
+    :param host: host name of the frame server
+    :type host: str
+
+    :return: list of frame files
+    :rtype: list[FrameFile]
+    """
+    from gwdatafind import find_urls
+
+    seg_start = start - seg_edge
+    seg_stop = stop + seg_edge
+
+    frame_list = []
+
+    url = find_urls(site, frametype, seg_start, seg_stop, host=host)
+    frame_paths = [fp.replace("file://localhost", "") for fp in url]
+
+    for frame_path in frame_paths:
+        # get the file name without the extension with pathlib
+        frame_name = Path(frame_path).stem
+        # get the gps time and duration with int type
+        gps_start, duration = [int(i) for i in frame_name.split("-")[-2:]]
+        # append the frame file to the list
+        frame_list.append(FrameFile(ifo, frame_path, gps_start, duration))
+
+    # TODO: check if the framefiles match with the seg_start and seg_stop
+
+    return frame_list
+
+