PycWB 0.24.0__tar.gz → 0.26.0__tar.gz

{pycwb-0.24.0 → pycwb-0.26.0}/.gitlab-ci.yml

@@ -20,7 +20,7 @@ image: buildpack-deps:jammy
 stages:          # List of stages for jobs, and their order of execution
   - build
   - deploy
-  - post-deploy
+  #- post-deploy
 
 .install_common: &install_common
   - wget -q https://root.cern/download/root_v6.26.14.Linux-ubuntu22-x86_64-gcc11.4.tar.gz
@@ -64,23 +64,24 @@ publish:
     - pip install twine
     - TWINE_PASSWORD=${PYPI_TOKEN} TWINE_USERNAME=__token__ python -m twine upload dist/*.tar.gz
 
-pages:
-  stage: post-deploy
-  rules:
-    - changes:
-        - docs/**/*
-    - if: '$CI_COMMIT_TAG'
-  script:
-    - *install_common
-    - pip install pycwb
-    - ln -fs /usr/share/zoneinfo/Etc/UTC /etc/localtime
-    - pip install "sphinx<7.0.0" sphinx_rtd_theme
-    - make doc
-    - mkdir public
-    - cp -r docs/build/html/* public/
-  artifacts:
-    paths:
-      - public
+#pages:
+#  stage: post-deploy
+#  rules:
+#    - changes:
+#        - docs/**/*
+#    - if: '$CI_COMMIT_TAG'
+#  script:
+#    - *install_common
+#    - pip install pycwb
+#    - ln -fs /usr/share/zoneinfo/Etc/UTC /etc/localtime
+#    - pip install "sphinx<7.0.0" sphinx_rtd_theme
+#    - make doc
+#    - mkdir public
+#    - cp -r docs/build/html/* public/
+#  artifacts:
+#    paths:
+#      - public
+
 #test-job:   # This job runs in the test stage.
 #  image: continuumio/miniconda3
 #  stage: test    # It only starts when the job in the build stage completes successfully.

{pycwb-0.24.0 → pycwb-0.26.0}/.readthedocs.yaml

@@ -4,6 +4,9 @@ build:
   os: "ubuntu-22.04"
   tools:
     python: "mambaforge-22.9"
+  jobs:
+    pre_install:
+      - git update-index --assume-unchanged envs/environment.yml docs/source/conf.py
 
 conda:
   environment: envs/environment.yml

{pycwb-0.24.0 → pycwb-0.26.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: PycWB
-Version: 0.24.0
+Version: 0.26.0
 Summary: A Python package for coherent gravitational wave burst analysis
 Home-page: https://git.ligo.org/yumeng.xu/pycwb
 Author-email: The PycWB team <yumeng.xu@ligo.org>
@@ -31,6 +31,7 @@ Requires-Dist: prefect-dask
 Requires-Dist: dask
 Requires-Dist: dask_jobqueue
 Requires-Dist: htcondor
+Requires-Dist: psutil
 
 # PycWB
 

{pycwb-0.24.0 → pycwb-0.26.0}/PycWB.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: PycWB
-Version: 0.24.0
+Version: 0.26.0
 Summary: A Python package for coherent gravitational wave burst analysis
 Home-page: https://git.ligo.org/yumeng.xu/pycwb
 Author-email: The PycWB team <yumeng.xu@ligo.org>
@@ -31,6 +31,7 @@ Requires-Dist: prefect-dask
 Requires-Dist: dask
 Requires-Dist: dask_jobqueue
 Requires-Dist: htcondor
+Requires-Dist: psutil
 
 # PycWB
 

{pycwb-0.24.0 → pycwb-0.26.0}/PycWB.egg-info/SOURCES.txt

@@ -231,6 +231,9 @@ prototypes/messenger-web-interface/src/views/nodes/node3.vue
 prototypes/messenger-web-interface/src/views/nodes/nodeHeader.vue
 prototypes/performance/numba_test.py
 prototypes/performance/taichi_test.py
+prototypes/pp_workflow/simple.yaml
+prototypes/pp_workflow/simple_module.yaml
+prototypes/pp_workflow/workflow.yaml
 pycwb/__init__.py
 pycwb/_version.py
 pycwb/cli/__init__.py
@@ -238,6 +241,7 @@ pycwb/cli/batch_runner.py
 pycwb/cli/batch_setup.py
 pycwb/cli/flow.py
 pycwb/cli/merge_catalog.py
+pycwb/cli/post_process.py
 pycwb/cli/run.py
 pycwb/cli/server.py
 pycwb/cli/xtalk.py
@@ -322,6 +326,10 @@ pycwb/modules/read_data/read_data.py
 pycwb/modules/reconstruction/__init__.py
 pycwb/modules/reconstruction/getMRAwaveform.py
 pycwb/modules/reconstruction/module.yaml
+pycwb/modules/report/__init__.py
+pycwb/modules/report/far_rho.py
+pycwb/modules/report/read_results.py
+pycwb/modules/report/report.py
 pycwb/modules/sparse_series/__init__.py
 pycwb/modules/sparse_series/module.yaml
 pycwb/modules/sparse_series/sparse_table.py
@@ -339,6 +347,7 @@ pycwb/modules/super_cluster/utils.py
 pycwb/modules/supercluster_naive/__init__.py
 pycwb/modules/supercluster_naive/supercluster2.py
 pycwb/modules/superlag/__init__.py
+pycwb/modules/superlag/module.yaml
 pycwb/modules/superlag/superlag.py
 pycwb/modules/web_viewer/__init__.py
 pycwb/modules/web_viewer/create.py
@@ -349,6 +358,8 @@ pycwb/modules/xtalk/__init__.py
 pycwb/modules/xtalk/monster.py
 pycwb/modules/xtalk/monster_old.py
 pycwb/modules/xtalk/xtalk_data.py
+pycwb/post_production/__init__.py
+pycwb/post_production/workflow.py
 pycwb/prefect_flow/__init__.py
 pycwb/prefect_flow/pycwb_flow.py
 pycwb/prefect_flow/tasks/__init__.py
@@ -370,6 +381,7 @@ pycwb/utils/async_write.py
 pycwb/utils/check_ROOT.py
 pycwb/utils/dataclass_object_io.py
 pycwb/utils/dep_check.py
+pycwb/utils/generate_params_table.py
 pycwb/utils/image.py
 pycwb/utils/module.py
 pycwb/utils/network.py
@@ -393,5 +405,4 @@ pycwb/workflow/run.py
 pycwb/workflow/subflow/__init__.py
 pycwb/workflow/subflow/postprocess_and_plots.py
 pycwb/workflow/subflow/prepare_job_runs.py
-pycwb/workflow/subflow/process_job_segment.py
-pycwb/workflow/subflow/supercluster_and_likelihood.py
+pycwb/workflow/subflow/process_job_segment.py

{pycwb-0.24.0 → pycwb-0.26.0}/PycWB.egg-info/requires.txt

@@ -21,3 +21,4 @@ prefect-dask
 dask
 dask_jobqueue
 htcondor
+psutil

{pycwb-0.24.0 → pycwb-0.26.0}/bin/pycwb

@@ -11,6 +11,7 @@ import pycwb.cli.batch_setup as batch_setup
 import pycwb.cli.batch_runner as batch_runner
 import pycwb.cli.xtalk as xtalk
 import pycwb.cli.merge_catalog as merge_catalog
+import pycwb.cli.post_process as post_process
 
 def main():
     parser = argparse.ArgumentParser(prog='pycwb')
@@ -46,6 +47,10 @@ def main():
     merge_catalog.init_parser(merge_catalog_parser)
     merge_catalog_parser.set_defaults(func=merge_catalog.command)
 
+    post_process_parser = subparsers.add_parser('post-process', help='Post process the results')
+    post_process.init_parser(post_process_parser)
+    post_process_parser.set_defaults(func=post_process.command)
+
     args = parser.parse_args()
     if hasattr(args, 'func'):
         args.func(args)

{pycwb-0.24.0 → pycwb-0.26.0}/docs/source/conf.py

@@ -27,7 +27,7 @@ exclude_patterns = ["pycwb/vendor/*", ".DS_Store"]
 # -- Options for HTML output -------------------------------------------------
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output
 
-html_theme = 'sphinx_rtd_theme'
+html_theme = "furo"
 html_static_path = ['_static']
 
 # -- external links ----------------------------------------------------------

pycwb-0.26.0/docs/source/schema.rst

@@ -0,0 +1,15 @@
+.. _schema:
+
+User Parameters
+----------------
+
+The default schema for user parameters is:
+
+.. exec::
+    import json
+    from pycwb.constants import user_parameters_schema
+    from pycwb.utils.generate_params_table import generate_rst_table, parse_description, parse_type_or_enum
+
+    print(generate_rst_table(user_parameters_schema["properties"]))
+    # json_obj = json.dumps(user_parameters_schema, sort_keys=True, indent=4)
+    # print('.. code-block:: JavaScript\n\n    %s\n\n' % json_obj)

{pycwb-0.24.0 → pycwb-0.26.0}/envs/environment.yml

@@ -12,4 +12,5 @@ dependencies:
   - python=3.10
   - root=6.26.10
   - setuptools_scm
-  - sphinx_rtd_theme
+  - sphinx_rtd_theme
+  - furo

pycwb-0.26.0/prototypes/pp_workflow/simple.yaml

@@ -0,0 +1,43 @@
+global:
+  work_dir: /Users/yumengxu/Downloads/pycwb_examples
+steps:
+  - action: read_triggers
+    args:
+      run_dir: background
+      filters:
+        - "lag[0]>0"
+    output_alias: 'background'
+  - action: read_live_time
+    args:
+      run_dir: background
+      filters:
+        - "lag[0]>0"
+    output_alias: 'livetime'
+  - action: far_rho # this module will calculate the ifar_rho and pass the output to the downstream modules
+    args:
+      source: 'background'
+      livetime_key: 'livetime'
+      ranking_par: rho[0]
+      bin_size: 0.0001
+    output_alias: 'far_rho'
+  - action: read_triggers
+    args:
+      run_dir: background
+      filters:
+        - "lag[0]==0"
+    output_alias: 'zero_lag'
+  - action: read_live_time
+    args:
+      run_dir: background
+      filters:
+        - "lag[0]==0"
+    output_alias: 'livetime_zerolag'
+  - action: report_zero_lag
+    args:
+      source: 'zero_lag'
+      livetime_key: 'livetime_zerolag'
+      far_rho_source: 'far_rho'
+  - action: report # generate report
+    args:
+      far_rho_source: 'far_rho'
+      output: public/

pycwb-0.26.0/prototypes/pp_workflow/simple_module.yaml

@@ -0,0 +1,44 @@
+global:
+  work_dir: /Users/yumengxu/Downloads/pycwb_examples
+steps:
+  - action: report.read_results.read_triggers
+    args:
+      run_dir: background
+      filters:
+        - "lag[0]>0"
+    output_alias: 'background'
+  - action: report.read_results.read_live_time
+    args:
+      run_dir: background
+      filters:
+        - "lag[0]>0"
+    output_alias: 'livetime'
+  - action: report.far_rho.far_rho # this module will calculate the far_rho and pass the output to the downstream modules
+    args:
+      source: 'background'
+      livetime_key: 'livetime'
+      ranking_par: rho[0]
+      bin_size: 0.0001
+      save: far_rho.json
+    output_alias: 'far_rho'
+  - action: report.read_results.read_triggers
+    args:
+      run_dir: background
+      filters:
+        - "lag[0]==0"
+    output_alias: 'zero_lag'
+  - action: report.read_results.read_live_time
+    args:
+      run_dir: background
+      filters:
+        - "lag[0]==0"
+    output_alias: 'livetime_zerolag'
+  - action: report.report.report_zero_lag
+    args:
+      source: 'zero_lag'
+      livetime_key: 'livetime_zerolag'
+      far_rho_source: 'far_rho'
+  - action: report.report.report # generate report
+    args:
+      far_rho_source: 'far_rho'
+      output: public/

pycwb-0.26.0/prototypes/pp_workflow/workflow.yaml

@@ -0,0 +1,40 @@
+global:
+  work_dir: /Users/yumengxu/Downloads/pycwb_examples/non-builtin-waveform
+steps:
+  - action: events_filter # this module will filter the event and pass to the downstream modules
+    args:
+      run_dir: O4_K01_BurstLD_BKG
+      conditions:
+        - "lag>0" # only process non-zero lag events
+    output_alias: 'background'
+  - action: events_filter # read the training set and name it as 'training_set'
+    args:
+      run_dir: O4_K01_BurstLD_SIM_TrainingSet1
+    output_alias: 'sim_training_set'
+  - action: events_splitter # split the background into bkg_training_set and bkg_test_set
+    args:
+      source: 'background'
+      percent: [0.5, 0.5]
+    output_alias: ['bkg_training_set', 'bkg_test_set']
+  - action: xgboost_model # take the training set from the upsteam
+    args:
+      background: ['bkg_training_set']
+      simulation: ['sim_training_set']
+      config: xgb_config.py
+      get_rhor: xgb_get_rhor.py
+      name: 'model_v1a'
+  - action: xgboost_apply # apply xgb on BKG
+    args:
+      source: 'bkg_test_set'
+      model: 'model_v1a'
+    output_alias: 'bkg_test_set.model_v1a'
+  - action: ifar_rho # this module will calculate the ifar_rho and pass the output to the downstream modules
+    args:
+      source: 'bkg_test_set.model_v1a'
+      ranking_par: rho[1]
+      ranking_par_bin: 0.0001
+    output_alias: 'bkg_test_set.model_v1a.ifar'
+  - action: report # generate report
+    args:
+      source: 'bkg_test_set.model_v1a.ifar'
+    output: public/

{pycwb-0.24.0 → pycwb-0.26.0}/pycwb/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.24.0'
-__version_tuple__ = version_tuple = (0, 24, 0)
+__version__ = version = '0.26.0'
+__version_tuple__ = version_tuple = (0, 26, 0)

{pycwb-0.24.0 → pycwb-0.26.0}/pycwb/cli/batch_setup.py

@@ -73,9 +73,25 @@ def init_parser(parser):
                         '-j',
                         metavar='job_per_worker',
                         type=int,
-                        default=5,
+                        default=1,
                         help='the number of jobs per worker')
 
+    # memory
+    parser.add_argument('--memory',
+                        '-m',
+                        metavar='memory',
+                        type=str,
+                        default='6GB',
+                        help='the memory for each job')
+
+    # disk
+    parser.add_argument('--disk',
+                        '-k',
+                        metavar='disk',
+                        type=str,
+                        default='4GB',
+                        help='the disk space for each job')
+
     # compress json
     parser.add_argument('--compress_json',
                         action='store_true',
@@ -124,6 +140,8 @@ def command(args):
                 conda_env=args.conda_env,
                 additional_init=args.additional_init,
                 n_proc=args.n_proc,
+                memory=args.memory,
+                disk=args.disk,
                 accounting_group=args.accounting_group,
                 job_per_worker=args.job_per_worker,
                 submit=args.submit,

pycwb-0.26.0/pycwb/cli/post_process.py

@@ -0,0 +1,15 @@
+import os
+
+
+def init_parser(parser):
+    # Add the arguments
+    parser.add_argument('workflow_file',
+                        metavar='file_path',
+                        type=str,
+                        help='the path to the workflow file')
+
+
+def command(args):
+    from pycwb.post_production.workflow import run_workflow
+
+    run_workflow(args.workflow_file)

{pycwb-0.24.0 → pycwb-0.26.0}/pycwb/modules/catalog/catalog.py

@@ -1,7 +1,7 @@
 import os
 import logging
 from filelock import SoftFileLock
-import json
+import orjson
 import pycwb
 from pycwb.config import Config
 from pycwb.types.job import WaveSegment
@@ -32,13 +32,13 @@ def create_catalog(filename: str, config: Config, jobs: list[WaveSegment]) -> No
     output = {
         "config": config.__dict__,
         "version": pycwb.__version__,
-        "jobs": [job.to_dict() for job in jobs],
+        "jobs": jobs,
         "events": []
     }
 
     with SoftFileLock(filename + ".lock", timeout=10):
-        with open(filename, 'w') as f:
-            json.dump(output, f, default=vars)
+        with open(filename, 'wb') as f:
+            f.write(orjson.dumps(output, option=orjson.OPT_SERIALIZE_NUMPY))
 
 
 def add_events_to_catalog(filename: str, events: list[Event]) -> None:
@@ -64,13 +64,13 @@ def add_events_to_catalog(filename: str, events: list[Event]) -> None:
     if os.path.exists(filename):
         with SoftFileLock(filename + ".lock", timeout=10):
             # read the json file
-            with open(filename, 'r+') as f:
-                catalog = json.load(f)
+            with open(filename, 'rb+') as f:
+                catalog = orjson.loads(f.read())
                 # append events
                 catalog["events"].extend(events)
                 # write the json file
                 f.seek(0)
-                json.dump(catalog, f)
+                f.write(orjson.dumps(catalog, option=orjson.OPT_SERIALIZE_NUMPY))
     else:
         logger.warning("Catalog file does not exist. Event will not be saved to catalog.")
 

{pycwb-0.24.0 → pycwb-0.26.0}/pycwb/modules/data_conditioning/whitening.py

@@ -24,6 +24,7 @@ def whitening(config, h):
     layers_white = 2 ** config.l_white if config.l_white > 0 else 2 ** config.l_high
     wdm_white = WDM(layers_white, layers_white, config.WDM_beta_order, config.WDM_precision)
 
+    # TODO: check the length of data and white parameters to prevent freezing
     # check if whitening WDM filter lenght is less than cwb scratch
     wdmFlen = wdm_white.m_H / config.rateANA
     if wdmFlen > config.segEdge + 0.001:

{pycwb-0.24.0 → pycwb-0.26.0}/pycwb/modules/job_segment/dq_segment.py

@@ -101,7 +101,7 @@ def get_seg_list(ifo, dq_list, seg_len, seg_mls, seg_edge):
     return []
 
 
-def get_job_list(ifos, dq_list, seg_len, seg_mls, seg_edge, sample_rate):
+def get_job_list(ifos, dq_list, seg_len, seg_mls, seg_edge, sample_rate, shift=None, index_start=0):
     """
     Build the job segment list.
 
@@ -128,6 +128,10 @@ def get_job_list(ifos, dq_list, seg_len, seg_mls, seg_edge, sample_rate):
     :type seg_edge:  int
     :param sample_rate:  sample rate [Hz]
     :type sample_rate:  int
+    :param shift:  list of shifts for each interferometer, used for superlags
+    :type shift:  list[float]
+    :param index_start:  index of the first segment
+    :type index_start:  int
     :return:  Return the job segment list
     :rtype:  list[WaveSegment]
     """
@@ -138,7 +142,7 @@ def get_job_list(ifos, dq_list, seg_len, seg_mls, seg_edge, sample_rate):
     lostlivetime = 0
     job_list = []
 
-    seg_index = 0
+    seg_index = index_start
 
     for i in range(len(dq_list[0])):
         start = math.ceil(dq_list[0][i]) + seg_edge
@@ -154,7 +158,7 @@ def get_job_list(ifos, dq_list, seg_len, seg_mls, seg_edge, sample_rate):
                 continue
             seg_index += 1
             job_list.append(WaveSegment(seg_index, ifos, start, stop,
-                                        seg_edge=seg_edge, sample_rate=sample_rate))
+                                        seg_edge=seg_edge, sample_rate=sample_rate, shift=shift))
             continue
         if n == 1:
             if length > seg_len:
@@ -163,39 +167,39 @@ def get_job_list(ifos, dq_list, seg_len, seg_mls, seg_edge, sample_rate):
                 if half >= seg_mls:
                     seg_index += 1
                     job_list.append(WaveSegment(seg_index, ifos, start, start + half,
-                                                seg_edge=seg_edge, sample_rate=sample_rate))
+                                                seg_edge=seg_edge, sample_rate=sample_rate, shift=shift))
 
                     seg_index += 1
                     job_list.append(WaveSegment(seg_index, ifos, start + half, stop,
-                                                seg_edge=seg_edge, sample_rate=sample_rate))
+                                                seg_edge=seg_edge, sample_rate=sample_rate, shift=shift))
                 else:
                     seg_index += 1
                     job_list.append(WaveSegment(seg_index, ifos, start, start + seg_len,
-                                                seg_edge=seg_edge, sample_rate=sample_rate))
+                                                seg_edge=seg_edge, sample_rate=sample_rate, shift=shift))
             else:
                 seg_index += 1
                 job_list.append(WaveSegment(i, ifos, start, stop,
-                                            seg_edge=seg_edge, sample_rate=sample_rate))
+                                            seg_edge=seg_edge, sample_rate=sample_rate, shift=shift))
             continue
 
         for j in range(n - 1):
             seg_index += 1
             job_list.append(WaveSegment(seg_index, ifos, seg_len * j + start, seg_len * j + start + seg_len,
-                                        seg_edge=seg_edge, sample_rate=sample_rate))
+                                        seg_edge=seg_edge, sample_rate=sample_rate, shift=shift))
 
         remainder = stop - job_list[-1].end_time
         half = int(remainder / 2)
         if half >= seg_mls:
             seg_index += 1
             job_list.append(WaveSegment(seg_index, ifos, job_list[-1].end_time, job_list[-1].end_time + half,
-                                        seg_edge=seg_edge, sample_rate=sample_rate))
+                                        seg_edge=seg_edge, sample_rate=sample_rate, shift=shift))
             seg_index += 1
             job_list.append(WaveSegment(seg_index, ifos, job_list[-1].end_time, stop,
-                                        seg_edge=seg_edge, sample_rate=sample_rate))
+                                        seg_edge=seg_edge, sample_rate=sample_rate, shift=shift))
         else:
             seg_index += 1
             job_list.append(WaveSegment(seg_index, ifos, job_list[-1].end_time, job_list[-1].end_time + seg_len,
-                                        seg_edge=seg_edge, sample_rate=sample_rate))
+                                        seg_edge=seg_edge, sample_rate=sample_rate, shift=shift))
 
     logger.info('lost livetime after building of the standard job list = %d sec' % lostlivetime)
 

{pycwb-0.24.0 → pycwb-0.26.0}/pycwb/modules/job_segment/frame.py

@@ -81,16 +81,16 @@ def select_frame_list(frame_list, start, stop, seg_edge):
     return frame_list
 
 
-def get_frame_files_from_gwdatafind(ifos, sites, frametypes, start, stop, seg_edge, host=None):
+def get_frame_files_from_gwdatafind(ifo, site, frametype, start, stop, seg_edge, host=None):
     """
     Use gwdatafind to get the frame files for the job segments
 
-    :param ifos: list of interferometers
-    :type ifos: list[str]
-    :param sites: list of sites
-    :type sites: list[str]
-    :param frametypes: list of frame types
-    :type frametypes: list[str]
+    :param ifo: name of the interferometer
+    :type ifo: str
+    :param site: site name of the interferometer
+    :type site: str
+    :param frametype: frame type
+    :type frametype: str
     :param start: start time of the segment
     :type start: int or float
     :param stop: stop time of the segment
@@ -109,17 +109,19 @@ def get_frame_files_from_gwdatafind(ifos, sites, frametypes, start, stop, seg_ed
     seg_stop = stop + seg_edge
 
     frame_list = []
-    for i, frametype in enumerate(frametypes):
-        url = find_urls(sites[i], frametype, seg_start, seg_stop, host=host)
-        frame_paths = [fp.replace("file://localhost", "") for fp in url]
 
-        for frame_path in frame_paths:
-            # get the file name without the extension with pathlib
-            frame_name = Path(frame_path).stem
-            # get the gps time and duration with int type
-            gps_start, duration = [int(i) for i in frame_name.split("-")[-2:]]
-            # append the frame file to the list
-            frame_list.append(FrameFile(ifos[i], frame_path, gps_start, duration))
+    url = find_urls(site, frametype, seg_start, seg_stop, host=host)
+    frame_paths = [fp.replace("file://localhost", "") for fp in url]
+
+    for frame_path in frame_paths:
+        # get the file name without the extension with pathlib
+        frame_name = Path(frame_path).stem
+        # get the gps time and duration with int type
+        gps_start, duration = [int(i) for i in frame_name.split("-")[-2:]]
+        # append the frame file to the list
+        frame_list.append(FrameFile(ifo, frame_path, gps_start, duration))
+
+    # TODO: check if the framefiles match with the seg_start and seg_stop
 
     return frame_list