gw-nomad 0.1.0__tar.gz

gw_nomad-0.1.0/.gitignore

@@ -0,0 +1,16 @@
+__pycache__/
+*.py[cod]
+*.egg-info/
+build/
+dist/
+site/
+.coverage*
+coverage.xml
+.pytest_cache/
+.mypy_cache/
+.ruff_cache/
+.venv/
+src/nomad/_version.py
+*.h5
+*.gwf
+*.pickle

gw_nomad-0.1.0/.gitlab-ci.yml

@@ -0,0 +1,78 @@
+include:
+  # -- python
+  - component: git.ligo.org/computing/gitlab/components/python/sdist@1
+  - component: git.ligo.org/computing/gitlab/components/python/wheel@1
+  - component: git.ligo.org/computing/gitlab/components/python/code-quality@1
+    inputs:
+      analyzer: "ruff"
+      requirements: "-r requirements-lint.txt"
+  - component: git.ligo.org/computing/gitlab/components/python/dependency-scanning@1
+  - component: git.ligo.org/computing/gitlab/components/python/type-checking@1
+    inputs:
+      project_dir: "src"
+  - component: git.ligo.org/computing/gitlab/components/python/test@1
+    inputs:
+      install_extra: test
+      python_versions:
+        - "3.11"
+        - "3.12"
+        - "3.13"
+  # -- docker
+  - component: git.ligo.org/computing/gitlab/components/docker/build@1
+    inputs:
+      image_tag: $CI_COMMIT_SHA
+  - component: git.ligo.org/computing/gitlab/components/docker/push@1
+    inputs:
+      pull_image_tag: $CI_COMMIT_SHA
+      push_image_tag: $CI_COMMIT_REF_NAME
+      push_when: "all"
+      tag_latest: false
+    rules:
+      - if: $CI_COMMIT_BRANCH
+      - if: $CI_COMMIT_TAG !~ /^(\d+\.)?(\d+\.)?(\*|\d+)$/
+  - component: git.ligo.org/computing/gitlab/components/docker/push@1
+    inputs:
+      pull_image_tag: $CI_COMMIT_SHA
+      push_image_tag: $CI_COMMIT_TAG
+      push_when: "tags"
+      tag_latest: true
+    rules:
+      # tag latest only on new versions
+      - if: $CI_COMMIT_TAG =~ /^(\d+\.)?(\d+\.)?(\*|\d+)$/
+  # -- docs
+  - component: git.ligo.org/computing/gitlab/components/mkdocs/build@1
+    inputs:
+      requirements: "-r requirements-docs.txt"
+  - component: git.ligo.org/computing/gitlab/components/mkdocs/pages@1
+    inputs:
+      pages_when: "default"
+
+# -- customizations
+docker_push_gitlab:
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+      when: never
+    - when: always
+
+ruff:
+  needs:
+    - requirements
+
+mkdocs:
+  needs:
+    - requirements
+
+# -- requirements
+requirements:
+  stage: build
+  image: python:3.12
+  script:
+    - python -m pip install pipx
+    - pipx ensurepath
+    - source ~/.bashrc
+    - pipx install hatch
+    - hatch dep show requirements --feature docs > requirements-docs.txt
+    - hatch dep show requirements --feature lint > requirements-lint.txt
+  artifacts:
+    paths:
+      - requirements-*.txt

gw_nomad-0.1.0/CHANGELOG.md

@@ -0,0 +1,34 @@
+# Changelog
+
+## Unreleased
+
+- Shaping filter support (model format version 2): nonsens models trained
+  with a frequency-domain `shaping_filter` now convert instead of being
+  rejected, and the engine applies the tabulated response to the noise
+  estimate as a streaming FIR (`--shaping-fir-duration`, default 1 s, half
+  of which becomes latency). Omitting it would leave `|T-1|` of the noise
+  in the target — ~10% at 100 Hz and worse-than-nothing above 1 kHz for the
+  O4 H1 jitter model.
+
+- Converter: models trained at the target rate (empty real resampling zpk
+  arrays in the pickle) no longer fail to write with an h5py dtype
+  conversion error.
+
+- Frame source: channels split across frame types (HOFT strain vs raw
+  witnesses) are located automatically. `--auxiliary-cache` (repeatable)
+  adds caches; entries are grouped by frame type, one file per type is
+  probed for its channel list, and one FrameReader is built per frame type.
+
+- Initial implementation: sgn reimplementation of NonSENS streaming noise
+  subtraction.
+  - HDF5 model format and `nomad-convert-model` converter for original
+    nonsens pickles.
+  - `SubtractionEngine` / `NomadSubtract` streaming element with
+    noise-estimate (`noise`) and subtracted (`hsub`) outputs, gap handling,
+    and a `--measure-only` mode for feeding the calibration pipeline's
+    NOLINES -> CLEAN subtractor.
+  - `nomad-subtract` CLI: arrakis/frames/fake sources, arrakis publishing,
+    frame writing.
+  - `nomad-train` CLI: native training (scipy L-BFGS over the analytic
+    cost/gradient, no TensorFlow), YAML config, target defaulting to the
+    NOLINES channel.

gw_nomad-0.1.0/Dockerfile

@@ -0,0 +1,39 @@
+# build stage: export locked dependencies
+FROM ghcr.io/astral-sh/uv:0.8-python3.13-bookworm AS build
+
+WORKDIR /app
+RUN apt-get update && apt-get install -y --no-install-recommends git && \
+    rm -rf /var/lib/apt/lists/*
+
+COPY . .
+
+RUN uv export --no-editable --no-emit-project --locked \
+    --format requirements.txt -o requirements.txt
+RUN uv build --wheel
+
+# install stage: install dependencies into virtualenv
+FROM python:3.13 AS install
+
+WORKDIR /app
+COPY --from=build /app/requirements.txt /app
+COPY --from=build /app/dist/*.whl /app
+
+RUN python -m venv /opt/venv
+ENV PATH="/opt/venv/bin:$PATH"
+
+RUN pip install --no-cache-dir -r requirements.txt
+RUN pip install --no-cache-dir --no-deps *.whl
+
+# final stage: minimal runtime image
+FROM python:3.13-slim
+
+COPY --from=install /opt/venv /opt/venv
+
+RUN groupadd --gid 1000 nomad && \
+    useradd --uid 1000 --gid 1000 -m nomad
+
+USER nomad
+
+ENV PATH="/opt/venv/bin:$PATH"
+
+ENTRYPOINT ["nomad-subtract"]

gw_nomad-0.1.0/PKG-INFO

@@ -0,0 +1,187 @@
+Metadata-Version: 2.4
+Name: gw-nomad
+Version: 0.1.0
+Summary: NOMAD: NOn-stationary Modulation-Aware Denoiser for sgn
+Author-email: Zach Yarbrough <zach.yarbrough@ligo.org>
+Maintainer-email: Zach Yarbrough <zach.yarbrough@ligo.org>
+License-Expression: GPL-3.0-or-later
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Scientific/Engineering :: Astronomy
+Classifier: Topic :: Scientific/Engineering :: Physics
+Requires-Python: >=3.11
+Requires-Dist: h5py
+Requires-Dist: numpy
+Requires-Dist: pyyaml
+Requires-Dist: scipy>=1.10
+Requires-Dist: sgn-arrakis
+Requires-Dist: sgn-ligo
+Requires-Dist: sgn-ts
+Requires-Dist: sgn>=0.6
+Provides-Extra: convert
+Requires-Dist: dill; extra == 'convert'
+Provides-Extra: dev
+Requires-Dist: dill; extra == 'dev'
+Requires-Dist: markdown-callouts>=0.2; extra == 'dev'
+Requires-Dist: markdown-exec>=0.5; extra == 'dev'
+Requires-Dist: matplotlib; extra == 'dev'
+Requires-Dist: mkdocs-coverage>=0.2; extra == 'dev'
+Requires-Dist: mkdocs-gen-files>=0.3; extra == 'dev'
+Requires-Dist: mkdocs-literate-nav>=0.4; extra == 'dev'
+Requires-Dist: mkdocs-material-igwn; extra == 'dev'
+Requires-Dist: mkdocs-section-index>=0.3; extra == 'dev'
+Requires-Dist: mkdocs>=1.3; extra == 'dev'
+Requires-Dist: mkdocstrings[python]; extra == 'dev'
+Requires-Dist: mypy; extra == 'dev'
+Requires-Dist: mypy-extensions; extra == 'dev'
+Requires-Dist: pip; extra == 'dev'
+Requires-Dist: pytest; extra == 'dev'
+Requires-Dist: pytest-cov; extra == 'dev'
+Requires-Dist: ruff; extra == 'dev'
+Requires-Dist: toml>=0.10; extra == 'dev'
+Provides-Extra: docs
+Requires-Dist: markdown-callouts>=0.2; extra == 'docs'
+Requires-Dist: markdown-exec>=0.5; extra == 'docs'
+Requires-Dist: mkdocs-coverage>=0.2; extra == 'docs'
+Requires-Dist: mkdocs-gen-files>=0.3; extra == 'docs'
+Requires-Dist: mkdocs-literate-nav>=0.4; extra == 'docs'
+Requires-Dist: mkdocs-material-igwn; extra == 'docs'
+Requires-Dist: mkdocs-section-index>=0.3; extra == 'docs'
+Requires-Dist: mkdocs>=1.3; extra == 'docs'
+Requires-Dist: mkdocstrings[python]; extra == 'docs'
+Requires-Dist: toml>=0.10; extra == 'docs'
+Provides-Extra: lint
+Requires-Dist: mypy; extra == 'lint'
+Requires-Dist: mypy-extensions; extra == 'lint'
+Requires-Dist: pip; extra == 'lint'
+Requires-Dist: ruff; extra == 'lint'
+Provides-Extra: plot
+Requires-Dist: matplotlib; extra == 'plot'
+Provides-Extra: test
+Requires-Dist: dill; extra == 'test'
+Requires-Dist: matplotlib; extra == 'test'
+Requires-Dist: pytest; extra == 'test'
+Requires-Dist: pytest-cov; extra == 'test'
+Description-Content-Type: text/markdown
+
+# NOMAD
+
+**NOn-stationary Modulation-Aware Denoiser** — an [sgn](https://git.ligo.org/greg/sgn)-based
+reimplementation of [NonSENS](https://git.ligo.org/gabriele-vajente/nonsens)
+(Vajente et al., PRD 101, 042003) streaming noise subtraction for
+gravitational-wave strain data.
+
+NOMAD measures excess noise in a target strain channel from auxiliary "noise
+witness" channels that couple into strain through transfer functions whose
+amplitude is modulated in time by slow "modulation witness" channels. The
+noise estimate is a sum of terms `alpha_i(f) * s_i(t)`, where each `s_i` is a
+modulated signal (noise witness × products of modulation witnesses) and each
+`alpha_i` is an optimal filter learned offline. The stationary case
+(`modulation_order = 0`) is plain multi-channel Wiener filtering.
+
+NOMAD's primary production role is the noise **measurement**: the SGN
+calibration pipeline's CAL-NOLINES → CAL-CLEAN stage is a plain subtractor
+that needs some other process to compute what to subtract (in O4 this was the
+NonSENS front-end installation). `nomad-subtract --measure-only
+--publish-noise` is that process, publishing the noise-estimate channel to
+arrakis on witnesses alone. It can also do the subtraction itself and publish
+or frame the cleaned channel directly.
+
+## Status
+
+- **v0.1**: streaming application of trained models as an sgn pipeline,
+  reading witness and target channels from
+  [arrakis](https://docs.ligo.org/ngdd/arrakis-python/) (or frame files) and
+  publishing the noise estimate and/or subtracted strain back to arrakis
+  (or frames).
+- **v0.2**: native model training (`nomad-train`, scipy L-BFGS over the
+  analytic cost/gradient — no TensorFlow), plus conversion of original
+  nonsens pickles (`nomad-convert-model`).
+
+## Installation
+
+```bash
+pip install .            # runtime
+pip install .[convert]   # + dill, for converting original nonsens pickles
+```
+
+## Usage
+
+Train a model (or convert an existing NonSENS pickle):
+
+```bash
+nomad-train --config lho_isc.yaml --data-source frames \
+  --frame-cache O4.cache --start 1400000000 --end 1400000600 \
+  --output lho_isc.h5
+
+nomad-convert-model lho_isc.pickle lho_isc.h5   # legacy pickles
+```
+
+The training config's `target` defaults to `{ifo}:GDS-CALIB_STRAIN_NOLINES`:
+the noise estimate is consumed downstream of the line subtraction, so
+training against NOLINES keeps already-removed lines out of the CSD
+estimates. Set `target` in the config (or pass `--target`) to train against
+any other channel, e.g. raw `GDS-CALIB_STRAIN` — as in the original nonsens,
+the target is simply whatever channel you name. A minimal config:
+
+```yaml
+ifo: H1
+noise_witnesses: [ASC-DHARD_P_OUT_DQ, ASC-DHARD_Y_OUT_DQ, 60.0]  # numbers = synthesized lines
+modulation_witnesses: []
+fs: 1024
+t_fft: 10
+fband: [8, 256]
+n_sos: 10
+glitches: {band: [15, 20], threshold: 3.0e-20}
+```
+
+Production: measure the excess noise and publish the estimate for the
+calibration pipeline's CLEAN subtractor (no target channel needed):
+
+```bash
+export ARRAKIS_SERVER=grpc://...
+nomad-subtract --model lho_isc.h5 --data-source arrakis --queue-timeout 30 \
+  --measure-only --publish-noise --publisher-id nomad-h1 \
+  --noise-channel-name H1:CAL-NOMAD_NOISE_ESTIMATE
+```
+
+Standalone cleaning — arrakis in, subtracted strain out:
+
+```bash
+nomad-subtract --model lho_isc.h5 --data-source arrakis --queue-timeout 30 \
+  --publish-arrakis --publisher-id nomad-h1 \
+  --output-channel-name H1:GDS-CALIB_STRAIN_NOMAD
+```
+
+Published channels must be pre-registered with the arrakis server under the
+given `--publisher-id`, at the target sample rate (float64). Note the noise
+estimate carries the model's FIR high-pass group delay (`length/2` seconds)
+as stream latency; timestamps are true GPS times, so a timestamp-aligned
+downstream subtractor handles it naturally.
+
+Run offline — frames in, frames out:
+
+```bash
+nomad-subtract --model lho_isc.h5 --data-source frames \
+  --frame-cache O3.cache --start 1242441180 --end 1242443180 \
+  --frame-output-path '{instruments}-{description}-{gps_start_time}-{duration}.gwf'
+```
+
+Channels rarely share a frame type (strain lives in HOFT frames, witnesses
+in raw R frames); pass each extra cache with `--auxiliary-cache`
+(repeatable). Cache entries are grouped by frame type, one file per type is
+probed for its channels, and each channel is read from whichever frame type
+provides it — a single concatenated mixed cache works too. This applies to
+`nomad-train` as well.
+
+## Development
+
+```bash
+hatch test           # run the test suite
+hatch run check      # mypy
+```
+
+Golden-comparison tests against the original nonsens implementation are
+skipped unless `NOMAD_GOLDEN=1` is set and `nonsens` is importable.

gw_nomad-0.1.0/README.md

@@ -0,0 +1,119 @@
+# NOMAD
+
+**NOn-stationary Modulation-Aware Denoiser** — an [sgn](https://git.ligo.org/greg/sgn)-based
+reimplementation of [NonSENS](https://git.ligo.org/gabriele-vajente/nonsens)
+(Vajente et al., PRD 101, 042003) streaming noise subtraction for
+gravitational-wave strain data.
+
+NOMAD measures excess noise in a target strain channel from auxiliary "noise
+witness" channels that couple into strain through transfer functions whose
+amplitude is modulated in time by slow "modulation witness" channels. The
+noise estimate is a sum of terms `alpha_i(f) * s_i(t)`, where each `s_i` is a
+modulated signal (noise witness × products of modulation witnesses) and each
+`alpha_i` is an optimal filter learned offline. The stationary case
+(`modulation_order = 0`) is plain multi-channel Wiener filtering.
+
+NOMAD's primary production role is the noise **measurement**: the SGN
+calibration pipeline's CAL-NOLINES → CAL-CLEAN stage is a plain subtractor
+that needs some other process to compute what to subtract (in O4 this was the
+NonSENS front-end installation). `nomad-subtract --measure-only
+--publish-noise` is that process, publishing the noise-estimate channel to
+arrakis on witnesses alone. It can also do the subtraction itself and publish
+or frame the cleaned channel directly.
+
+## Status
+
+- **v0.1**: streaming application of trained models as an sgn pipeline,
+  reading witness and target channels from
+  [arrakis](https://docs.ligo.org/ngdd/arrakis-python/) (or frame files) and
+  publishing the noise estimate and/or subtracted strain back to arrakis
+  (or frames).
+- **v0.2**: native model training (`nomad-train`, scipy L-BFGS over the
+  analytic cost/gradient — no TensorFlow), plus conversion of original
+  nonsens pickles (`nomad-convert-model`).
+
+## Installation
+
+```bash
+pip install .            # runtime
+pip install .[convert]   # + dill, for converting original nonsens pickles
+```
+
+## Usage
+
+Train a model (or convert an existing NonSENS pickle):
+
+```bash
+nomad-train --config lho_isc.yaml --data-source frames \
+  --frame-cache O4.cache --start 1400000000 --end 1400000600 \
+  --output lho_isc.h5
+
+nomad-convert-model lho_isc.pickle lho_isc.h5   # legacy pickles
+```
+
+The training config's `target` defaults to `{ifo}:GDS-CALIB_STRAIN_NOLINES`:
+the noise estimate is consumed downstream of the line subtraction, so
+training against NOLINES keeps already-removed lines out of the CSD
+estimates. Set `target` in the config (or pass `--target`) to train against
+any other channel, e.g. raw `GDS-CALIB_STRAIN` — as in the original nonsens,
+the target is simply whatever channel you name. A minimal config:
+
+```yaml
+ifo: H1
+noise_witnesses: [ASC-DHARD_P_OUT_DQ, ASC-DHARD_Y_OUT_DQ, 60.0]  # numbers = synthesized lines
+modulation_witnesses: []
+fs: 1024
+t_fft: 10
+fband: [8, 256]
+n_sos: 10
+glitches: {band: [15, 20], threshold: 3.0e-20}
+```
+
+Production: measure the excess noise and publish the estimate for the
+calibration pipeline's CLEAN subtractor (no target channel needed):
+
+```bash
+export ARRAKIS_SERVER=grpc://...
+nomad-subtract --model lho_isc.h5 --data-source arrakis --queue-timeout 30 \
+  --measure-only --publish-noise --publisher-id nomad-h1 \
+  --noise-channel-name H1:CAL-NOMAD_NOISE_ESTIMATE
+```
+
+Standalone cleaning — arrakis in, subtracted strain out:
+
+```bash
+nomad-subtract --model lho_isc.h5 --data-source arrakis --queue-timeout 30 \
+  --publish-arrakis --publisher-id nomad-h1 \
+  --output-channel-name H1:GDS-CALIB_STRAIN_NOMAD
+```
+
+Published channels must be pre-registered with the arrakis server under the
+given `--publisher-id`, at the target sample rate (float64). Note the noise
+estimate carries the model's FIR high-pass group delay (`length/2` seconds)
+as stream latency; timestamps are true GPS times, so a timestamp-aligned
+downstream subtractor handles it naturally.
+
+Run offline — frames in, frames out:
+
+```bash
+nomad-subtract --model lho_isc.h5 --data-source frames \
+  --frame-cache O3.cache --start 1242441180 --end 1242443180 \
+  --frame-output-path '{instruments}-{description}-{gps_start_time}-{duration}.gwf'
+```
+
+Channels rarely share a frame type (strain lives in HOFT frames, witnesses
+in raw R frames); pass each extra cache with `--auxiliary-cache`
+(repeatable). Cache entries are grouped by frame type, one file per type is
+probed for its channels, and each channel is read from whichever frame type
+provides it — a single concatenated mixed cache works too. This applies to
+`nomad-train` as well.
+
+## Development
+
+```bash
+hatch test           # run the test suite
+hatch run check      # mypy
+```
+
+Golden-comparison tests against the original nonsens implementation are
+skipped unless `NOMAD_GOLDEN=1` is set and `nonsens` is importable.

gw_nomad-0.1.0/docs/gen_ref_nav.py

@@ -0,0 +1,58 @@
+# Copyright (c) 2021, Timothee Mazzucotelli
+#
+# Permission to use, copy, modify, and/or distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+"""Generate the code reference pages and navigation."""
+
+from pathlib import Path
+
+import mkdocs_gen_files
+
+nav = mkdocs_gen_files.Nav()
+
+src = Path("src")
+
+for path in sorted(src.joinpath("nomad").rglob("*.py")):
+    if path.name[0] == "_" or "tests" in path.parts:
+        continue
+
+    # Module path relative to src/ (e.g., nomad/core/subtract)
+    module_path = path.relative_to(src).with_suffix("")
+    # Doc path mirrors module structure
+    doc_path = path.relative_to(src).with_suffix(".md")
+    full_doc_path = Path("reference", doc_path)
+
+    parts = tuple(module_path.parts)
+
+    if parts[-1] == "__init__":
+        parts = parts[:-1]
+        doc_path = doc_path.with_name("index.md")
+        full_doc_path = full_doc_path.with_name("index.md")
+    elif parts[-1] == "__main__":
+        continue
+
+    nav[parts] = doc_path.as_posix()
+
+    with mkdocs_gen_files.open(full_doc_path, "w") as fd:
+        ident = ".".join(parts)
+        fd.write(f"::: {ident}")
+
+    mkdocs_gen_files.set_edit_path(full_doc_path, path.relative_to(src))
+
+with mkdocs_gen_files.open("reference/SUMMARY.md", "w") as nav_file:
+    nav_file.writelines(nav.build_literate_nav())
+
+changelog = Path("CHANGELOG.md")
+if changelog.exists():
+    with mkdocs_gen_files.open("changelog.md", "w") as fd:
+        fd.write(changelog.read_text())

gw_nomad-0.1.0/docs/index.md

@@ -0,0 +1 @@
+--8<-- "README.md"

gw_nomad-0.1.0/mkdocs.yml

@@ -0,0 +1,48 @@
+site_name: NOMAD
+site_description: "NOMAD: NOn-stationary Modulation-Aware Denoiser for sgn"
+site_url: https://detchar.docs.ligo.org/sgn-dq/nomad
+repo_url: https://git.ligo.org/detchar/sgn-dq/nomad
+repo_name: detchar/sgn-dq/nomad
+site_dir: site
+watch: [README.md, src/nomad]
+
+nav:
+- Home:
+  - Overview: index.md
+# defer to gen-files + literate-nav
+- API Reference: reference/
+- Changelog: changelog.md
+
+theme:
+  name: material_igwn
+  features:
+    - content.code.copy
+    - content.code.annotate
+
+markdown_extensions:
+- admonition
+- callouts:
+    strip_period: no
+- pymdownx.highlight
+- pymdownx.inlinehilite
+- pymdownx.magiclink
+- pymdownx.snippets:
+    check_paths: true
+- pymdownx.superfences
+- toc:
+    permalink: true
+    toc_depth: 4
+
+plugins:
+- search
+- markdown-exec
+- gen-files:
+    scripts:
+    - docs/gen_ref_nav.py
+- literate-nav:
+    nav_file: SUMMARY.md
+- section-index
+- mkdocstrings:
+    handlers:
+      python:
+        paths: [src]