dq-genesis 0.1.0__tar.gz

dq_genesis-0.1.0/.claude/settings.local.json

@@ -0,0 +1,13 @@
+{
+  "permissions": {
+    "allow": [
+      "Read(//Users/zachyarbrough/miniforge3/envs/sgn-dq-dev/lib/python3.12/site-packages/sgn_arrakis/**)",
+      "Bash(python -c \"import arrakis, inspect; print\\(inspect.signature\\(arrakis.Publisher.__init__\\)\\); print\\('---register---'\\); print\\(inspect.signature\\(arrakis.Publisher.register\\)\\); print\\('---publish---'\\); print\\(inspect.signature\\(arrakis.Publisher.publish\\)\\); print\\('---Channel---'\\); print\\(inspect.signature\\(arrakis.Channel.__init__\\) if hasattr\\(arrakis.Channel,'__init__'\\) else arrakis.Channel\\); print\\('---SeriesBlock---'\\); print\\(inspect.signature\\(arrakis.SeriesBlock.__init__\\) if hasattr\\(arrakis,'SeriesBlock'\\) else 'n/a'\\)\")",
+      "Read(//private/tmp/publisher-configurations/LHO/**)",
+      "Read(//private/tmp/publisher-configurations/**)",
+      "Read(//Users/zachyarbrough/projects/ligo/sgn-ligo/**)",
+      "Read(//Users/zachyarbrough/projects/ligo/sgndq/sgn-ligo/**)",
+      "Read(//Users/zachyarbrough/projects/**)"
+    ]
+  }
+}

dq_genesis-0.1.0/.gitignore

@@ -0,0 +1,10 @@
+*.egg-info
+build/
+dist/
+site/
+__pycache__
+_version.py
+.coverage
+.vscode/
+.idea/
+*.patch

dq_genesis-0.1.0/.gitlab-ci.yml

@@ -0,0 +1,78 @@
+include:
+  # -- python
+  - component: git.ligo.org/computing/gitlab/components/python/sdist@1
+  - component: git.ligo.org/computing/gitlab/components/python/wheel@1
+  - component: git.ligo.org/computing/gitlab/components/python/code-quality@1
+    inputs:
+      analyzer: "ruff"
+      requirements: "-r requirements-lint.txt"
+  - component: git.ligo.org/computing/gitlab/components/python/dependency-scanning@1
+  - component: git.ligo.org/computing/gitlab/components/python/type-checking@1
+    inputs:
+      project_dir: "src"
+  - component: git.ligo.org/computing/gitlab/components/python/test@1
+    inputs:
+      install_extra: test
+      python_versions:
+        - "3.11"
+        - "3.12"
+        - "3.13"
+  # -- docker
+  - component: git.ligo.org/computing/gitlab/components/docker/build@1
+    inputs:
+      image_tag: $CI_COMMIT_SHA
+  - component: git.ligo.org/computing/gitlab/components/docker/push@1
+    inputs:
+      pull_image_tag: $CI_COMMIT_SHA
+      push_image_tag: $CI_COMMIT_REF_NAME
+      push_when: "all"
+      tag_latest: false
+    rules:
+      - if: $CI_COMMIT_BRANCH
+      - if: $CI_COMMIT_TAG !~ /^(\d+\.)?(\d+\.)?(\*|\d+)$/
+  - component: git.ligo.org/computing/gitlab/components/docker/push@1
+    inputs:
+      pull_image_tag: $CI_COMMIT_SHA
+      push_image_tag: $CI_COMMIT_TAG
+      push_when: "tags"
+      tag_latest: true
+    rules:
+      # tag latest only on new versions
+      - if: $CI_COMMIT_TAG =~ /^(\d+\.)?(\d+\.)?(\*|\d+)$/
+  # -- docs
+  - component: git.ligo.org/computing/gitlab/components/mkdocs/build@1
+    inputs:
+      requirements: "-r requirements-docs.txt"
+  - component: git.ligo.org/computing/gitlab/components/mkdocs/pages@1
+    inputs:
+      pages_when: "default"
+
+# -- customizations
+docker_push_gitlab:
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+      when: never
+    - when: always
+
+ruff:
+  needs:
+    - requirements
+
+mkdocs:
+  needs:
+    - requirements
+
+# -- requirements
+requirements:
+  stage: build
+  image: python:3.12
+  script:
+    - python -m pip install pipx
+    - pipx ensurepath
+    - source ~/.bashrc
+    - pipx install hatch
+    - hatch dep show requirements --feature docs > requirements-docs.txt
+    - hatch dep show requirements --feature lint > requirements-lint.txt
+  artifacts:
+    paths:
+      - requirements-*.txt

dq_genesis-0.1.0/CHANGELOG.md

@@ -0,0 +1,9 @@
+# Changelog
+
+## [Unreleased]
+
+- Add Arrakis data source (`--arrakis`) for bounded replay or live streaming.
+- Initial release: YAML flag config parsing, pipeline construction, and CLI
+  with offline (frame cache) and online (shared-memory) data sources.
+
+[unreleased]: https://git.ligo.org/detchar/sgn-dq/dq-genesis/-/commits/main

dq_genesis-0.1.0/Dockerfile

@@ -0,0 +1,39 @@
+# build stage: export locked dependencies
+FROM ghcr.io/astral-sh/uv:0.8-python3.13-bookworm AS build
+
+WORKDIR /app
+RUN apt-get update && apt-get install -y --no-install-recommends git && \
+    rm -rf /var/lib/apt/lists/*
+
+COPY . .
+
+RUN uv export --no-editable --no-emit-project --locked \
+    --format requirements.txt -o requirements.txt
+RUN uv build --wheel
+
+# install stage: install dependencies into virtualenv
+FROM python:3.13 AS install
+
+WORKDIR /app
+COPY --from=build /app/requirements.txt /app
+COPY --from=build /app/dist/*.whl /app
+
+RUN python -m venv /opt/venv
+ENV PATH="/opt/venv/bin:$PATH"
+
+RUN pip install --no-cache-dir -r requirements.txt
+RUN pip install --no-cache-dir --no-deps *.whl
+
+# final stage: minimal runtime image
+FROM python:3.13-slim
+
+COPY --from=install /opt/venv /opt/venv
+
+RUN groupadd --gid 1000 dqgenesis && \
+    useradd --uid 1000 --gid 1000 -m dqgenesis
+
+USER dqgenesis
+
+ENV PATH="/opt/venv/bin:$PATH"
+
+ENTRYPOINT ["dq-genesis"]

dq_genesis-0.1.0/Makefile

@@ -0,0 +1,42 @@
+ifeq ($(PYTHONPATH),)
+	PYTHONPATH := src
+else
+	PYTHONPATH := src:$(PYTHONPATH)
+endif
+export PYTHONPATH
+
+.PHONY: all
+all : format lint check test
+
+.PHONY: help
+help :
+	@echo
+	@echo 'Commands:'
+	@echo
+	@echo '  make test                  run tests'
+	@echo '  make lint                  run linter'
+	@echo '  make format                run code formatter'
+	@echo '  make check                 run static type checker'
+	@echo '  make docs                  build documentation'
+	@echo
+
+.PHONY: test
+test :
+	pytest -v --cov=dqgenesis --cov-report=term-missing src/dqgenesis/tests
+
+.PHONY: lint
+lint :
+	ruff check
+
+.PHONY: format
+format :
+	ruff format
+	ruff check --fix
+
+.PHONY: check
+check :
+	mypy src
+
+.PHONY: docs
+docs :
+	mkdocs build -c

dq_genesis-0.1.0/PKG-INFO

@@ -0,0 +1,133 @@
+Metadata-Version: 2.4
+Name: dq-genesis
+Version: 0.1.0
+Summary: Pipeline construction and execution for the SGN DQ module
+Project-URL: Homepage, https://git.ligo.org/detchar/sgn-dq/dq-genesis
+Project-URL: Documentation, https://detchar.docs.ligo.org/sgn-dq/dq-genesis
+Project-URL: Repository, https://git.ligo.org/detchar/sgn-dq/dq-genesis.git
+Project-URL: Issues, https://git.ligo.org/detchar/sgn-dq/dq-genesis/issues
+Author-email: Derek Davis <derek.davis@ligo.org>, Olivia Godwin <olivia.godwin@ligo.org>, Zach Yarbrough <zach.yarbrough@ligo.org>
+Maintainer-email: Derek Davis <derek.davis@ligo.org>, Olivia Godwin <olivia.godwin@ligo.org>, Zach Yarbrough <zach.yarbrough@ligo.org>
+License-Expression: GPL-3.0-or-later
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Scientific/Engineering :: Astronomy
+Classifier: Topic :: Scientific/Engineering :: Physics
+Requires-Python: >=3.11
+Requires-Dist: pyyaml>=6.0
+Requires-Dist: scipy>=1.10
+Requires-Dist: sgn-arrakis
+Requires-Dist: sgn-dq>=0.0.4
+Requires-Dist: sgn-gwframe
+Requires-Dist: sgn-ligo
+Requires-Dist: sgn-ts
+Requires-Dist: sgn>=0.6
+Provides-Extra: dev
+Requires-Dist: markdown-callouts>=0.2; extra == 'dev'
+Requires-Dist: markdown-exec>=0.5; extra == 'dev'
+Requires-Dist: mkdocs-coverage>=0.2; extra == 'dev'
+Requires-Dist: mkdocs-gen-files>=0.3; extra == 'dev'
+Requires-Dist: mkdocs-literate-nav>=0.4; extra == 'dev'
+Requires-Dist: mkdocs-material-igwn; extra == 'dev'
+Requires-Dist: mkdocs-section-index>=0.3; extra == 'dev'
+Requires-Dist: mkdocs>=1.3; extra == 'dev'
+Requires-Dist: mkdocstrings[python]; extra == 'dev'
+Requires-Dist: mypy; extra == 'dev'
+Requires-Dist: mypy-extensions; extra == 'dev'
+Requires-Dist: pip; extra == 'dev'
+Requires-Dist: pytest; extra == 'dev'
+Requires-Dist: pytest-cov; extra == 'dev'
+Requires-Dist: ruff; extra == 'dev'
+Requires-Dist: toml>=0.10; extra == 'dev'
+Provides-Extra: docs
+Requires-Dist: markdown-callouts>=0.2; extra == 'docs'
+Requires-Dist: markdown-exec>=0.5; extra == 'docs'
+Requires-Dist: mkdocs-coverage>=0.2; extra == 'docs'
+Requires-Dist: mkdocs-gen-files>=0.3; extra == 'docs'
+Requires-Dist: mkdocs-literate-nav>=0.4; extra == 'docs'
+Requires-Dist: mkdocs-material-igwn; extra == 'docs'
+Requires-Dist: mkdocs-section-index>=0.3; extra == 'docs'
+Requires-Dist: mkdocs>=1.3; extra == 'docs'
+Requires-Dist: mkdocstrings[python]; extra == 'docs'
+Requires-Dist: toml>=0.10; extra == 'docs'
+Provides-Extra: lint
+Requires-Dist: mypy; extra == 'lint'
+Requires-Dist: mypy-extensions; extra == 'lint'
+Requires-Dist: pip; extra == 'lint'
+Requires-Dist: ruff; extra == 'lint'
+Provides-Extra: test
+Requires-Dist: pytest; extra == 'test'
+Requires-Dist: pytest-cov; extra == 'test'
+Description-Content-Type: text/markdown
+
+# dq-genesis
+
+Pipeline construction and execution for the creation of the data quality vector.
+
+`dq-genesis` parses a YAML DQ flag config, assembles an `sgn.Pipeline` from the
+evaluators and transforms provided by `sgn-dq`, and runs it against frame data
+(offline frame cache), shared-memory frames (online), or an Arrakis stream. The
+output is a uint32 `DQG-DQ_VECTOR` channel written to a GWF frame stream.
+
+## Install
+
+```bash
+pip install -e /path/to/dq-genesis
+```
+
+`dq-genesis` depends on `sgn-dq` (evaluators, `Not`, `Parity`), `sgn-ts`,
+`sgn-ligo`, and `sgn`.
+
+## Quick start
+
+```bash
+# Validate a config
+dq-genesis --flags configs/dqmodule_h1.yaml --dry-run
+
+# Offline run against a frame cache
+dq-genesis --flags configs/dqmodule_h1.yaml \
+  --gps-start 1234567890 --gps-end 1234568000 \
+  --frame-cache h1.cache --output-dir /tmp/out
+
+# Bounded replay from Arrakis (omit --gps-start/--gps-end for a live stream)
+dq-genesis --flags configs/dqmodule_h1.yaml \
+  --arrakis --gps-start 1234567890 --gps-end 1234568000 \
+  --output-dir /tmp/out
+```
+
+See `src/dqgenesis/cli.py --help` for the full argument surface.
+
+## Output destinations
+
+The assembled DQ vector can be fanned out to one or more sinks. Select any
+combination of the `--publish-*` flags; if none is given it defaults to disk,
+preserving the original frame-writing behaviour.
+
+- `--publish-disk` — write the DQ vector to GWF frame files. Control the
+  destination with `--output-dir` (and `--write-path`, `--frame-duration`).
+- `--publish-arrakis` — publish the DQ vector channel back to an Arrakis
+  server. Requires `--publisher-id`; pass `--replay-id` to publish into a
+  replay namespace.
+
+```bash
+# Save to GWF frames (default)
+dq-genesis --flags configs/dqmodule_h1.yaml \
+  --gps-start 1234567890 --gps-end 1234568000 \
+  --frame-cache h1.cache --publish-disk --output-dir /tmp/out
+
+# Publish back into Arrakis
+dq-genesis --flags configs/dqmodule_h1.yaml \
+  --arrakis --gps-start 1234567890 --gps-end 1234568000 \
+  --publish-arrakis --publisher-id <publisher_id>
+
+# Both at once — the DQ vector is fanned out to disk and Arrakis
+dq-genesis --flags configs/dqmodule_h1.yaml \
+  --arrakis --gps-start 1234567890 --gps-end 1234568000 \
+  --publish-disk --output-dir /tmp/out \
+  --publish-arrakis --publisher-id <publisher_id>
+```
+
+The output channel is named `<IFO>:DQG-DQ_VECTOR` by default; override it with
+`--output-channel`.

dq_genesis-0.1.0/README.md

@@ -0,0 +1,69 @@
+# dq-genesis
+
+Pipeline construction and execution for the creation of the data quality vector.
+
+`dq-genesis` parses a YAML DQ flag config, assembles an `sgn.Pipeline` from the
+evaluators and transforms provided by `sgn-dq`, and runs it against frame data
+(offline frame cache), shared-memory frames (online), or an Arrakis stream. The
+output is a uint32 `DQG-DQ_VECTOR` channel written to a GWF frame stream.
+
+## Install
+
+```bash
+pip install -e /path/to/dq-genesis
+```
+
+`dq-genesis` depends on `sgn-dq` (evaluators, `Not`, `Parity`), `sgn-ts`,
+`sgn-ligo`, and `sgn`.
+
+## Quick start
+
+```bash
+# Validate a config
+dq-genesis --flags configs/dqmodule_h1.yaml --dry-run
+
+# Offline run against a frame cache
+dq-genesis --flags configs/dqmodule_h1.yaml \
+  --gps-start 1234567890 --gps-end 1234568000 \
+  --frame-cache h1.cache --output-dir /tmp/out
+
+# Bounded replay from Arrakis (omit --gps-start/--gps-end for a live stream)
+dq-genesis --flags configs/dqmodule_h1.yaml \
+  --arrakis --gps-start 1234567890 --gps-end 1234568000 \
+  --output-dir /tmp/out
+```
+
+See `src/dqgenesis/cli.py --help` for the full argument surface.
+
+## Output destinations
+
+The assembled DQ vector can be fanned out to one or more sinks. Select any
+combination of the `--publish-*` flags; if none is given it defaults to disk,
+preserving the original frame-writing behaviour.
+
+- `--publish-disk` — write the DQ vector to GWF frame files. Control the
+  destination with `--output-dir` (and `--write-path`, `--frame-duration`).
+- `--publish-arrakis` — publish the DQ vector channel back to an Arrakis
+  server. Requires `--publisher-id`; pass `--replay-id` to publish into a
+  replay namespace.
+
+```bash
+# Save to GWF frames (default)
+dq-genesis --flags configs/dqmodule_h1.yaml \
+  --gps-start 1234567890 --gps-end 1234568000 \
+  --frame-cache h1.cache --publish-disk --output-dir /tmp/out
+
+# Publish back into Arrakis
+dq-genesis --flags configs/dqmodule_h1.yaml \
+  --arrakis --gps-start 1234567890 --gps-end 1234568000 \
+  --publish-arrakis --publisher-id <publisher_id>
+
+# Both at once — the DQ vector is fanned out to disk and Arrakis
+dq-genesis --flags configs/dqmodule_h1.yaml \
+  --arrakis --gps-start 1234567890 --gps-end 1234568000 \
+  --publish-disk --output-dir /tmp/out \
+  --publish-arrakis --publisher-id <publisher_id>
+```
+
+The output channel is named `<IFO>:DQG-DQ_VECTOR` by default; override it with
+`--output-channel`.

dq_genesis-0.1.0/configs/dqmodule_h1.yaml

@@ -0,0 +1,53 @@
+# H1 DQ Flag Definitions
+# Equivalent to: H-DQ+Gate_O3.json (dq_flags section only)
+#
+# These flags are evaluated at 16 Hz (flag_step = 0.0625s) and assembled
+# into the H1:DQG-DQ_VECTOR channel. Bit convention: 1 = good, 0 = problem.
+# Flags with invert: true produce buffer when the problem IS detected, so
+# the bit is cleared (set to 0) when the evaluator fires.
+
+parity_bit: 0
+flag_step: 0.0625
+
+flags:
+  - name: "OMC_DCPD_ADC_OVERFLOW"
+    bit: 1
+    evaluator: overflow
+    invert: true
+    channels:
+      - "H1:FEC-8_ADC_OVERFLOW_ACC_0_12"
+      - "H1:FEC-8_ADC_OVERFLOW_ACC_0_13"
+
+  - name: "BLRMS_STRAIN_70_110"
+    bit: 5
+    evaluator: band_rms
+    channels:
+      - "H1:CAL-DELTAL_EXTERNAL_DQ"
+    params:
+      rms_max: 3.0e-7
+      settle: 2.0
+      bandpass:
+        type: elliptic
+        order: 8
+        ripple_db: 1.0
+        stop_atten_db: 100.0
+        low_hz: 70
+        high_hz: 110
+    test_channel: "H1:DMT-BLRMS_STRAIN_70_110"
+
+  - name: "BLRMS_STRAIN_25_50"
+    bit: 6
+    evaluator: band_rms
+    channels:
+      - "H1:CAL-DELTAL_EXTERNAL_DQ"
+    params:
+      rms_max: 5.0e-7
+      settle: 2.0
+      bandpass:
+        type: elliptic
+        order: 8
+        ripple_db: 1.0
+        stop_atten_db: 100.0
+        low_hz: 25
+        high_hz: 50
+    test_channel: "H1:DMT-BLRMS_STRAIN_25_50"

dq_genesis-0.1.0/docs/gen_ref_nav.py

@@ -0,0 +1,58 @@
+# Copyright (c) 2021, Timothee Mazzucotelli
+#
+# Permission to use, copy, modify, and/or distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+"""Generate the code reference pages and navigation."""
+
+from pathlib import Path
+
+import mkdocs_gen_files
+
+nav = mkdocs_gen_files.Nav()
+
+src = Path("src")
+
+for path in sorted(src.joinpath("dqgenesis").rglob("*.py")):
+    if path.name[0] == "_" or "tests" in path.parts:
+        continue
+
+    # Module path relative to src/ (e.g., dqgenesis/pipeline/builder)
+    module_path = path.relative_to(src).with_suffix("")
+    # Doc path mirrors module structure
+    doc_path = path.relative_to(src).with_suffix(".md")
+    full_doc_path = Path("reference", doc_path)
+
+    parts = tuple(module_path.parts)
+
+    if parts[-1] == "__init__":
+        parts = parts[:-1]
+        doc_path = doc_path.with_name("index.md")
+        full_doc_path = full_doc_path.with_name("index.md")
+    elif parts[-1] == "__main__":
+        continue
+
+    nav[parts] = doc_path.as_posix()
+
+    with mkdocs_gen_files.open(full_doc_path, "w") as fd:
+        ident = ".".join(parts)
+        fd.write(f"::: {ident}")
+
+    mkdocs_gen_files.set_edit_path(full_doc_path, path.relative_to(src))
+
+with mkdocs_gen_files.open("reference/SUMMARY.md", "w") as nav_file:
+    nav_file.writelines(nav.build_literate_nav())
+
+changelog = Path("CHANGELOG.md")
+if changelog.exists():
+    with mkdocs_gen_files.open("changelog.md", "w") as fd:
+        fd.write(changelog.read_text())

dq_genesis-0.1.0/docs/index.md

@@ -0,0 +1 @@
+--8<-- "README.md"

dq_genesis-0.1.0/mkdocs.yml

@@ -0,0 +1,60 @@
+site_name: dq-genesis
+site_description: Pipeline construction and execution for the SGN DQ module
+site_url: https://detchar.docs.ligo.org/sgn-dq/dq-genesis
+repo_url: https://git.ligo.org/detchar/sgn-dq/dq-genesis
+repo_name: detchar/sgn-dq/dq-genesis
+site_dir: site
+watch: [README.md, src/dqgenesis]
+
+nav:
+- Home:
+  - Overview: index.md
+# defer to gen-files + literate-nav
+- API Reference: reference/
+- Changelog: changelog.md
+
+theme:
+  name: material_igwn
+  features:
+    - content.code.copy
+    - content.code.annotate
+
+markdown_extensions:
+- admonition
+- callouts:
+    strip_period: no
+- pymdownx.highlight
+- pymdownx.inlinehilite
+- pymdownx.magiclink
+- pymdownx.snippets:
+    check_paths: true
+- pymdownx.superfences
+- toc:
+    permalink: true
+    toc_depth: 4
+
+plugins:
+- search
+- markdown-exec
+- gen-files:
+    scripts:
+    - docs/gen_ref_nav.py
+- literate-nav:
+    nav_file: SUMMARY.md
+- section-index
+- mkdocstrings:
+    handlers:
+      python:
+        paths: [src]
+        inventories:
+          - https://docs.python.org/3/objects.inv
+          - https://mkdocstrings.github.io/objects.inv
+          - https://mkdocstrings.github.io/griffe/objects.inv
+          - https://numpy.org/doc/stable/objects.inv
+        options:
+          docstring_style: numpy
+          docstring_options:
+            ignore_init_summary: yes
+          merge_init_into_class: yes
+          separate_signature: yes
+          show_root_full_path: no