roman-snpit-snappl 0.10.0__tar.gz → 0.27.0__tar.gz

{roman_snpit_snappl-0.10.0 → roman_snpit_snappl-0.27.0}/.github/workflows/changelog.yml

@@ -23,6 +23,9 @@ jobs:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2
         with:
           fetch-depth: 0
+      - run: sudo apt-get -y update && sudo apt-get install -y libfftw3-double3
+      - run: sudo ln -s /usr/lib/x86_64-linux-gnu/libfftw3.so.3 /usr/lib/libfftw3.so
+      - run: ls -l /usr/lib/libff*
       - run: pip install .
       - run: pip install towncrier
       - run: towncrier check
@@ -34,4 +37,4 @@ jobs:
         with:
           fetch-depth: 0
       - name: prevent direct changes to `CHANGES.rst`
-        run: git diff HEAD ${{ github.event.pull_request.base.sha }} --no-patch --exit-code CHANGES.rst
+        run: git diff HEAD ${{ github.event.pull_request.base.sha }} --no-patch --exit-code CHANGES.rst

roman_snpit_snappl-0.27.0/.github/workflows/run_snappl_tests.yml

@@ -0,0 +1,77 @@
+name: Run SNAPPL Tests
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  tests:
+    name: run snappl tests in docker container
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Dump docker logs on failure
+        if: failure()
+        uses: jwalton/gh-docker-logs@v2
+
+      - name: checkout code
+        uses: actions/checkout@v5
+        with:
+          submodules: recursive
+
+      - name: run ruff
+        uses: astral-sh/ruff-action@v3
+
+      - name: checkout photometry test data
+        run: |
+          cd ..
+          git clone http://github.com/Roman-Supernova-PIT/photometry_test_data
+
+      - name: log into github container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: cleanup
+        run: |
+          # try to save HDD space on the runner by removing some unneeded stuff (~8GB)
+          # ref: https://github.com/actions/runner-images/issues/2840#issuecomment-790492173
+          # (For some perverse reason, this step seems to take 1.5-2 minutes on github runners!
+          # For that reason, this is commented out, for now.  Uncomment it if we run into trouble.)
+          df -h
+          # echo "------"
+          # sudo rm -rf /usr/share/dotnet
+          # sudo rm -rf /opt/ghc
+          # sudo rm -rf "/usr/local/share/boost"
+          # sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+          # echo "------"
+          # df -h
+
+      - name: pull environment archive
+        run: |
+          cd ..
+          git clone https://github.com/Roman-Supernova-PIT/environment.git
+
+
+      - name: pull docker images
+        run: |
+          cd ../environment/test-docker-environment
+          docker compose pull mailhog postgres webserver shell
+
+      - name: run test
+        run: |
+          cd ..
+          ls
+          echo "--------"
+          ls snappl
+          echo "--------"
+          cd environment/test-docker-environment
+          docker compose up -d webserver shell
+          docker compose exec shell bash -c 'cd /home/snappl && pip install -e .[test]'
+          docker compose exec shell bash -c 'cd /home/snappl/snappl/tests && pytest -v'
+          docker compose down -v

{roman_snpit_snappl-0.10.0/roman_snpit_snappl.egg-info → roman_snpit_snappl-0.27.0}/PKG-INFO

@@ -1,7 +1,7 @@
 Metadata-Version: 2.4
 Name: roman_snpit_snappl
-Version: 0.10.0
-Summary: Photometry utilities for the Roman SNPIT
+Version: 0.27.0
+Summary: General, database, and photometry utilities for the Roman SNPIT
 Author: Roman Supernova Project Infrastructure Team
 Maintainer-email: Roman SN PIT <raknop@lbl.gov>
 Project-URL: repository, https://github.com/Roman-Supernova-PIT
@@ -10,18 +10,22 @@ Classifier: Programming Language :: Python
 Requires-Python: >=3.11
 Description-Content-Type: text/x-rst
 License-File: LICENSE
-Requires-Dist: numpy
-Requires-Dist: scipy
-Requires-Dist: astropy
-Requires-Dist: photutils
-Requires-Dist: pandas
-Requires-Dist: h5py
-Requires-Dist: roman-datamodels
-Requires-Dist: snpit_utils
-Requires-Dist: towncrier
-Requires-Dist: setuptools_scm
-Requires-Dist: cruft
-Requires-Dist: coverage
+Requires-Dist: astropy<8.0.0,>=7.1.0
+Requires-Dist: fitsio<2.0.0,>=1.2.5
+Requires-Dist: galsim<3.0.0,>=2.7.2
+Requires-Dist: gwcs<0.27.0,>=0.26.0
+Requires-Dist: h5py<4.0.0,>=3.14.0
+Requires-Dist: numpy<3.0.0,>=2.2.6
+Requires-Dist: pandas<3.0.0,>=2.3.3
+Requires-Dist: photutils<3.0.0,>=2.3.0
+Requires-Dist: pycryptodome<4.0.0,>=3.23.0
+Requires-Dist: python-dateutil<3.0.0,>=2.9.0.post0
+Requires-Dist: pytz>=2025.2
+Requires-Dist: pyyaml<7.0.0,>=6.0.3
+Requires-Dist: rkwebutil<3.0.0,>=2.4.1
+Requires-Dist: roman-datamodels<0.28.0,>=0.27.0
+Requires-Dist: scipy<2.0.0,>=1.16.2
+Requires-Dist: simplejson<4.0.0,>=3.20.2
 Provides-Extra: test
 Requires-Dist: pytest; extra == "test"
 Requires-Dist: pytest-doctestplus; extra == "test"
@@ -29,6 +33,7 @@ Requires-Dist: pytest-cov; extra == "test"
 Requires-Dist: requests; extra == "test"
 Requires-Dist: tox; extra == "test"
 Requires-Dist: devpi_process; extra == "test"
+Requires-Dist: psycopg<4.0.0,>=3.2.9; extra == "test"
 Provides-Extra: docs
 Requires-Dist: sphinx; extra == "docs"
 Requires-Dist: sphinx-automodapi; extra == "docs"

roman_snpit_snappl-0.27.0/changes/103.snappl.rst

@@ -0,0 +1 @@
+Small changes for Nov 2025 run

roman_snpit_snappl-0.27.0/changes/105.docs.rst

@@ -0,0 +1 @@
+Write recipes in Usage docs for the nov2025 run

roman_snpit_snappl-0.27.0/changes/106.docs.rst

@@ -0,0 +1 @@
+Config file in docs for nov 2025

roman_snpit_snappl-0.27.0/changes/107.feature.rst

@@ -0,0 +1 @@
+Add position_angle to segmap and l2image database tables

roman_snpit_snappl-0.27.0/changes/109.bugfix.rst

@@ -0,0 +1 @@
+Fix issue #90 (spurious error messages when saving a provenance to database)

roman_snpit_snappl-0.27.0/changes/111.docs.rst

@@ -0,0 +1 @@
+Editing usage.rst

roman_snpit_snappl-0.27.0/changes/112.snappl.rst

@@ -0,0 +1 @@
+Fixed diaobject id bug in diaobject2024

roman_snpit_snappl-0.27.0/changes/116.docs.rst

@@ -0,0 +1 @@
+add some provenance tags to usage.rst for nov 2025 run

roman_snpit_snappl-0.27.0/changes/118.feature.rst

@@ -0,0 +1 @@
+Implement reread option in Config.get()

roman_snpit_snappl-0.27.0/changes/120.feature.rst

@@ -0,0 +1 @@
+Image refactor to allow loading of Image properties from database

roman_snpit_snappl-0.27.0/changes/122.feature.rst

@@ -0,0 +1 @@
+Add reading, writing, and database saving saving of 1d transient spectra

roman_snpit_snappl-0.27.0/changes/61.bugfix.rst

@@ -0,0 +1 @@
+Fix mjd getter for OpenUniverse2024FITSImage

roman_snpit_snappl-0.27.0/changes/62.snappl.rst

@@ -0,0 +1 @@
+Fixed an issue with the manual diaobj constructor where extra args were passed

roman_snpit_snappl-0.27.0/changes/63.snappl.rst

@@ -0,0 +1 @@
+Added the pointing, sca, mjd, and header properties to ManualFITSImages.

roman_snpit_snappl-0.27.0/changes/65.bugfix.rst

@@ -0,0 +1 @@
+self.zeropoint returned None because it was not set to self._get_zeropoint.

roman_snpit_snappl-0.27.0/changes/68.feature.rst

@@ -0,0 +1 @@
+Add GaussianPSF

roman_snpit_snappl-0.27.0/changes/72.snappl.rst

@@ -0,0 +1 @@
+Replaced input_wcs with image

roman_snpit_snappl-0.27.0/changes/73.feature.rst

@@ -0,0 +1 @@
+Add image_simulator.py ; redo image.py FITS classes ; fitsio reading/writing

roman_snpit_snappl-0.27.0/changes/74.bugfix.rst

@@ -0,0 +1 @@
+Fix an issue in ImageSimulator where pointsources were all 2× too bright.

roman_snpit_snappl-0.27.0/changes/79.snappl.rst

@@ -0,0 +1 @@
+Make the URL of the roman-desc-simdex configurable

roman_snpit_snappl-0.27.0/changes/81.snappl.rst

@@ -0,0 +1 @@
+Allowed for none paths and none data

roman_snpit_snappl-0.27.0/changes/82.snappl.rst

@@ -0,0 +1 @@
+Playing with dependencies

roman_snpit_snappl-0.27.0/changes/83.snappl.rst

@@ -0,0 +1 @@
+Added extremely basic static source sim to snappl.

roman_snpit_snappl-0.27.0/changes/84.feature.rst

@@ -0,0 +1,3 @@
+Incorporate functionality of snpit_utils and roman-snpit-db.
+
+Add the ability to get DiaObject from the Roman SNPIT internal database.

roman_snpit_snappl-0.27.0/changes/85.feature.rst

@@ -0,0 +1,2 @@
+Add database connection to l2image, go back to using collection in
+DiaObject, some other small changes.

roman_snpit_snappl-0.27.0/changes/86.snappl.rst

@@ -0,0 +1 @@
+Added lightcurve class.

roman_snpit_snappl-0.27.0/changes/87.docs.rst

@@ -0,0 +1 @@
+Add info about dbclient, Config, Provenance to usage.rst in documentation.

roman_snpit_snappl-0.27.0/changes/89.feature.rst

@@ -0,0 +1,2 @@
+DiaObject.save_object will now associate with existing objects when
+saving objects, and not save new ones if there already is one close enough.

roman_snpit_snappl-0.27.0/changes/91.feature.rst

@@ -0,0 +1,3 @@
+Allow loading and saving of diaobject positions from the database
+
+Also cleans up some database interface issues (race conditions, parameter mismatches) in db.py

roman_snpit_snappl-0.27.0/changes/92.docs.rst

@@ -0,0 +1,2 @@
+Documentation for Nov2025 test run; code for loading ou2024 images into
+database for that test run.

roman_snpit_snappl-0.27.0/changes/94.snappl.rst

@@ -0,0 +1 @@
+Rename l2image mjd_start field to mjd for backwards compatibility

roman_snpit_snappl-0.27.0/changes/97.snappl.rst

@@ -0,0 +1 @@
+LC ecsv writing bug fix

roman_snpit_snappl-0.27.0/changes/98.docs.rst

@@ -0,0 +1 @@
+Add lightcurve reading and writing to the nov 2025 primer in the usage docs

roman_snpit_snappl-0.27.0/changes/99.feature.rst

@@ -0,0 +1 @@
+Add storage and database tracking of segmentation maps

roman_snpit_snappl-0.27.0/docker/postgres/Dockerfile

@@ -0,0 +1,122 @@
+#
+# DOCKER_BUILDKIT=1 docker build --target postgres -t <image>:<label> .
+#
+# Rob, use:
+#   DOCKER_BUILDKIT=1 docker build --target postgres -t registry.nersc.gov/m4385/rknop/snpit-db-postgres:rknop-dev .
+#
+
+FROM debian:bookworm-20250630 AS base
+LABEL maintainer="Rob Knop <raknop@lbl.gov>"
+
+SHELL ["/bin/bash", "-c"]
+
+RUN apt-get update \
+  && DEBIAN_FRONTEND="nointeractive" apt-get -y upgrade \
+  && DEBIAN_FRONTEND="noninteractive" TZ="UTC" apt-get -y install -y --no-install-recommends \
+      postgresql-15 postgresql-client-15 pgtop libssl3 libreadline8 zlib1g netcat-openbsd \
+      libzstd1 liblz4-1 \
+      tmux emacs-nox less procps \
+  && apt-get clean \
+  && rm -rf /var/lib/apt/lists/*
+
+RUN mkdir -p /var/lib/postgresql/data \
+  && chmod 700 /var/lib/postgresql/data \
+  && chown postgres:postgres /var/lib/postgresql/data
+
+RUN cat /etc/locale.gen | perl -pe 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' > /etc/locale.gen.new \
+    && mv /etc/locale.gen.new /etc/locale.gen
+RUN locale-gen en_US.UTF-8
+ENV LANG=en_US.UTF-8
+ENV LANGUAGE=en_US:en
+ENV LC_ALL=en_US.UTF-8
+
+
+RUN echo "host all all 0.0.0.0/0 md5" >> /etc/postgresql/15/main/pg_hba.conf
+COPY postgresql.conf /etc/postgresql/15/main/postgresql.conf
+ENV POSTGRES_DATA_DIR=/var/lib/postgresql/data
+
+ENV LESS=-XLRi
+
+# ======================================================================
+FROM base AS build
+
+RUN mkdir /build
+WORKDIR /build
+
+RUN apt-get update \
+  && DEBIAN_FRONTEND="noninteractive" TZ="UTC" apt-get -y install -y --no-install-recommends \
+      make git gcc libssl-dev libreadline-dev zlib1g-dev libzstd-dev liblz4-dev curl \
+      postgresql-server-dev-15 ca-certificates pkg-config
+
+RUN git clone https://github.com/segasai/q3c.git
+RUN cd q3c \
+  && make \
+  && make install
+
+RUN curl -L https://github.com/ossc-db/pg_hint_plan/archive/refs/tags/REL15_1_5_1.tar.gz \
+         -o pg_hint_plan-REL15_1_5_1.tar.gz \
+  && tar -xpf pg_hint_plan-REL15_1_5_1.tar.gz \
+  && cd pg_hint_plan-REL15_1_5_1 \
+  && make \
+  && make install
+
+
+# ENV CARGO_HOME=/opt/rust
+# ENV RUSTUP_HOME=/opt/rust
+# ENV PATH="${PATH}:/opt/rust/bin"
+# RUN mkdir -p /opt/rust \
+#   && chown postgres -R /opt/rust
+#
+# ENV PG_PARQUET_VERSION=0.4.0
+# ENV PGRX_VERSION=0.14.1
+# RUN curl https://sh.rustup.rs -sSf | sh -s -- --no-modify-path --profile minimal --default-toolchain stable -y
+# RUN cargo install --locked cargo-pgrx --version ${PGRX_VERSION}
+# RUN cargo pgrx init --pg15 $(which pg_config)
+#
+# RUN mkdir -p ~/.pgrx/data-15 \
+#   && echo "shared_preload_libraries = 'pg_parquet'" >> ~/.pgrx/data-15/postgresql.conf
+# RUN cd /opt/rust \
+#   && curl -LOJ https://github.com/CrunchyData/pg_parquet/archive/refs/tags/v${PG_PARQUET_VERSION}.tar.gz \
+#   && tar -xvzf pg_parquet-${PG_PARQUET_VERSION}.tar.gz \
+#   && cd pg_parquet-${PG_PARQUET_VERSION} \
+#   && cargo pgrx install --release
+
+RUN chown postgres -R /usr/share/postgresql/ \
+  && chown postgres -R /usr/lib/postgresql
+
+
+# ======================================================================
+FROM base AS postgres
+
+COPY --from=build /usr/lib/postgresql/15/lib/q3c.so /usr/lib/postgresql/15/lib/q3c.so
+COPY --from=build /usr/lib/postgresql/15/lib/bitcode/q3c /usr/share/postgresql/lib/bitcode/q3c
+COPY --from=build /usr/lib/postgresql/15/lib/bitcode/q3c.index.bc /usr/share/postgresql/lib/bitcode/q3c.index.bc
+COPY --from=build /usr/share/postgresql/15/extension/q3c* /usr/share/postgresql/15/extension/
+COPY --from=build /usr/lib/postgresql/15/lib/pg_hint_plan.so usr/lib/postgresql/15/lib/pg_hint_plan.so
+COPY --from=build /usr/lib/postgresql/15/lib/bitcode/pg_hint_plan /usr/share/postgresql/lib/bitcode/pg_hint_plan
+COPY --from=build /usr/lib/postgresql/15/lib/bitcode/pg_hint_plan.index.bc /usr/share/postgresql/lib/bitcode/pg_hint_plan.index.bc
+COPY --from=build /usr/share/postgresql/15/extension/pg_hint_plan* /usr/share/postgresql/15/extension/
+# COPY --from=build /usr/lib/postgresql/15/lib/pg_parquet.so /usr/lib/postgresql/15/lib/
+# COPY --from=build /usr/share/postgresql/15/extension/pg_parquet* /usr/share/postgresql/15/extension/
+
+# Make sure this matches what is in the config file (created just above)
+# (There is some futzing about here to make sure the right permissions are
+# on the file regardless of the permissions as it exists on the machine
+# where the docker image is built.  I don't know why a simple ADD
+# followed by CHMOD didn't work.)
+COPY run_postgres.sh /tmp/run_postgres.sh
+RUN cp /tmp/run_postgres.sh /run_postgres.sh && chmod 755 /run_postgres.sh
+
+USER postgres
+CMD ["bash", "/run_postgres.sh"]
+
+# ======================================================================
+
+FROM base AS pgdump
+
+RUN mkdir=/home/pgdump
+ENV HOME=/home/pgdump
+ADD run_pgdump.sh /home/pgdump/run_pgdump.sh
+
+USER postgres
+ENTRYPOINT [ "/bin/bash", "/home/pgdump/run_pgdump.sh" ]