rubin-sim 2.6.1a4__tar.gz → 2.6.1.dev1__tar.gz

{rubin_sim-2.6.1a4 → rubin_sim-2.6.1.dev1}/.github/workflows/ruff.yaml

@@ -1,7 +1,12 @@
-name: Lint
+name: Ruff and iSort
 on:
+  # Trigger the workflow on push (to main) or pull request
   push:
+    branches:
+      - main
   pull_request:
+    branches:
+      - main
   workflow_dispatch:
 
 jobs:
@@ -16,13 +21,4 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - uses: astral-sh/ruff-action@v3
-  black:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - uses: psf/black@stable
-        with:
-          options: "--check --verbose"
-          version: "~= 26.1"
-
+      - uses: astral-sh/ruff-action@v3

{rubin_sim-2.6.1a4/rubin_sim.egg-info → rubin_sim-2.6.1.dev1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rubin-sim
-Version: 2.6.1a4
+Version: 2.6.1.dev1
 Summary: Scheduler, survey strategy analysis, and other simulation tools for Rubin Observatory.
 License: GPL
 Project-URL: documentation, https://rubin-sim.lsst.io
@@ -47,11 +47,6 @@ Requires-Dist: documenteer[guide]; extra == "dev"
 Provides-Extra: extras
 Requires-Dist: george; extra == "extras"
 Requires-Dist: pyoorb; extra == "extras"
-Provides-Extra: vseqarchive
-Requires-Dist: click; extra == "vseqarchive"
-Requires-Dist: psycopg2; extra == "vseqarchive"
-Requires-Dist: botocore; extra == "vseqarchive"
-Requires-Dist: boto3; extra == "vseqarchive"
 Dynamic: license-file
 
 # rubin_sim

rubin_sim-2.6.1.dev1/batch/compile_prenight_metadata_cache.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+#SBATCH --account=rubin:developers      # Account name
+#SBATCH --job-name=auxtel_prenight_daily   # Job name
+#SBATCH --output=/sdf/data/rubin/shared/scheduler/prenight/sbatch/compile_prenight_metadata_cache.out # Output file (stdout)
+#SBATCH --error=/sdf/data/rubin/shared/scheduler/prenight/sbatch/compile_prenight_metadata_cache.err  # Error file (stderr)
+#SBATCH --partition=milano              # Partition (queue) names
+#SBATCH --nodes=1                       # Number of nodes
+#SBATCH --ntasks=1                      # Number of tasks run in parallel
+#SBATCH --cpus-per-task=1               # Number of CPUs per task
+#SBATCH --mem=4G                       # Requested memory
+#SBATCH --time=1:00:00                 # Wall time (hh:mm:ss)
+
+echo "******** START of compile_prenight_metadata_cache.sh **********"
+
+# Source global definitions
+if [ -f /etc/bashrc ]; then
+	. /etc/bashrc
+fi
+
+# SLAC S3DF - source all files under ~/.profile.d
+if [[ -e ~/.profile.d && -n "$(ls -A ~/.profile.d/)" ]]; then
+  source <(cat $(find -L  ~/.profile.d -name '*.conf'))
+fi
+
+source /sdf/group/rubin/sw/w_latest/loadLSST.sh
+conda activate /sdf/data/rubin/shared/scheduler/envs/prenight
+export AWS_PROFILE=prenight
+WORK_DIR=$(date '+/sdf/data/rubin/shared/scheduler/prenight/work/compile_prenight_metadata_cache/%Y-%m-%dT%H%M%S' --utc)
+echo "Working in $WORK_DIR"
+mkdir ${WORK_DIR}
+cd ${WORK_DIR}
+printenv > env.out
+compile_sim_archive_metadata_resource --append
+echo "******* END of compile_prenight_metadata_cache.sh *********"

rubin_sim-2.6.1.dev1/batch/run_auxtel_prenight_sims.sh

@@ -0,0 +1,163 @@
+#!/usr/bin/env bash
+#SBATCH --account=rubin:developers      # Account name
+#SBATCH --job-name=auxtel_prenight_daily   # Job name
+#SBATCH --output=/sdf/data/rubin/shared/scheduler/prenight/sbatch/run_prenight_sims_%A_%a.out # Output file (stdout)
+#SBATCH --error=/sdf/data/rubin/shared/scheduler/prenight/sbatch/run_prenight_sims_%A_%a.out  # Error file (stderr)
+#SBATCH --partition=milano              # Partition (queue) names
+#SBATCH --nodes=1                       # Number of nodes
+#SBATCH --ntasks=1                      # Number of tasks run in parallel
+#SBATCH --cpus-per-task=1               # Number of CPUs per task
+#SBATCH --mem=8G                       # Requested memory
+#SBATCH --time=1:30:00                 # Wall time (hh:mm:ss)
+
+echo "******** START of run_prenight_sims.sh **********"
+
+# Source global definitions
+if [ -f /etc/bashrc ]; then
+	. /etc/bashrc
+fi
+
+# SLAC S3DF - source all files under ~/.profile.d
+if [[ -e ~/.profile.d && -n "$(ls -A ~/.profile.d/)" ]]; then
+  source <(cat $(find -L  ~/.profile.d -name '*.conf'))
+fi
+
+date --iso=s
+LATEST_TAGGED_STACK=$(
+    find /cvmfs/sw.lsst.eu/almalinux-x86_64/lsst_distrib \
+        -maxdepth 1 \
+        -regex '.*/v[0-9]+\.[0-9]+\.[0-9]+' \
+        -printf "%f\n" |
+    sort -V |
+    tail -1
+)
+source /cvmfs/sw.lsst.eu/almalinux-x86_64/lsst_distrib/${LATEST_TAGGED_STACK}/loadLSST-ext.sh
+
+set -o xtrace
+
+export AWS_PROFILE=prenight
+WORK_DIR=$(date '+/sdf/data/rubin/shared/scheduler/prenight/work/run_prenight_sims/%Y-%m-%dT%H%M%S' --utc)
+echo "Working in $WORK_DIR"
+mkdir ${WORK_DIR}
+cd ${WORK_DIR}
+
+# Install required python packages
+PACKAGE_DIR=$(readlink -f ${WORK_DIR}/packages)
+mkdir ${PACKAGE_DIR}
+export PYTHONPATH=${PACKAGE_DIR}:${PYTHONPATH}
+export PATH=${PACKAGE_DIR}/bin:${PATH}
+
+if false ; then
+  # Get latest tagged versions of everything
+  RUBIN_SCHEDULER_REFERENCE=$(curl -s https://api.github.com/repos/lsst/rubin_scheduler/tags | jq -r '.[].name' | egrep '^v[0-9]+.[0-9]+.[0-9]+$' | sort -V | tail -1)
+  RUBIN_SIM_REFERENCE=$(curl -s https://api.github.com/repos/lsst/rubin_sim/tags | jq -r '.[].name' | egrep '^v[0-9]+.[0-9]+.[0-9]+$' | sort -V | tail -1)
+  SCHEDVIEW_REFERENCE=$(curl -s https://api.github.com/repos/lsst/schedview/tags | jq -r '.[].name' | egrep '^v[0-9]+.[0-9]+.[0-9]+$' | sort -V | tail -1)
+  TS_FBS_UTILS_REFERENCE=$(curl -s https://api.github.com/repos/lsst-ts/ts_fbs_utils/tags | jq -r '.[].name' | egrep '^v[0-9]+.[0-9]+.[0-9]+$' | sort -V | tail -1)
+  SIMS_SV_SURVEY_REFERENCE=$(curl -s https://api.github.com/repos/lsst-sims/sims_sv_survey/tags | jq -r '.[].name' | egrep '^v[0-9]+.[0-9]+.[0-9]+$' | sort -V | tail -1)
+  RUBIN_NIGHTS_REFERENCE=$(curl -s https://api.github.com/repos/lsst-sims/rubin_nights/tags | jq -r '.[].name' | egrep '^v[0-9]+.[0-9]+.[0-9]+$' | sort -V | tail -1)
+else
+  # alternately, set specific versions
+  RUBIN_SCHEDULER_REFERENCE="v3.18.1"
+  RUBIN_SIM_REFERENCE="tickets/SP-2709"
+  SCHEDVIEW_REFERENCE="tickets/SP-2167"
+  TS_FBS_UTILS_REFERENCE=$(curl -s https://api.github.com/repos/lsst-ts/ts_fbs_utils/tags | jq -r '.[].name' | egrep '^v[0-9]+.[0-9]+.[0-9]+$' | sort -V | tail -1)
+  LSST_SURVEY_SIM_REFERENCE="tickets/SP-2709a"
+  RUBIN_NIGHTS_REFERENCE="v0.7.0"
+fi
+
+pip install --no-deps --target=${PACKAGE_DIR} \
+  git+https://github.com/lsst/rubin_scheduler.git@${RUBIN_SCHEDULER_REFERENCE} \
+  git+https://github.com/lsst/rubin_sim.git@${RUBIN_SIM_REFERENCE} \
+  git+https://github.com/lsst/schedview.git@${SCHEDVIEW_REFERENCE} \
+  git+https://github.com/lsst-ts/ts_fbs_utils.git@${TS_FBS_UTILS_REFERENCE} \
+  git+https://github.com/lsst-sims/lsst_survey_sim.git@${LSST_SURVEY_SIM_REFERENCE} \
+  git+https://github.com/lsst-sims/rubin_nights.git@${RUBIN_NIGHTS_REFERENCE} \
+  lsst-resources
+
+if false ; then
+  # Get the scheduler version from the EFD and install it.
+  # We have to do this after the others, because we want
+  # the version of obs_version_at_time supplied by the
+  # version of schedview we specify.
+  # Go ahead and install any missing dependencies as well.
+  RUBIN_SCHEDULER_REFERENCE=v$(obs_version_at_time rubin_scheduler)
+  echo "Using rubin_scheduler $RUBIN_SCHEDULER_REFERENCE"
+  pip install --ignore-installed --no-deps --upgrade --target=${PACKAGE_DIR} git+https://github.com/lsst/rubin_scheduler.git@${RUBIN_SCHEDULER_REFERENCE}
+fi
+
+# Get the scheduler configuration script
+# It lives in ts_config_scheduler
+TS_CONFIG_SCHEDULER_REFERENCE="develop"
+SCHED_CONFIG_FNAME="ts_config_scheduler/Scheduler/feature_scheduler/auxtel/fbs_spec_flex_survey.py"
+echo "Using ts_config_scheduler ${SCHED_CONFIG_FNAME} from ${TS_CONFIG_SCHEDULER_REFERENCE}"
+git clone --depth 1 https://github.com/lsst-ts/ts_config_scheduler
+cd ts_config_scheduler
+git fetch --depth 1 origin "${TS_CONFIG_SCHEDULER_REFERENCE}"
+git checkout FETCH_HEAD
+cd ${WORK_DIR}
+
+export DAYOBS="$(date -u --date='-12 hours' +'%Y%m%d')"
+export NEXT_DAYOBS="$(date -u --date='+12 hours' +'%Y%m%d')"
+export LAST_DAYOBS="$(date -u --date='+36 hours' +'%Y%m%d')"
+export DAYOBS_SIMULATED="$DAYOBS $NEXT_DAYOBS $LAST_DAYOBS"
+export LASTNIGHTISO="$(date --date='-36 hours' -u +'%F')"
+
+export ARCHIVE="s3://rubin:rubin-scheduler-prenight/opsim/vseq/"
+export VSARCHIVE_PGDATABASE="opsim_log"
+export VSARCHIVE_PGHOST="usdf-maf-visit-seq-archive-tx.sdf.slac.stanford.edu"
+export VSARCHIVE_PGUSER="writer"
+export VSARCHIVE_PGSCHEMA="vsmd"
+
+# Get an empty set of completed visits so we have something
+# to pass make_lsst_scheduler
+fetch_lsst_visits 20000101 completed_visits.db ~/.lsst/usdf_access_token
+
+echo "Creating scheduler pickle"
+date --iso=s
+make_lsst_scheduler scheduler.p --opsim completed_visits.db --config-script ${SCHED_CONFIG_FNAME}
+
+echo "Creating model observatory"
+date --iso=s
+ideal_model_observatory scheduler.p observatory.p
+
+# make dir for output
+OPSIM_RESULT_DIR=${WORK_DIR}/opsim_results
+mkdir ${OPSIM_RESULT_DIR}
+
+echo "Running nominal LSST simulation"
+OPSIMRUN="prenight_nominal_$(date --iso=s)"
+LABEL="Nominal start and overhead, ideal conditions, run at $(date --iso=s)"
+date --iso=s
+run_lsst_sim scheduler.p observatory.p "" ${DAYOBS} 3 "${OPSIMRUN}" \
+  --keep_rewards --label "${LABEL}" \
+  --delay 0 --anom_overhead_scale 0 \
+  --results ${OPSIM_RESULT_DIR}
+
+
+echo "Creating entry in metadatdata database"
+date --iso=s
+SIM_UUID=$(vseqarchive record-visitseq-metadata \
+    simulations \
+    ${OPSIM_RESULT_DIR}/opsim.db \
+    "${LABEL}" \
+    --telescope auxtel \
+    --first_day_obs ${DAYOBS} \
+    --last_day_obs ${LAST_DAYOBS}
+    )
+
+vseqarchive update-visitseq-metadata ${SIM_UUID} scheduler_version "${RUBIN_SCHEDULER_REFERENCE}"
+vseqarchive archive-file ${SIM_UUID} ${OPSIM_RESULT_DIR}/opsim.db visits --archive-base ${ARCHIVE}
+vseqarchive archive-file ${SIM_UUID} ${OPSIM_RESULT_DIR}/rewards.h5 rewards --archive-base ${ARCHIVE}
+vseqarchive tag ${SIM_UUID} prenight ideal nominal
+
+CONDA_HASH=$(vseqarchive record-conda-env)
+vseqarchive update-visitseq-metadata ${SIM_UUID} conda_env_sha256 ${CONDA_HASH}
+
+vseqarchive get-file ${SIM_UUID} visits visits.h5
+vseqarchive add-nightly-stats ${SIM_UUID} visits.h5 azimuth altitude
+
+rm visits.h5 ${OPSIM_RESULT_DIR}/opsim.db ${OPSIM_RESULT_DIR}/rewards.h5 ${OPSIM_RESULT_DIR}/obs_stats.txt ${OPSIM_RESULT_DIR}/observatory.p ${OPSIM_RESULT_DIR}/scheduler.p ${OPSIM_RESULT_DIR}/sim_metadata.yaml
+
+for DAYOBS_TO_INDEX in ${DAYOBS_SIMULATED}; do
+  vseqarchive make-prenight-index ${DAYOBS_TO_INDEX} auxtel
+done

rubin_sim-2.6.1.dev1/batch/run_prenight_sims.sh

@@ -0,0 +1,239 @@
+#!/usr/bin/env bash
+#SBATCH --account=rubin:developers      # Account name
+#SBATCH --job-name=prenight_simonyi_daily   # Job name
+#SBATCH --output=/sdf/data/rubin/shared/scheduler/prenight/sbatch/run_prenight_sims_%A_%a.out # Output file (stdout)
+#SBATCH --error=/sdf/data/rubin/shared/scheduler/prenight/sbatch/run_prenight_sims_%A_%a.out  # Error file (stderr)
+#SBATCH --partition=milano              # Partition (queue) names
+#SBATCH --nodes=1                       # Number of nodes
+#SBATCH --ntasks=1                      # Number of tasks run in parallel
+#SBATCH --cpus-per-task=1               # Number of CPUs per task
+#SBATCH --mem=8G                       # Requested memory
+#SBATCH --time=1:30:00                 # Wall time (hh:mm:ss)
+
+echo "******** START of run_prenight_sims.sh **********"
+
+# Source global definitions
+if [ -f /etc/bashrc ]; then
+	. /etc/bashrc
+fi
+
+# SLAC S3DF - source all files under ~/.profile.d
+if [[ -e ~/.profile.d && -n "$(ls -A ~/.profile.d/)" ]]; then
+  source <(cat $(find -L  ~/.profile.d -name '*.conf'))
+fi
+
+date --iso=s
+LATEST_TAGGED_STACK=$(
+    find /cvmfs/sw.lsst.eu/almalinux-x86_64/lsst_distrib \
+        -maxdepth 1 \
+        -regex '.*/v[0-9]+\.[0-9]+\.[0-9]+' \
+        -printf "%f\n" |
+    sort -V |
+    tail -1
+)
+source /cvmfs/sw.lsst.eu/almalinux-x86_64/lsst_distrib/${LATEST_TAGGED_STACK}/loadLSST-ext.sh
+
+set -o xtrace
+
+export AWS_PROFILE=prenight
+WORK_DIR=$(date '+/sdf/data/rubin/shared/scheduler/prenight/work/run_prenight_sims/%Y-%m-%dT%H%M%S' --utc)
+echo "Working in $WORK_DIR"
+mkdir ${WORK_DIR}
+cd ${WORK_DIR}
+
+# Install required python packages
+PACKAGE_DIR=$(readlink -f ${WORK_DIR}/packages)
+mkdir ${PACKAGE_DIR}
+export PYTHONPATH=${PACKAGE_DIR}:${PYTHONPATH}
+export PATH=${PACKAGE_DIR}/bin:${PATH}
+
+if false ; then
+  # Get latest tagged versions of everything
+  RUBIN_SCHEDULER_REFERENCE=$(curl -s https://api.github.com/repos/lsst/rubin_scheduler/tags | jq -r '.[].name' | egrep '^v[0-9]+.[0-9]+.[0-9]+$' | sort -V | tail -1)
+  RUBIN_SIM_REFERENCE=$(curl -s https://api.github.com/repos/lsst/rubin_sim/tags | jq -r '.[].name' | egrep '^v[0-9]+.[0-9]+.[0-9]+$' | sort -V | tail -1)
+  SCHEDVIEW_REFERENCE=$(curl -s https://api.github.com/repos/lsst/schedview/tags | jq -r '.[].name' | egrep '^v[0-9]+.[0-9]+.[0-9]+$' | sort -V | tail -1)
+  TS_FBS_UTILS_REFERENCE=$(curl -s https://api.github.com/repos/lsst-ts/ts_fbs_utils/tags | jq -r '.[].name' | egrep '^v[0-9]+.[0-9]+.[0-9]+$' | sort -V | tail -1)
+  LSST_SURVEY_SIM_REFERENCE=$(curl -s https://api.github.com/repos/lsst-sims/lsst_survey_sim/tags | jq -r '.[].name' | egrep '^v[0-9]+.[0-9]+.[0-9]+$' | sort -V | tail -1)
+  RUBIN_NIGHTS_REFERENCE=$(curl -s https://api.github.com/repos/lsst-sims/rubin_nights/tags | jq -r '.[].name' | egrep '^v[0-9]+.[0-9]+.[0-9]+$' | sort -V | tail -1)
+else
+  # alternately, set specific versions
+  RUBIN_SCHEDULER_REFERENCE="v3.19.0"
+  RUBIN_SIM_REFERENCE="tickets/SP-2709"
+  SCHEDVIEW_REFERENCE="tickets/SP-2167"
+  TS_FBS_UTILS_REFERENCE="v0.19.0"
+  LSST_SURVEY_SIM_REFERENCE="v0.2.0"
+  RUBIN_NIGHTS_REFERENCE="v0.7.0"
+fi
+
+pip install --no-deps --target=${PACKAGE_DIR} \
+  git+https://github.com/lsst/rubin_scheduler.git@${RUBIN_SCHEDULER_REFERENCE} \
+  git+https://github.com/lsst/rubin_sim.git@${RUBIN_SIM_REFERENCE} \
+  git+https://github.com/lsst/schedview.git@${SCHEDVIEW_REFERENCE} \
+  git+https://github.com/lsst-ts/ts_fbs_utils.git@${TS_FBS_UTILS_REFERENCE} \
+  git+https://github.com/lsst-sims/lsst_survey_sim.git@${LSST_SURVEY_SIM_REFERENCE} \
+  git+https://github.com/lsst-sims/rubin_nights.git@${RUBIN_NIGHTS_REFERENCE} \
+  lsst-resources
+
+if false ; then
+  # Get the scheduler version from the EFD and install it.
+  # We have to do this after the others, because we want
+  # the version of obs_version_at_time supplied by the
+  # version of schedview we specify.
+  RUBIN_SCHEDULER_REFERENCE=v$(obs_version_at_time rubin_scheduler)
+  echo "Using rubin_scheduler $RUBIN_SCHEDULER_REFERENCE"
+  pip install --ignore-installed --no-deps --upgrade --target=${PACKAGE_DIR} git+https://github.com/lsst/rubin_scheduler.git@${RUBIN_SCHEDULER_REFERENCE}
+fi
+
+# Get the scheduler configuration script
+# It lives in ts_config_scheduler
+TS_CONFIG_SCHEDULER_REFERENCE="develop"
+SCHED_CONFIG_FNAME="ts_config_scheduler/Scheduler/feature_scheduler/maintel/fbs_config_lsst_survey.py"
+echo "Using ts_config_scheduler ${SCHED_CONFIG_FNAME} from ${TS_CONFIG_SCHEDULER_REFERENCE}"
+git clone --depth 1 https://github.com/lsst-ts/ts_config_scheduler
+cd ts_config_scheduler
+git fetch --depth 1 origin "${TS_CONFIG_SCHEDULER_REFERENCE}"
+git checkout FETCH_HEAD
+cd ${WORK_DIR}
+
+export DAYOBS="$(date -u --date='-12 hours' +'%Y%m%d')"
+export NEXT_DAYOBS="$(date -u --date='+12 hours' +'%Y%m%d')"
+export LAST_DAYOBS="$(date -u --date='+36 hours' +'%Y%m%d')"
+export DAYOBS_SIMULATED="$DAYOBS $NEXT_DAYOBS $LAST_DAYOBS"
+export LASTNIGHTISO="$(date --date='-36 hours' -u +'%F')"
+
+export ARCHIVE="s3://rubin:rubin-scheduler-prenight/opsim/vseq/"
+export VSARCHIVE_PGDATABASE="opsim_log"
+export VSARCHIVE_PGHOST="usdf-maf-visit-seq-archive-tx.sdf.slac.stanford.edu"
+export VSARCHIVE_PGUSER="writer"
+export VSARCHIVE_PGSCHEMA="vsmd"
+
+echo "Fetching completed visits"
+date --iso=s
+fetch_lsst_visits ${DAYOBS} completed_visits.db ~/.lsst/usdf_access_token
+
+# Recording hash of fetched visits
+COMPLETED=$(vseqarchive record-visitseq-metadata \
+    completed \
+    completed_visits.db \
+    "Consdb query through ${LASTNIGHTISO}" \
+    --first_day_obs 20250620 \
+    --last_day_obs ${LASTNIGHTISO})
+
+echo "Creating scheduler pickle"
+date --iso=s
+make_lsst_scheduler scheduler.p --opsim completed_visits.db --config-script ${SCHED_CONFIG_FNAME}
+
+echo "Creating model observatory"
+date --iso=s
+make_model_observatory observatory.p
+
+echo "Creating the band scheduler"
+date --iso=s
+make_band_scheduler band_scheduler.p
+
+# make dir for output
+OPSIM_RESULT_DIR=${WORK_DIR}/opsim_results
+mkdir ${OPSIM_RESULT_DIR}
+
+echo "Running nominal LSST simulation"
+OPSIMRUN="prenight_nominal_$(date --iso=s)"
+LABEL="Nominal start and overhead, ideal conditions, run at $(date --iso=s)"
+date --iso=s
+run_lsst_sim scheduler.p observatory.p "" ${DAYOBS} 3 "${OPSIMRUN}" \
+  --keep_rewards --label "${LABEL}" \
+  --delay 0 --anom_overhead_scale 0 \
+  --results ${OPSIM_RESULT_DIR}
+
+echo "Creating entry in metadatdata database"
+date --iso=s
+SIM_UUID=$(vseqarchive record-visitseq-metadata \
+    simulations \
+    ${OPSIM_RESULT_DIR}/opsim.db \
+    "${LABEL}" \
+    --first_day_obs ${DAYOBS} \
+    --last_day_obs ${LAST_DAYOBS}
+    )
+vseqarchive update-visitseq-metadata ${SIM_UUID} parent_visitseq_uuid ${COMPLETED}
+vseqarchive update-visitseq-metadata ${SIM_UUID} parent_last_day_obs ${LASTNIGHTISO}
+
+vseqarchive update-visitseq-metadata ${SIM_UUID} scheduler_version "${RUBIN_SCHEDULER_REFERENCE}"
+vseqarchive archive-file ${SIM_UUID} ${OPSIM_RESULT_DIR}/opsim.db visits --archive-base ${ARCHIVE}
+vseqarchive archive-file ${SIM_UUID} ${OPSIM_RESULT_DIR}/rewards.h5 rewards --archive-base ${ARCHIVE}
+vseqarchive tag ${SIM_UUID} prenight ideal nominal
+
+CONDA_HASH=$(vseqarchive record-conda-env)
+vseqarchive update-visitseq-metadata ${SIM_UUID} conda_env_sha256 ${CONDA_HASH}
+
+vseqarchive get-file ${SIM_UUID} visits visits.h5
+vseqarchive add-nightly-stats ${SIM_UUID} visits.h5 azimuth altitude
+
+rm visits.h5 ${OPSIM_RESULT_DIR}/opsim.db ${OPSIM_RESULT_DIR}/rewards.h5 ${OPSIM_RESULT_DIR}/obs_stats.txt ${OPSIM_RESULT_DIR}/observatory.p ${OPSIM_RESULT_DIR}/scheduler.p ${OPSIM_RESULT_DIR}/sim_metadata.yaml
+
+for DELAY in 60 240 ; do
+  echo "Running SV simulation delayed ${DELAY}"
+  OPSIMRUN="prenight_delay${DELAY}_$(date --iso=s)"
+  LABEL="Start time delayed by ${DELAY} minutes, nominal slew and visit overhead, ideal conditions, run at $(date --iso=s)"
+  date --iso=s
+  run_lsst_sim scheduler.p observatory.p "" ${DAYOBS} 3 "${OPSIMRUN}" \
+    --keep_rewards --label "${LABEL}" \
+    --delay ${DELAY} --anom_overhead_scale 0 \
+    --results ${OPSIM_RESULT_DIR}
+
+  SIM_UUID=$(vseqarchive record-visitseq-metadata \
+      simulations \
+      ${OPSIM_RESULT_DIR}/opsim.db \
+      "${LABEL}" \
+      --first_day_obs ${DAYOBS} \
+      --last_day_obs ${LAST_DAYOBS}
+      )
+  vseqarchive update-visitseq-metadata ${SIM_UUID} parent_visitseq_uuid ${COMPLETED}
+  vseqarchive update-visitseq-metadata ${SIM_UUID} parent_last_day_obs ${LASTNIGHTISO}
+  vseqarchive update-visitseq-metadata ${SIM_UUID} scheduler_version "${RUBIN_SCHEDULER_REFERENCE}"
+  vseqarchive archive-file ${SIM_UUID} ${OPSIM_RESULT_DIR}/opsim.db visits --archive-base ${ARCHIVE}
+  vseqarchive archive-file ${SIM_UUID} ${OPSIM_RESULT_DIR}/rewards.h5 rewards --archive-base ${ARCHIVE}
+  vseqarchive tag ${SIM_UUID} prenight ideal delay_${DELAY}
+  vseqarchive update-visitseq-metadata ${SIM_UUID} conda_env_sha256 ${CONDA_HASH}
+  vseqarchive get-file ${SIM_UUID} visits visits.h5
+  vseqarchive add-nightly-stats ${SIM_UUID} visits.h5 azimuth altitude
+
+  rm visits.h5 ${OPSIM_RESULT_DIR}/opsim.db ${OPSIM_RESULT_DIR}/rewards.h5 ${OPSIM_RESULT_DIR}/obs_stats.txt ${OPSIM_RESULT_DIR}/observatory.p ${OPSIM_RESULT_DIR}/scheduler.p ${OPSIM_RESULT_DIR}/sim_metadata.yaml
+done
+
+ANOM_SCALE="0.1"
+for ANOM_SEED in 101 102 ; do
+  echo "Running SV simulation with anomalous overhead seed ${ANOM_SEED}"
+  OPSIMRUN="prenight_anom${ANOM_SEED}_$(date --iso=s)"
+  LABEL="Anomalous overhead (${ANOM_SEED}, ${ANOM_SCALE}), nominal start, ideal conditions, run at $(date --iso=s)"
+  date --iso=s
+  run_lsst_sim scheduler.p observatory.p "" ${DAYOBS} 3 "${OPSIMRUN}" \
+    --keep_rewards --label "${LABEL}" \
+    --delay 0 \
+    --anom_overhead_scale ${ANOM_SCALE} \
+    --anom_overhead_seed ${ANOM_SEED} \
+    --results ${OPSIM_RESULT_DIR}
+
+  SIM_UUID=$(vseqarchive record-visitseq-metadata \
+      simulations \
+      ${OPSIM_RESULT_DIR}/opsim.db \
+      "${LABEL}" \
+      --first_day_obs ${DAYOBS} \
+      --last_day_obs ${LAST_DAYOBS}
+      )
+  vseqarchive update-visitseq-metadata ${SIM_UUID} parent_visitseq_uuid ${COMPLETED}
+  vseqarchive update-visitseq-metadata ${SIM_UUID} parent_last_day_obs ${LASTNIGHTISO}
+  vseqarchive update-visitseq-metadata ${SIM_UUID} scheduler_version "${RUBIN_SCHEDULER_REFERENCE}"
+  vseqarchive archive-file ${SIM_UUID} ${OPSIM_RESULT_DIR}/opsim.db visits --archive-base ${ARCHIVE}
+  vseqarchive archive-file ${SIM_UUID} ${OPSIM_RESULT_DIR}/rewards.h5 rewards --archive-base ${ARCHIVE}
+  vseqarchive tag ${SIM_UUID} prenight ideal anomalous_overhead
+  vseqarchive update-visitseq-metadata ${SIM_UUID} conda_env_sha256 ${CONDA_HASH}
+  vseqarchive get-file ${SIM_UUID} visits visits.h5
+  vseqarchive add-nightly-stats ${SIM_UUID} visits.h5 azimuth altitude
+
+  rm visits.h5 ${OPSIM_RESULT_DIR}/opsim.db ${OPSIM_RESULT_DIR}/rewards.h5 ${OPSIM_RESULT_DIR}/obs_stats.txt ${OPSIM_RESULT_DIR}/observatory.p ${OPSIM_RESULT_DIR}/scheduler.p ${OPSIM_RESULT_DIR}/sim_metadata.yaml
+done
+
+rm observatory.p scheduler.p
+
+for DAYOBS_TO_INDEX in ${DAYOBS_SIMULATED}; do
+  vseqarchive make-prenight-index ${DAYOBS_TO_INDEX} simonyi
+done

{rubin_sim-2.6.1a4 → rubin_sim-2.6.1.dev1}/docs/data-download.rst

@@ -52,7 +52,7 @@ to the directories originating from `rubin_scheduler <https://rubin-scheduler.ls
 * sim_baseline (containing the current baseline simulation output)
 * skybrightness (containing information needed for the skybrightness module)
 * throughputs (current baseline throughput information)
-* tests (containing data for unit tests)
+* test (containing data for unit tests)
 
 
 Note that the data will only be downloaded for the directories which do

rubin_sim-2.6.1.dev1/docs/prenight.rst

@@ -0,0 +1,117 @@
+.. py:currentmodule:: rubin_sim.data
+
+.. _prenight:
+
+============================================================
+Running pre-night simulations and adding them to the archive
+============================================================
+
+Introduction
+============
+
+Prenight simulation tools generate simulations of a few nights of observing, and add them to an archive.
+These simulations can then be seen by `schedview`'s reporting tools, and used to generate pre-night brifing reports.
+Normally, these simulation are run by submitting the `batch/run_prenight_sims.sh` script as a batch job, either by hand or using a cron job.
+For more fine-grained control, the simulations and be run and added to the archive using lower-level tools,
+
+Running a standard set of pre-night simulations
+===============================================
+
+The standard set of pre-night simulatios can be run by calling the `batch/run_prenight_sims.sh` shell (for SV) or `batch/run_auxtel_prenight_sims.sh` (for auxtel) scripts.
+
+If `RUBIN_SIM_DIR` is the `rubin_sim` root director, it can be called thus::
+
+    ${RUBIN_SIM_DIR}/batch/run_prenight_sims.sh
+
+or thus::
+
+    ${RUBIN_SIM_DIR}/batch/run_auxtel_prenight_sims.sh auxtel
+
+or they can be submitted as a batch job to slurm.
+At the USDF, there is an installation of `rubin_sim` at `/sdf/data/rubin/shared/scheduler/packages/rubin_sim` such that the commands become::
+
+    /sdf/data/rubin/shared/scheduler/packages/rubin_sim/batch/run_auxtel_prenight_sims.sh
+    sdf/data/rubin/shared/scheduler/packages/rubin_sim/batch/run_prenight_sims.sh
+
+
+These batch jobs will record their output in:
+
+* `/sdf/data/rubin/shared/scheduler/prenight/sbatch/run_prenight_sims_%A_%a.out` and
+* `/sdf/data/rubin/shared/scheduler/prenight/sbatch/run_prenight_sims_%A_%a.err`
+
+The working files can be found in `/sdf/data/rubin/shared/scheduler/prenight/work/run_prenight_sims/%Y-%m-%dT%H%M%`.
+
+Automated runs of prenight simulations
+======================================
+
+Batch jobs are submitted automatically each morning following the `sbatch` commands shown above.
+The `crontab` entries are::
+
+    15 6 * * * /opt/slurm/slurm-curr/bin/sbatch /sdf/data/rubin/shared/scheduler/packages/rubin_sim/batch/run_auxtel_prenight_sims.sh 2>&1 >> /sdf/data/rubin/shared/scheduler/prenight/daily/daily_auxtel_cron.out
+    30 6 * * * /opt/slurm/slurm-curr/bin/sbatch /sdf/data/rubin/shared/scheduler/packages/rubin_sim/batch/run_prenight_sims.sh 2>&1 >> /sdf/data/rubin/shared/scheduler/prenight/daily/daily_simonyi_cron.out
+
+
+Custom runs of prenight simulations
+===================================
+
+Much of the `run_prenight_sims.sh` script is dedicated to figuring out what environment to use and configuring it.
+If these defaults are not desired, a user can configure an environment to their liking, and run the simulation with the following steps.
+This environment must have the following installed:
+
+* `rubin_scheduler` (`github.com/lsst/rubin_sim`)
+* `schedview`  (`github.com/lsst/schedview`)
+* `ts_fbs_utils` (`github.com/lsst-ts/ts_fbs_utils.git`)
+* `sims_sv_survey` (`github.com/lsst-sims/sims_sv_survey`)
+* `rubin_nights` (`github.com/lsst-sims/rubin_night`)
+* `lsst_resources` (`github.com/lsst/resources`)
+
+First, retrieve the set of completed visits at the start of the nights to be simulated.
+For example::
+
+    DAYOBS="20250819"
+    COMPLETED_OPSIM_DB="completed_visits.db"
+    TOKEN_FILE="~/.usdf_access_token"
+    fetch_sv_visits ${DAYOBS} ${COMPLETED_OPSIM_DB} ${TOKEN_FILE}
+
+Second, create a pickle of the scheduler you want to run::
+
+    SCHEDULER_CONFIG_SCRIPT="ts_config_ocs/Scheduler/feature_scheduler/maintel/fbs_config_sv_survey.py"
+    SCHEDULER_PICKLE="scheduler.p"
+    COMPLETED_OPSIM_DB="completed_visits.db"
+    make_sv_scheduler ${SCHEDULER_PICKLE} --opsim ${COMPLETED_OPSIM_DB} --config-script ${SCHEDULER_CONFIG_SCRIPT}
+
+Third, create a model observatory pickle::
+
+    make_model_observatory observatory.p
+
+Fourch, create an band scheduler pickle::
+
+    make_band_scheduler band_scheduler.p
+
+Finally, run the sim (and add it to the archive)::
+
+    OPSIMRUN="prenight_nominal_$(date --iso=s)"
+    LABEL="Nominal start and overhead, ideal conditions, run at $(date --iso=s)"
+    NUM_NIGHTS=3
+    ARCHIVE="s3://rubin:rubin-scheduler-prenight/opsim/"
+    DELAY=0
+    ANOM_SCALE=0
+    ANOM_SEED=1
+    run_sv_sim \
+        ${SCHEDULER_PICKLE} \
+        observatory.p \
+        "" \
+        ${DAYOBS} \
+        ${NUM_NIGHTS} \
+        ${OPSIMRUN} \
+        --keep-rewards \
+        --no-downtime \
+        --label ${LABEL} \
+        --archive ${ARCHIVE} \
+        --capture-env \
+        --delay ${DELAY} \
+        --anom-overhead-scale ${ANOM_SCALE} \
+        --anom-overhead-seed ${ANOM_SEED} \
+        --tags mytag1 mytag2 mytag3
+
+See the output of `run_sv_sim --help` for the meanings of the options.

{rubin_sim-2.6.1a4 → rubin_sim-2.6.1.dev1}/optional-requirements.txt

@@ -1,10 +1,6 @@
-botocore
-boto3
-click
 conda
 openorb
 openorb-data-de405
-psycopg2
 george
 jinja2
 tornado

{rubin_sim-2.6.1a4 → rubin_sim-2.6.1.dev1}/pyproject.toml

@@ -58,12 +58,6 @@ extras = [
     "george",
     "pyoorb",
 ]
-vseqarchive = [
-    "click",
-    "psycopg2",
-    "botocore",
-    "boto3",
-]
 
 [project.scripts]
 rs_download_data = "rubin_sim.data.rs_download_data:rs_download_data"

{rubin_sim-2.6.1a4 → rubin_sim-2.6.1.dev1}/rubin_sim/maf/__init__.py

@@ -20,7 +20,6 @@
 # see <http://www.lsstcorp.org/LegalNotices/>.
 #
 """Python interface to the metrics analysis framework."""
-
 from .batches import *
 from .db import *
 from .maf_contrib import *