rubin-sim 2.6.1.dev2__py3-none-any.whl

batch/compile_prenight_metadata_cache.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+#SBATCH --account=rubin:developers      # Account name
+#SBATCH --job-name=auxtel_prenight_daily   # Job name
+#SBATCH --output=/sdf/data/rubin/shared/scheduler/prenight/sbatch/compile_prenight_metadata_cache.out # Output file (stdout)
+#SBATCH --error=/sdf/data/rubin/shared/scheduler/prenight/sbatch/compile_prenight_metadata_cache.err  # Error file (stderr)
+#SBATCH --partition=milano              # Partition (queue) names
+#SBATCH --nodes=1                       # Number of nodes
+#SBATCH --ntasks=1                      # Number of tasks run in parallel
+#SBATCH --cpus-per-task=1               # Number of CPUs per task
+#SBATCH --mem=4G                       # Requested memory
+#SBATCH --time=1:00:00                 # Wall time (hh:mm:ss)
+
+echo "******** START of compile_prenight_metadata_cache.sh **********"
+
+# Source global definitions
+if [ -f /etc/bashrc ]; then
+	. /etc/bashrc
+fi
+
+# SLAC S3DF - source all files under ~/.profile.d
+if [[ -e ~/.profile.d && -n "$(ls -A ~/.profile.d/)" ]]; then
+  source <(cat $(find -L  ~/.profile.d -name '*.conf'))
+fi
+
+source /sdf/group/rubin/sw/w_latest/loadLSST.sh
+conda activate /sdf/data/rubin/shared/scheduler/envs/prenight
+export AWS_PROFILE=prenight
+WORK_DIR=$(date '+/sdf/data/rubin/shared/scheduler/prenight/work/compile_prenight_metadata_cache/%Y-%m-%dT%H%M%S' --utc)
+echo "Working in $WORK_DIR"
+mkdir ${WORK_DIR}
+cd ${WORK_DIR}
+printenv > env.out
+compile_sim_archive_metadata_resource --append
+echo "******* END of compile_prenight_metadata_cache.sh *********"

batch/run_auxtel_prenight_sims.sh

@@ -0,0 +1,163 @@
+#!/usr/bin/env bash
+#SBATCH --account=rubin:developers      # Account name
+#SBATCH --job-name=auxtel_prenight_daily   # Job name
+#SBATCH --output=/sdf/data/rubin/shared/scheduler/prenight/sbatch/run_prenight_sims_%A_%a.out # Output file (stdout)
+#SBATCH --error=/sdf/data/rubin/shared/scheduler/prenight/sbatch/run_prenight_sims_%A_%a.out  # Error file (stderr)
+#SBATCH --partition=milano              # Partition (queue) names
+#SBATCH --nodes=1                       # Number of nodes
+#SBATCH --ntasks=1                      # Number of tasks run in parallel
+#SBATCH --cpus-per-task=1               # Number of CPUs per task
+#SBATCH --mem=8G                       # Requested memory
+#SBATCH --time=1:30:00                 # Wall time (hh:mm:ss)
+
+echo "******** START of run_prenight_sims.sh **********"
+
+# Source global definitions
+if [ -f /etc/bashrc ]; then
+	. /etc/bashrc
+fi
+
+# SLAC S3DF - source all files under ~/.profile.d
+if [[ -e ~/.profile.d && -n "$(ls -A ~/.profile.d/)" ]]; then
+  source <(cat $(find -L  ~/.profile.d -name '*.conf'))
+fi
+
+date --iso=s
+LATEST_TAGGED_STACK=$(
+    find /cvmfs/sw.lsst.eu/almalinux-x86_64/lsst_distrib \
+        -maxdepth 1 \
+        -regex '.*/v[0-9]+\.[0-9]+\.[0-9]+' \
+        -printf "%f\n" |
+    sort -V |
+    tail -1
+)
+source /cvmfs/sw.lsst.eu/almalinux-x86_64/lsst_distrib/${LATEST_TAGGED_STACK}/loadLSST-ext.sh
+
+set -o xtrace
+
+export AWS_PROFILE=prenight
+WORK_DIR=$(date '+/sdf/data/rubin/shared/scheduler/prenight/work/run_prenight_sims/%Y-%m-%dT%H%M%S' --utc)
+echo "Working in $WORK_DIR"
+mkdir ${WORK_DIR}
+cd ${WORK_DIR}
+
+# Install required python packages
+PACKAGE_DIR=$(readlink -f ${WORK_DIR}/packages)
+mkdir ${PACKAGE_DIR}
+export PYTHONPATH=${PACKAGE_DIR}:${PYTHONPATH}
+export PATH=${PACKAGE_DIR}/bin:${PATH}
+
+if false ; then
+  # Get latest tagged versions of everything
+  RUBIN_SCHEDULER_REFERENCE=$(curl -s https://api.github.com/repos/lsst/rubin_scheduler/tags | jq -r '.[].name' | egrep '^v[0-9]+.[0-9]+.[0-9]+$' | sort -V | tail -1)
+  RUBIN_SIM_REFERENCE=$(curl -s https://api.github.com/repos/lsst/rubin_sim/tags | jq -r '.[].name' | egrep '^v[0-9]+.[0-9]+.[0-9]+$' | sort -V | tail -1)
+  SCHEDVIEW_REFERENCE=$(curl -s https://api.github.com/repos/lsst/schedview/tags | jq -r '.[].name' | egrep '^v[0-9]+.[0-9]+.[0-9]+$' | sort -V | tail -1)
+  TS_FBS_UTILS_REFERENCE=$(curl -s https://api.github.com/repos/lsst-ts/ts_fbs_utils/tags | jq -r '.[].name' | egrep '^v[0-9]+.[0-9]+.[0-9]+$' | sort -V | tail -1)
+  SIMS_SV_SURVEY_REFERENCE=$(curl -s https://api.github.com/repos/lsst-sims/sims_sv_survey/tags | jq -r '.[].name' | egrep '^v[0-9]+.[0-9]+.[0-9]+$' | sort -V | tail -1)
+  RUBIN_NIGHTS_REFERENCE=$(curl -s https://api.github.com/repos/lsst-sims/rubin_nights/tags | jq -r '.[].name' | egrep '^v[0-9]+.[0-9]+.[0-9]+$' | sort -V | tail -1)
+else
+  # alternately, set specific versions
+  RUBIN_SCHEDULER_REFERENCE="v3.18.1"
+  RUBIN_SIM_REFERENCE="tickets/SP-2709"
+  SCHEDVIEW_REFERENCE="tickets/SP-2167"
+  TS_FBS_UTILS_REFERENCE=$(curl -s https://api.github.com/repos/lsst-ts/ts_fbs_utils/tags | jq -r '.[].name' | egrep '^v[0-9]+.[0-9]+.[0-9]+$' | sort -V | tail -1)
+  LSST_SURVEY_SIM_REFERENCE="tickets/SP-2709a"
+  RUBIN_NIGHTS_REFERENCE="v0.7.0"
+fi
+
+pip install --no-deps --target=${PACKAGE_DIR} \
+  git+https://github.com/lsst/rubin_scheduler.git@${RUBIN_SCHEDULER_REFERENCE} \
+  git+https://github.com/lsst/rubin_sim.git@${RUBIN_SIM_REFERENCE} \
+  git+https://github.com/lsst/schedview.git@${SCHEDVIEW_REFERENCE} \
+  git+https://github.com/lsst-ts/ts_fbs_utils.git@${TS_FBS_UTILS_REFERENCE} \
+  git+https://github.com/lsst-sims/lsst_survey_sim.git@${LSST_SURVEY_SIM_REFERENCE} \
+  git+https://github.com/lsst-sims/rubin_nights.git@${RUBIN_NIGHTS_REFERENCE} \
+  lsst-resources
+
+if false ; then
+  # Get the scheduler version from the EFD and install it.
+  # We have to do this after the others, because we want
+  # the version of obs_version_at_time supplied by the
+  # version of schedview we specify.
+  # Go ahead and install any missing dependencies as well.
+  RUBIN_SCHEDULER_REFERENCE=v$(obs_version_at_time rubin_scheduler)
+  echo "Using rubin_scheduler $RUBIN_SCHEDULER_REFERENCE"
+  pip install --ignore-installed --no-deps --upgrade --target=${PACKAGE_DIR} git+https://github.com/lsst/rubin_scheduler.git@${RUBIN_SCHEDULER_REFERENCE}
+fi
+
+# Get the scheduler configuration script
+# It lives in ts_config_scheduler
+TS_CONFIG_SCHEDULER_REFERENCE="develop"
+SCHED_CONFIG_FNAME="ts_config_scheduler/Scheduler/feature_scheduler/auxtel/fbs_spec_flex_survey.py"
+echo "Using ts_config_scheduler ${SCHED_CONFIG_FNAME} from ${TS_CONFIG_SCHEDULER_REFERENCE}"
+git clone --depth 1 https://github.com/lsst-ts/ts_config_scheduler
+cd ts_config_scheduler
+git fetch --depth 1 origin "${TS_CONFIG_SCHEDULER_REFERENCE}"
+git checkout FETCH_HEAD
+cd ${WORK_DIR}
+
+export DAYOBS="$(date -u --date='-12 hours' +'%Y%m%d')"
+export NEXT_DAYOBS="$(date -u --date='+12 hours' +'%Y%m%d')"
+export LAST_DAYOBS="$(date -u --date='+36 hours' +'%Y%m%d')"
+export DAYOBS_SIMULATED="$DAYOBS $NEXT_DAYOBS $LAST_DAYOBS"
+export LASTNIGHTISO="$(date --date='-36 hours' -u +'%F')"
+
+export ARCHIVE="s3://rubin:rubin-scheduler-prenight/opsim/vseq/"
+export VSARCHIVE_PGDATABASE="opsim_log"
+export VSARCHIVE_PGHOST="usdf-maf-visit-seq-archive-tx.sdf.slac.stanford.edu"
+export VSARCHIVE_PGUSER="writer"
+export VSARCHIVE_PGSCHEMA="vsmd"
+
+# Get an empty set of completed visits so we have something
+# to pass make_lsst_scheduler
+fetch_lsst_visits 20000101 completed_visits.db ~/.lsst/usdf_access_token
+
+echo "Creating scheduler pickle"
+date --iso=s
+make_lsst_scheduler scheduler.p --opsim completed_visits.db --config-script ${SCHED_CONFIG_FNAME}
+
+echo "Creating model observatory"
+date --iso=s
+ideal_model_observatory scheduler.p observatory.p
+
+# make dir for output
+OPSIM_RESULT_DIR=${WORK_DIR}/opsim_results
+mkdir ${OPSIM_RESULT_DIR}
+
+echo "Running nominal LSST simulation"
+OPSIMRUN="prenight_nominal_$(date --iso=s)"
+LABEL="Nominal start and overhead, ideal conditions, run at $(date --iso=s)"
+date --iso=s
+run_lsst_sim scheduler.p observatory.p "" ${DAYOBS} 3 "${OPSIMRUN}" \
+  --keep_rewards --label "${LABEL}" \
+  --delay 0 --anom_overhead_scale 0 \
+  --results ${OPSIM_RESULT_DIR}
+
+
+echo "Creating entry in metadatdata database"
+date --iso=s
+SIM_UUID=$(vseqarchive record-visitseq-metadata \
+    simulations \
+    ${OPSIM_RESULT_DIR}/opsim.db \
+    "${LABEL}" \
+    --telescope auxtel \
+    --first_day_obs ${DAYOBS} \
+    --last_day_obs ${LAST_DAYOBS}
+    )
+
+vseqarchive update-visitseq-metadata ${SIM_UUID} scheduler_version "${RUBIN_SCHEDULER_REFERENCE}"
+vseqarchive archive-file ${SIM_UUID} ${OPSIM_RESULT_DIR}/opsim.db visits --archive-base ${ARCHIVE}
+vseqarchive archive-file ${SIM_UUID} ${OPSIM_RESULT_DIR}/rewards.h5 rewards --archive-base ${ARCHIVE}
+vseqarchive tag ${SIM_UUID} prenight ideal nominal
+
+CONDA_HASH=$(vseqarchive record-conda-env)
+vseqarchive update-visitseq-metadata ${SIM_UUID} conda_env_sha256 ${CONDA_HASH}
+
+vseqarchive get-file ${SIM_UUID} visits visits.h5
+vseqarchive add-nightly-stats ${SIM_UUID} visits.h5 azimuth altitude
+
+rm visits.h5 ${OPSIM_RESULT_DIR}/opsim.db ${OPSIM_RESULT_DIR}/rewards.h5 ${OPSIM_RESULT_DIR}/obs_stats.txt ${OPSIM_RESULT_DIR}/observatory.p ${OPSIM_RESULT_DIR}/scheduler.p ${OPSIM_RESULT_DIR}/sim_metadata.yaml
+
+for DAYOBS_TO_INDEX in ${DAYOBS_SIMULATED}; do
+  vseqarchive make-prenight-index ${DAYOBS_TO_INDEX} auxtel
+done

batch/run_prenight_sims.sh

@@ -0,0 +1,239 @@
+#!/usr/bin/env bash
+#SBATCH --account=rubin:developers      # Account name
+#SBATCH --job-name=prenight_simonyi_daily   # Job name
+#SBATCH --output=/sdf/data/rubin/shared/scheduler/prenight/sbatch/run_prenight_sims_%A_%a.out # Output file (stdout)
+#SBATCH --error=/sdf/data/rubin/shared/scheduler/prenight/sbatch/run_prenight_sims_%A_%a.out  # Error file (stderr)
+#SBATCH --partition=milano              # Partition (queue) names
+#SBATCH --nodes=1                       # Number of nodes
+#SBATCH --ntasks=1                      # Number of tasks run in parallel
+#SBATCH --cpus-per-task=1               # Number of CPUs per task
+#SBATCH --mem=8G                       # Requested memory
+#SBATCH --time=1:30:00                 # Wall time (hh:mm:ss)
+
+echo "******** START of run_prenight_sims.sh **********"
+
+# Source global definitions
+if [ -f /etc/bashrc ]; then
+	. /etc/bashrc
+fi
+
+# SLAC S3DF - source all files under ~/.profile.d
+if [[ -e ~/.profile.d && -n "$(ls -A ~/.profile.d/)" ]]; then
+  source <(cat $(find -L  ~/.profile.d -name '*.conf'))
+fi
+
+date --iso=s
+LATEST_TAGGED_STACK=$(
+    find /cvmfs/sw.lsst.eu/almalinux-x86_64/lsst_distrib \
+        -maxdepth 1 \
+        -regex '.*/v[0-9]+\.[0-9]+\.[0-9]+' \
+        -printf "%f\n" |
+    sort -V |
+    tail -1
+)
+source /cvmfs/sw.lsst.eu/almalinux-x86_64/lsst_distrib/${LATEST_TAGGED_STACK}/loadLSST-ext.sh
+
+set -o xtrace
+
+export AWS_PROFILE=prenight
+WORK_DIR=$(date '+/sdf/data/rubin/shared/scheduler/prenight/work/run_prenight_sims/%Y-%m-%dT%H%M%S' --utc)
+echo "Working in $WORK_DIR"
+mkdir ${WORK_DIR}
+cd ${WORK_DIR}
+
+# Install required python packages
+PACKAGE_DIR=$(readlink -f ${WORK_DIR}/packages)
+mkdir ${PACKAGE_DIR}
+export PYTHONPATH=${PACKAGE_DIR}:${PYTHONPATH}
+export PATH=${PACKAGE_DIR}/bin:${PATH}
+
+if false ; then
+  # Get latest tagged versions of everything
+  RUBIN_SCHEDULER_REFERENCE=$(curl -s https://api.github.com/repos/lsst/rubin_scheduler/tags | jq -r '.[].name' | egrep '^v[0-9]+.[0-9]+.[0-9]+$' | sort -V | tail -1)
+  RUBIN_SIM_REFERENCE=$(curl -s https://api.github.com/repos/lsst/rubin_sim/tags | jq -r '.[].name' | egrep '^v[0-9]+.[0-9]+.[0-9]+$' | sort -V | tail -1)
+  SCHEDVIEW_REFERENCE=$(curl -s https://api.github.com/repos/lsst/schedview/tags | jq -r '.[].name' | egrep '^v[0-9]+.[0-9]+.[0-9]+$' | sort -V | tail -1)
+  TS_FBS_UTILS_REFERENCE=$(curl -s https://api.github.com/repos/lsst-ts/ts_fbs_utils/tags | jq -r '.[].name' | egrep '^v[0-9]+.[0-9]+.[0-9]+$' | sort -V | tail -1)
+  LSST_SURVEY_SIM_REFERENCE=$(curl -s https://api.github.com/repos/lsst-sims/lsst_survey_sim/tags | jq -r '.[].name' | egrep '^v[0-9]+.[0-9]+.[0-9]+$' | sort -V | tail -1)
+  RUBIN_NIGHTS_REFERENCE=$(curl -s https://api.github.com/repos/lsst-sims/rubin_nights/tags | jq -r '.[].name' | egrep '^v[0-9]+.[0-9]+.[0-9]+$' | sort -V | tail -1)
+else
+  # alternately, set specific versions
+  RUBIN_SCHEDULER_REFERENCE="v3.19.0"
+  RUBIN_SIM_REFERENCE="v2.6.1.dev1"
+  SCHEDVIEW_REFERENCE="v0.20.0.dev1"
+  TS_FBS_UTILS_REFERENCE="v0.19.0"
+  LSST_SURVEY_SIM_REFERENCE="v0.2.0"
+  RUBIN_NIGHTS_REFERENCE="v0.7.0"
+fi
+
+pip install --no-deps --target=${PACKAGE_DIR} \
+  git+https://github.com/lsst/rubin_scheduler.git@${RUBIN_SCHEDULER_REFERENCE} \
+  git+https://github.com/lsst/rubin_sim.git@${RUBIN_SIM_REFERENCE} \
+  git+https://github.com/lsst/schedview.git@${SCHEDVIEW_REFERENCE} \
+  git+https://github.com/lsst-ts/ts_fbs_utils.git@${TS_FBS_UTILS_REFERENCE} \
+  git+https://github.com/lsst-sims/lsst_survey_sim.git@${LSST_SURVEY_SIM_REFERENCE} \
+  git+https://github.com/lsst-sims/rubin_nights.git@${RUBIN_NIGHTS_REFERENCE} \
+  lsst-resources
+
+if false ; then
+  # Get the scheduler version from the EFD and install it.
+  # We have to do this after the others, because we want
+  # the version of obs_version_at_time supplied by the
+  # version of schedview we specify.
+  RUBIN_SCHEDULER_REFERENCE=v$(obs_version_at_time rubin_scheduler)
+  echo "Using rubin_scheduler $RUBIN_SCHEDULER_REFERENCE"
+  pip install --ignore-installed --no-deps --upgrade --target=${PACKAGE_DIR} git+https://github.com/lsst/rubin_scheduler.git@${RUBIN_SCHEDULER_REFERENCE}
+fi
+
+# Get the scheduler configuration script
+# It lives in ts_config_scheduler
+TS_CONFIG_SCHEDULER_REFERENCE="develop"
+SCHED_CONFIG_FNAME="ts_config_scheduler/Scheduler/feature_scheduler/maintel/fbs_config_lsst_survey.py"
+echo "Using ts_config_scheduler ${SCHED_CONFIG_FNAME} from ${TS_CONFIG_SCHEDULER_REFERENCE}"
+git clone --depth 1 https://github.com/lsst-ts/ts_config_scheduler
+cd ts_config_scheduler
+git fetch --depth 1 origin "${TS_CONFIG_SCHEDULER_REFERENCE}"
+git checkout FETCH_HEAD
+cd ${WORK_DIR}
+
+export DAYOBS="$(date -u --date='-12 hours' +'%Y%m%d')"
+export NEXT_DAYOBS="$(date -u --date='+12 hours' +'%Y%m%d')"
+export LAST_DAYOBS="$(date -u --date='+36 hours' +'%Y%m%d')"
+export DAYOBS_SIMULATED="$DAYOBS $NEXT_DAYOBS $LAST_DAYOBS"
+export LASTNIGHTISO="$(date --date='-36 hours' -u +'%F')"
+
+export ARCHIVE="s3://rubin:rubin-scheduler-prenight/opsim/vseq/"
+export VSARCHIVE_PGDATABASE="opsim_log"
+export VSARCHIVE_PGHOST="usdf-maf-visit-seq-archive-tx.sdf.slac.stanford.edu"
+export VSARCHIVE_PGUSER="writer"
+export VSARCHIVE_PGSCHEMA="vsmd"
+
+echo "Fetching completed visits"
+date --iso=s
+fetch_lsst_visits ${DAYOBS} completed_visits.db ~/.lsst/usdf_access_token
+
+# Recording hash of fetched visits
+COMPLETED=$(vseqarchive record-visitseq-metadata \
+    completed \
+    completed_visits.db \
+    "Consdb query through ${LASTNIGHTISO}" \
+    --first_day_obs 20250620 \
+    --last_day_obs ${LASTNIGHTISO})
+
+echo "Creating scheduler pickle"
+date --iso=s
+make_lsst_scheduler scheduler.p --opsim completed_visits.db --config-script ${SCHED_CONFIG_FNAME}
+
+echo "Creating model observatory"
+date --iso=s
+make_model_observatory observatory.p
+
+echo "Creating the band scheduler"
+date --iso=s
+make_band_scheduler band_scheduler.p
+
+# make dir for output
+OPSIM_RESULT_DIR=${WORK_DIR}/opsim_results
+mkdir ${OPSIM_RESULT_DIR}
+
+echo "Running nominal LSST simulation"
+OPSIMRUN="prenight_nominal_$(date --iso=s)"
+LABEL="Nominal start and overhead, ideal conditions, run at $(date --iso=s)"
+date --iso=s
+run_lsst_sim scheduler.p observatory.p "" ${DAYOBS} 3 "${OPSIMRUN}" \
+  --keep_rewards --label "${LABEL}" \
+  --delay 0 --anom_overhead_scale 0 \
+  --results ${OPSIM_RESULT_DIR}
+
+echo "Creating entry in metadatdata database"
+date --iso=s
+SIM_UUID=$(vseqarchive record-visitseq-metadata \
+    simulations \
+    ${OPSIM_RESULT_DIR}/opsim.db \
+    "${LABEL}" \
+    --first_day_obs ${DAYOBS} \
+    --last_day_obs ${LAST_DAYOBS}
+    )
+vseqarchive update-visitseq-metadata ${SIM_UUID} parent_visitseq_uuid ${COMPLETED}
+vseqarchive update-visitseq-metadata ${SIM_UUID} parent_last_day_obs ${LASTNIGHTISO}
+
+vseqarchive update-visitseq-metadata ${SIM_UUID} scheduler_version "${RUBIN_SCHEDULER_REFERENCE}"
+vseqarchive archive-file ${SIM_UUID} ${OPSIM_RESULT_DIR}/opsim.db visits --archive-base ${ARCHIVE}
+vseqarchive archive-file ${SIM_UUID} ${OPSIM_RESULT_DIR}/rewards.h5 rewards --archive-base ${ARCHIVE}
+vseqarchive tag ${SIM_UUID} prenight ideal nominal
+
+CONDA_HASH=$(vseqarchive record-conda-env)
+vseqarchive update-visitseq-metadata ${SIM_UUID} conda_env_sha256 ${CONDA_HASH}
+
+vseqarchive get-file ${SIM_UUID} visits visits.h5
+vseqarchive add-nightly-stats ${SIM_UUID} visits.h5 azimuth altitude
+
+rm visits.h5 ${OPSIM_RESULT_DIR}/opsim.db ${OPSIM_RESULT_DIR}/rewards.h5 ${OPSIM_RESULT_DIR}/obs_stats.txt ${OPSIM_RESULT_DIR}/observatory.p ${OPSIM_RESULT_DIR}/scheduler.p ${OPSIM_RESULT_DIR}/sim_metadata.yaml
+
+for DELAY in 60 240 ; do
+  echo "Running SV simulation delayed ${DELAY}"
+  OPSIMRUN="prenight_delay${DELAY}_$(date --iso=s)"
+  LABEL="Start time delayed by ${DELAY} minutes, nominal slew and visit overhead, ideal conditions, run at $(date --iso=s)"
+  date --iso=s
+  run_lsst_sim scheduler.p observatory.p "" ${DAYOBS} 3 "${OPSIMRUN}" \
+    --keep_rewards --label "${LABEL}" \
+    --delay ${DELAY} --anom_overhead_scale 0 \
+    --results ${OPSIM_RESULT_DIR}
+
+  SIM_UUID=$(vseqarchive record-visitseq-metadata \
+      simulations \
+      ${OPSIM_RESULT_DIR}/opsim.db \
+      "${LABEL}" \
+      --first_day_obs ${DAYOBS} \
+      --last_day_obs ${LAST_DAYOBS}
+      )
+  vseqarchive update-visitseq-metadata ${SIM_UUID} parent_visitseq_uuid ${COMPLETED}
+  vseqarchive update-visitseq-metadata ${SIM_UUID} parent_last_day_obs ${LASTNIGHTISO}
+  vseqarchive update-visitseq-metadata ${SIM_UUID} scheduler_version "${RUBIN_SCHEDULER_REFERENCE}"
+  vseqarchive archive-file ${SIM_UUID} ${OPSIM_RESULT_DIR}/opsim.db visits --archive-base ${ARCHIVE}
+  vseqarchive archive-file ${SIM_UUID} ${OPSIM_RESULT_DIR}/rewards.h5 rewards --archive-base ${ARCHIVE}
+  vseqarchive tag ${SIM_UUID} prenight ideal delay_${DELAY}
+  vseqarchive update-visitseq-metadata ${SIM_UUID} conda_env_sha256 ${CONDA_HASH}
+  vseqarchive get-file ${SIM_UUID} visits visits.h5
+  vseqarchive add-nightly-stats ${SIM_UUID} visits.h5 azimuth altitude
+
+  rm visits.h5 ${OPSIM_RESULT_DIR}/opsim.db ${OPSIM_RESULT_DIR}/rewards.h5 ${OPSIM_RESULT_DIR}/obs_stats.txt ${OPSIM_RESULT_DIR}/observatory.p ${OPSIM_RESULT_DIR}/scheduler.p ${OPSIM_RESULT_DIR}/sim_metadata.yaml
+done
+
+ANOM_SCALE="0.1"
+for ANOM_SEED in 101 102 ; do
+  echo "Running SV simulation with anomalous overhead seed ${ANOM_SEED}"
+  OPSIMRUN="prenight_anom${ANOM_SEED}_$(date --iso=s)"
+  LABEL="Anomalous overhead (${ANOM_SEED}, ${ANOM_SCALE}), nominal start, ideal conditions, run at $(date --iso=s)"
+  date --iso=s
+  run_lsst_sim scheduler.p observatory.p "" ${DAYOBS} 3 "${OPSIMRUN}" \
+    --keep_rewards --label "${LABEL}" \
+    --delay 0 \
+    --anom_overhead_scale ${ANOM_SCALE} \
+    --anom_overhead_seed ${ANOM_SEED} \
+    --results ${OPSIM_RESULT_DIR}
+
+  SIM_UUID=$(vseqarchive record-visitseq-metadata \
+      simulations \
+      ${OPSIM_RESULT_DIR}/opsim.db \
+      "${LABEL}" \
+      --first_day_obs ${DAYOBS} \
+      --last_day_obs ${LAST_DAYOBS}
+      )
+  vseqarchive update-visitseq-metadata ${SIM_UUID} parent_visitseq_uuid ${COMPLETED}
+  vseqarchive update-visitseq-metadata ${SIM_UUID} parent_last_day_obs ${LASTNIGHTISO}
+  vseqarchive update-visitseq-metadata ${SIM_UUID} scheduler_version "${RUBIN_SCHEDULER_REFERENCE}"
+  vseqarchive archive-file ${SIM_UUID} ${OPSIM_RESULT_DIR}/opsim.db visits --archive-base ${ARCHIVE}
+  vseqarchive archive-file ${SIM_UUID} ${OPSIM_RESULT_DIR}/rewards.h5 rewards --archive-base ${ARCHIVE}
+  vseqarchive tag ${SIM_UUID} prenight ideal anomalous_overhead
+  vseqarchive update-visitseq-metadata ${SIM_UUID} conda_env_sha256 ${CONDA_HASH}
+  vseqarchive get-file ${SIM_UUID} visits visits.h5
+  vseqarchive add-nightly-stats ${SIM_UUID} visits.h5 azimuth altitude
+
+  rm visits.h5 ${OPSIM_RESULT_DIR}/opsim.db ${OPSIM_RESULT_DIR}/rewards.h5 ${OPSIM_RESULT_DIR}/obs_stats.txt ${OPSIM_RESULT_DIR}/observatory.p ${OPSIM_RESULT_DIR}/scheduler.p ${OPSIM_RESULT_DIR}/sim_metadata.yaml
+done
+
+rm observatory.p scheduler.p
+
+for DAYOBS_TO_INDEX in ${DAYOBS_SIMULATED}; do
+  vseqarchive make-prenight-index ${DAYOBS_TO_INDEX} simonyi
+done

docs/.gitignore

@@ -0,0 +1,3 @@
+_build/*
+source/*
+maf-metric-list.rst

docs/Makefile

@@ -0,0 +1,32 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?= --keep-going -T -n
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = .
+BUILDDIR      = _build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+clean:
+	rm -rf $(BUILDDIR)
+
+linkcheck:
+	$(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck
+	@echo
+	@echo "Link check complete; look for any errors in the above output " \
+	      "or in $(BUILDDIR)/linkcheck/output.txt."
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	python metric_list.py
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+

docs/api.rst

@@ -0,0 +1,25 @@
+.. py:currentmodule:: rubin_sim
+
+.. _api:
+
+API
+===
+
+.. toctree::
+    :maxdepth: 2
+
+    Data <data-api>
+
+    Metrics Analysis Framework (MAF) <maf-api>
+
+    Moving Objects <moving-objects-api>
+
+    Phot Utils <phot-utils-api>
+
+    Satellite Constellations <satellite-constellations-api>
+
+    Self Calibration <selfcal-api>
+
+    Sim archive <sim-archive-api>
+
+    Skybrightness <skybrightness-api>

docs/conf.py

@@ -0,0 +1,8 @@
+# Configuration file for the Sphinx documentation builder.
+#
+# For the full list of built-in configuration values, see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+from documenteer.conf.guide import *  # noqa: F403, import *
+
+linkcheck_retries = 2

docs/data-api.rst

@@ -0,0 +1,12 @@
+.. py:currentmodule:: rubin_sim.data
+
+.. _data-api:
+
+========
+Data API
+========
+
+.. automodule:: rubin_sim.data
+    :imported-members:
+    :members:
+    :show-inheritance:

docs/data-download.rst

@@ -0,0 +1,70 @@
+.. py:currentmodule:: rubin_sim.data
+
+.. _data-download:
+
+=============
+Data Download
+=============
+
+The ``rubin_sim.data`` module provides a script to download the data
+required to run various modules in ``rubin_sim``, as well as to check the
+expected versions of the data. It also provides utilities to interpret
+the location of this $RUBIN_SIM_DATA_DIR on disk and to return the
+path to the current baseline simulation output (one of the datafiles
+downloaded by this module).
+
+With the split of ``rubin_sim`` into ``rubin_sim`` + ``rubin_scheduler``, the
+required data download utilities now live in the
+`rubin_scheduler.data <https://rubin-scheduler.lsst.io/data-download.html>`_
+package. ``rubin_scheduler`` is a necessary dependency of ``rubin_sim`` and
+should have
+been installed during the :ref:`installation <installation>` process.
+The ``rubin_sim.data`` module simply provides additional information on the
+data files necessary for ``rubin_sim``, then calls the scripts from
+``rubin_scheduler.data`` to execute the download.
+
+
+Downloading Necessary Data
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Please see the information in the
+`rubin-scheduler "Downloading Necessary Data" documentation <https://rubin-scheduler.lsst.io/data-download.html#downloading-necessary-data>`_
+for more details on setting up $RUBIN_SIM_DATA_DIR (which is
+shared between ``rubin_scheduler``, ``rubin_sim`` and ``schedview``).
+
+Using either the default path to $RUBIN_SIM_DATA_DIR, or after setting it
+explicitly, first download the necessary data for ``rubin_scheduler`` and
+then add the (larger) data set for ``rubin_sim``:
+
+.. code-block:: bash
+
+    scheduler_download_data
+    rs_download_data
+
+This creates a series of directories at $RUBIN_SIM_DATA_DIR (in addition
+to the directories originating from `rubin_scheduler <https://rubin-scheduler.lsst.io/data-download.html#downloading-necessary-data>`_):
+
+* maf (containing data used for various metrics)
+* maps (containing various stellar density and 2-D and 3-D dust maps)
+* movingObjects (containing asteroid SEDs)
+* orbits (containing orbits for Solar System population samples)
+* orbits_precompute (precomputed daily orbits for the samples above)
+* sim_baseline (containing the current baseline simulation output)
+* skybrightness (containing information needed for the skybrightness module)
+* throughputs (current baseline throughput information)
+* test (containing data for unit tests)
+
+
+Note that the data will only be downloaded for the directories which do
+not already exist, regardless of whether the version on disk is up to date.
+To force an update to a version which matches the ``rubin_scheduler`` version:
+
+.. code-block:: bash
+
+    rs_download_data --update
+
+This can also be applied only to certain directories, using the
+``--dirs`` flag. It may be worth noting that some of the above directories
+are more sizeable than others -- the ``maps``, ``maf`` and
+``orbits_precompute`` directories are the largest and if not needed, can
+be skipped in download by using ``--dirs``.

docs/developer.rst

@@ -0,0 +1,14 @@
+.. py:currentmodule:: rubin_sim
+
+.. _developer:
+
+##########
+User Guide
+##########
+
+
+.. toctree::
+
+    Visit Sequence Archive Architecture <vseqarchive-architecture>
+
+    Visit Sequence Archive Setup <vseqarchive-setup>

docs/documenteer.toml

@@ -0,0 +1,11 @@
+[project]
+title = "rubin_sim"
+copyright = "2023 Association of Universities for Research in Astronomy, Inc. (AURA)"
+base_url = "https://rubin-sim.lsst.io"
+github_url = "https://github.com/lsst/rubin_sim"
+
+[project.python]
+package="rubin_sim"
+
+[sphinx]
+extensions = ["sphinx.ext.viewcode"]

docs/index.rst

@@ -0,0 +1,22 @@
+.. py:currentmodule:: rubin_sim
+
+.. _rubin-sim-index:
+
+#########
+rubin_sim
+#########
+
+Helping simulate and evaluate the performance of
+Rubin C. Observatory's Legacy Survey of Space and Time (the LSST),
+through evaluating prototypes of various analysis, simulating photometry
+and providing a framework for analyzing survey strategy progress and outcomes.
+
+
+.. toctree::
+    :maxdepth: 2
+
+    Introduction <introduction>
+    Installation <installation>
+    User Guide <user-guide>
+    Developer's documenation <developer>
+    API <api>