rubin-sim 2.6.1.dev2__py3-none-any.whl

sql/make_vseqmeta.sql

@@ -0,0 +1,251 @@
+CREATE SCHEMA {};
+
+SET SEARCH_PATH TO {};
+
+CREATE TABLE visitseq (
+    visitseq_uuid   UUID PRIMARY KEY DEFAULT gen_random_uuid(),   -- RFC 4122 Universally Unique IDentifier
+    visitseq_sha256 BYTEA NOT NULL,     -- hash of the visit table
+    visitseq_label  TEXT NOT NULL,      -- label for plots and table
+    visitseq_url    TEXT,               -- If null, the actual visits are not available
+    telescope       TEXT NOT NULL,      -- (probably) "simonyi" or "auxtel"
+    first_day_obs   DATE,               -- local calendar date of evening of first night in the set
+    last_day_obs    DATE,                -- local calendar date of the eveninig of the last night in the set
+    creation_time   TIMESTAMP WITH TIME ZONE DEFAULT NOW() -- when the simulation was run
+);
+
+CREATE TABLE simulations (
+    scheduler_version   TEXT,                               -- version of rubin_scheduler
+    config_url          TEXT,                               -- URL for the config script, perhaps on github based on hash
+    conda_env_sha256    BYTEA,                              -- SHA256 hash of output of conda list --json
+    parent_visitseq_uuid    UUID,                           -- UUID of visitseq loaded into scheduler before running
+    sim_runner_kwargs   JSONB,                              -- arguments to sim runner as a json dict
+    parent_last_day_obs DATE,                               -- day_obs of last visit loaded into scheduler before running
+    PRIMARY KEY (visitseq_uuid)
+) INHERITS (visitseq);
+
+CREATE TABLE completed (
+    query           TEXT,                                   -- The query to the consdb used
+    PRIMARY KEY (visitseq_uuid)
+) INHERITS (visitseq);
+
+CREATE TABLE mixedvisitseq (
+    last_early_day_obs  DATE,                               -- the last day obs drawn from the early parent
+    first_late_day_obs  DATE,                               -- the first day obs drawn from the late parent
+    early_parent_uuid   UUID,                               -- the UUID of the early parent
+    late_parent_uuid    UUID,                               -- the UUID of the late parent
+    PRIMARY KEY (visitseq_uuid)
+) INHERITS (visitseq);
+
+-- All of the tables that follow include a visitseq_uuid that refereneces
+-- visitseq or one of its children.
+-- These are not declared as foreign keys, because postgresql cannot enforce
+-- a FK on both a table and its children: we would need to specify which child
+-- the uuid would refer to, when really we need it to be allowed to reference
+-- any of the children.
+-- (In principle, we could do something complicated with triggers.)
+
+CREATE TABLE tags (
+    visitseq_uuid   UUID,                                   -- RFC 4122 Universally Unique IDentifier
+    tag             TEXT NOT NULL                           -- The tag
+);
+-- we will most often want to search for all runs with a given tag.
+CREATE INDEX tag_idx ON tags(tag);
+
+CREATE TABLE comments (
+    visitseq_uuid   UUID,                                   -- RFC 4122 Universally Unique IDentifier
+    comment_time    TIMESTAMP WITH TIME ZONE DEFAULT NOW() ,  -- The date and time at which the comment was added
+    author          TEXT,                                   -- The user who added the comment
+    comment         TEXT NOT NULL                           -- The comment
+);
+-- we will most often want to search for all comments for a given visitseq
+CREATE INDEX comments_visitseq_idx ON comments(visitseq_uuid);
+
+CREATE TABLE files (
+    visitseq_uuid   UUID,                                   -- RFC 4122 Universally Unique IDentifier
+    file_type       TEXT NOT NULL,                          -- Examples include "rewards", "scheduler", and "observatory"
+    file_sha256     BYTEA NOT NULL,                         -- hash of the file
+    file_url        TEXT,                                   -- URL for the file, may be null if we wish only to store the hash
+    PRIMARY KEY (visitseq_uuid, file_type)
+);
+CREATE INDEX files_visitseq_idx ON files(visitseq_uuid);
+
+-- A view to make it easy to query simulations and get tags, comments, and files back.
+CREATE VIEW simulations_extra AS
+    SELECT
+        DATE(s.creation_time - INTERVAL '12 hours') AS sim_creation_day_obs,
+        ROW_NUMBER() OVER (PARTITION BY DATE(s.creation_time - INTERVAL '12 hours') ORDER BY s.creation_time - INTERVAL '12 hours') AS daily_id,
+        s.visitseq_uuid,
+        s.visitseq_label,
+        s.visitseq_url,
+        s.telescope,
+        s.first_day_obs,
+        s.last_day_obs,
+        s.creation_time,
+        s.scheduler_version,
+        s.config_url,
+        s.sim_runner_kwargs,
+        s.conda_env_sha256,
+        s.parent_visitseq_uuid,
+        s.parent_last_day_obs,
+        COALESCE (
+            JSONB_AGG(DISTINCT t.tag) FILTER (WHERE t.tag IS NOT NULL),
+            '[]'::JSONB
+        ) AS tags,
+        COALESCE (
+             JSONB_OBJECT_AGG(c.comment_time, c.comment) FILTER (WHERE c.comment_time IS NOT NULL),
+             '{{}}'::JSONB
+        ) AS comments,
+        COALESCE (
+             JSONB_OBJECT_AGG(f.file_type, f.file_url) FILTER (WHERE f.file_type IS NOT NULL),
+             '{{}}'::JSONB
+        ) AS files
+    FROM simulations AS s
+    LEFT JOIN tags AS t ON t.visitseq_uuid=s.visitseq_uuid
+    LEFT JOIN comments AS c ON c.visitseq_uuid=s.visitseq_uuid
+    LEFT JOIN files AS f ON f.visitseq_uuid=s.visitseq_uuid
+    GROUP BY s.visitseq_uuid,
+        s.visitseq_label,
+        s.visitseq_url,
+        s.telescope,
+        s.first_day_obs,
+        s.last_day_obs,
+        s.creation_time,
+        s.scheduler_version,
+        s.config_url,
+        s.sim_runner_kwargs,
+        s.conda_env_sha256,
+        s.parent_visitseq_uuid,
+        s.parent_last_day_obs;
+
+CREATE TABLE conda_env (
+    conda_env_hash  BYTEA PRIMARY KEY,                      -- SHA256 hash of output of `conda list --json`
+    conda_env       JSONB NOT NULL                          -- output of `conda list --json`
+);
+
+-- Make a view so we do not need to remember
+-- postgresql syntex for jsonb.
+CREATE VIEW conda_packages AS (
+    SELECT
+        c.conda_env_hash,
+        p.name AS package_name,
+        p.channel,
+        p.version AS package_version,
+        p.base_url,
+        p.platform,
+        p.dist_name,
+        p.build_number,
+        p.build_string
+    FROM
+        conda_env AS c,
+        jsonb_to_recordset(conda_env) AS p (
+            name TEXT,
+            channel TEXT,
+            version TEXT,
+            base_url TEXT,
+            platform TEXT,
+            dist_name TEXT,
+            build_number TEXT,
+            build_string TEXT
+        )
+);
+
+CREATE VIEW simulation_packages AS (
+    SELECT s.visitseq_uuid, p.*
+    FROM conda_packages AS p
+    JOIN simulations AS s ON p.conda_env_hash = s.conda_env_sha256
+);
+
+CREATE TABLE nightly_stats (
+    visitseq_uuid   UUID NOT NULL,                          -- RFC 4122 Universally Unique IDentifier
+    day_obs         DATE NOT NULL,                          -- The day obs of the visits included
+    value_name      TEXT NOT NULL,                          -- metric or column name
+    accumulated     BOOLEAN NOT NULL,                       -- Whether the statistics include all data through night day_obs, or only data on night day_obs
+    count           INTEGER,                                -- number of values in distribution
+    mean            DOUBLE PRECISION,                       -- mean
+    std             DOUBLE PRECISION,                       -- standard deviation
+    min             DOUBLE PRECISION,                       -- min value
+    p05             DOUBLE PRECISION,                       -- 5% quantile
+    q1              DOUBLE PRECISION,                       -- first quartile
+    median          DOUBLE PRECISION,                       -- median
+    q3              DOUBLE PRECISION,                       -- third quartile
+    p95             DOUBLE PRECISION,                       -- 95% quantile
+    max             DOUBLE PRECISION,                       -- maximum
+    PRIMARY KEY (visitseq_uuid, day_obs, value_name, accumulated)
+);
+
+-- Tables to support functionality like that of rubin_sim.maf.run_comparison
+
+-- Use tags to mark visit sequences as being runs that are part of a given run family.
+
+-- Table to keep track of exactly what metrics were run in maf.
+CREATE TABLE maf_metrics (
+    maf_metric_name     TEXT PRIMARY KEY,                   -- name for this maf metric
+    rubin_sim_version   TEXT,                               -- the version of rubin_sim used
+    maf_constraint      TEXT,                               -- constraint imposed in maf
+    metric_class_name   TEXT,                               -- class name of the metric
+    metric_args         JSONB,                              -- arguments to the metric constructor
+    slicer_class_name   TEXT,                               -- class name of the slicer
+    slicer_args         JSONB                               -- arguments to the slicer constructor
+);
+CREATE INDEX maf_metrics_name_idx ON maf_metrics(maf_metric_name);
+
+-- Support functionality like that of rubin_sim.maf.run_comparison run summaries,
+-- but sampled at whatever dates are of interest rather than just the end.
+-- To get it into the "wide" format like that in run_comparison, use either
+-- a tablefunc in the postgresql query, or (probably easier) use unstack on
+-- the returned dataframe.
+CREATE TABLE maf_summary_metrics (
+    visitseq_uuid   UUID,                                   -- RFC 4122 Universally Unique IDentifier
+    maf_metric_name   TEXT REFERENCES maf_metrics(maf_metric_name), -- The name of the maf metric
+    day_obs         DATE,                                   -- The day obs of the visits included
+    accumulated     BOOLEAN,                                -- Whether the statistics include all data through night day_obs, or only data on night day_obs
+    summary_value   DOUBLE PRECISION,
+    PRIMARY KEY (visitseq_uuid, maf_metric_name)
+);
+CREATE INDEX maf_summary_metrics_name_idx ON maf_summary_metrics(maf_metric_name);
+
+-- metric sets for rubin_sim.maf.run_comparison
+CREATE TABLE maf_metric_sets (
+    metric_set      TEXT,                                   -- name of the metric set
+    maf_metric_name   TEXT REFERENCES maf_metrics(maf_metric_name), -- name of the maf metric
+    short_name      TEXT,                                   -- a shorter label to use in plots
+    style           TEXT,                                   -- matplotlib line style
+    invert          BOOLEAN,                                -- negative is lower is better
+    mag             BOOLEAN,                                 -- value is a magnitude
+    PRIMARY KEY (metric_set, maf_metric_name)
+);
+CREATE INDEX maf_metric_sets_metric_name_idx ON maf_metric_sets(maf_metric_name);
+CREATE INDEX maf_metric_sets_set_idx ON maf_metric_sets(metric_set);
+
+-- Create a view to make it easy to get everything for the summary metrics for one metric set
+-- applied to runs with specified tags.
+CREATE VIEW maf_summary AS
+    SELECT mms.metric_set, mms.maf_metric_name, v.visitseq_label,
+        msm.day_obs, msm.accumulated, msm.summary_value,
+        mms.short_name, mms.style, mms.invert, mms.mag, t.tag
+    FROM maf_metric_sets AS mms
+    JOIN maf_summary_metrics AS msm ON msm.maf_metric_name=mms.maf_metric_name
+    JOIN tags AS t ON t.visitseq_uuid=msm.visitseq_uuid
+    JOIN visitseq AS v ON msm.visitseq_uuid=v.visitseq_uuid
+    ;
+
+
+CREATE TABLE maf_healpix_stats (
+    visitseq_uuid   UUID NOT NULL,                          -- RFC 4122 Universally Unique IDentifier
+    maf_metric_name   TEXT REFERENCES maf_metrics(maf_metric_name), -- The name of the maf statistic
+    day_obs         DATE NOT NULL,                          -- The day obs of the visits included
+    accumulated     BOOLEAN NOT NULL,                       -- Whether the statistics include all data through night day_obs, or only data on night day_obs
+    nside           INTEGER NOT NULL,                       -- The nside of the healpix array
+    count           INTEGER NOT NULL,                       -- The number of unmasked values in the distribution
+    mean            DOUBLE PRECISION,                       -- mean
+    std             DOUBLE PRECISION,                       -- standard deviation
+    min             DOUBLE PRECISION,                       -- min value
+    p05             DOUBLE PRECISION,                       -- 5% quantile
+    q1              DOUBLE PRECISION,                       -- first quartile
+    median          DOUBLE PRECISION,                       -- median
+    q3              DOUBLE PRECISION,                       -- third quartile
+    p95             DOUBLE PRECISION,                       -- 95% quantile
+    max             DOUBLE PRECISION,                       -- maximum
+    url             TEXT,                                   -- URL for the array of healpix values, if saved
+    PRIMARY KEY (visitseq_uuid, maf_metric_name, day_obs, accumulated, nside)
+);

tests/data/test_data.py

@@ -0,0 +1,26 @@
+import os
+import unittest
+
+from rubin_scheduler.data import data_versions, get_data_dir
+
+from rubin_sim.data import get_baseline
+from rubin_sim.data import get_data_dir as gdd
+
+
+class DataTest(unittest.TestCase):
+    def testBaseline(self):
+        """
+        Get the baseline sim location
+        """
+        data_dir = get_data_dir()
+        dd2 = gdd()
+
+        assert data_dir == dd2
+
+        if "sim_baseline" in os.listdir(data_dir):
+            _ = get_baseline()
+        _ = data_versions()
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/data/test_ddf_grid.py

@@ -0,0 +1,17 @@
+import unittest
+
+from rubin_scheduler.scheduler.surveys import generate_ddf_grid
+
+
+class GenerateDDFTest(unittest.TestCase):
+    def testGenDDF(self):
+        """
+        Test the DDF grid generator over in rubin_scheduler
+        """
+        # This triggers several RunTimeErrors (intentionally?).
+        result = generate_ddf_grid(survey_length=0.01, verbose=False)
+        assert result is not None
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/maf/test_3x2fom.py

@@ -0,0 +1,68 @@
+import os
+import shutil
+import tempfile
+import unittest
+
+from rubin_scheduler.data import get_data_dir
+from rubin_scheduler.utils.code_utilities import sims_clean_up
+
+import rubin_sim.maf as maf
+
+
+class Test3x2(unittest.TestCase):
+    @classmethod
+    def tearDownClass(cls):
+        sims_clean_up()
+
+    def setUp(self):
+        self.out_dir = tempfile.mkdtemp(prefix="TMB")
+
+    @unittest.skipUnless(
+        os.path.isdir(os.path.join(get_data_dir(), "maps")),
+        "Skipping 3x3 metric test because no dust maps.",
+    )
+    def test_3x2(self):
+        # Only testing that the metric successfully runs, not checking that
+        # the output values are valid.
+        bundle_list = []
+        nside = 64
+        colmap = maf.batches.col_map_dict("fbs_sim")
+        nfilters_needed = 6
+        lim_ebv = 0.2
+        ptsrc_lim_mag_i_band = 25.9
+        m = maf.metrics.ExgalM5WithCuts(
+            m5_col=colmap["fiveSigmaDepth"],
+            filter_col="band",
+            lsst_filter="i",
+            n_filters=nfilters_needed,
+            extinction_cut=lim_ebv,
+            depth_cut=ptsrc_lim_mag_i_band,
+        )
+        s = maf.slicers.HealpixSlicer(nside=nside, use_cache=False)
+        sql = "scheduler_note not like 'DD%' and night < 365"
+        threeby_two_summary_simple = maf.metrics.StaticProbesFoMEmulatorMetricSimple(
+            metric_name="3x2ptFoM_simple"
+        )
+        threeby_two_summary = maf.maf_contrib.StaticProbesFoMEmulatorMetric(metric_name="3x2ptFoM")
+        bundle_list.append(
+            maf.metric_bundles.MetricBundle(
+                m,
+                s,
+                sql,
+                summary_metrics=[threeby_two_summary, threeby_two_summary_simple],
+            )
+        )
+
+        database = os.path.join(get_data_dir(), "sim_baseline", "baseline.db")
+        results_db = maf.db.ResultsDb(out_dir=self.out_dir)
+        bd = maf.metric_bundles.make_bundles_dict_from_list(bundle_list)
+        bg = maf.metric_bundles.MetricBundleGroup(bd, database, out_dir=self.out_dir, results_db=results_db)
+        bg.run_all()
+
+    def tearDown(self):
+        if os.path.isdir(self.out_dir):
+            shutil.rmtree(self.out_dir)
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/maf/test_archive.py

@@ -0,0 +1,162 @@
+# imports
+import os
+import sys
+import unittest
+from os import path
+from tempfile import TemporaryDirectory
+from unittest.mock import patch
+
+import pandas as pd
+
+from rubin_sim import maf
+from rubin_sim.data import get_data_dir
+
+# constants
+
+URLROOT = "https://raw.githubusercontent.com/lsst-pst/survey_strategy/main/fbs_2.0/"
+FAMILY_SOURCE = URLROOT + "runs_v2.1.json"
+METRIC_SUBSET_SOURCE = os.path.join(get_data_dir(), "maf", "metric_subsets.json")
+SUMMARY_SOURCE = os.path.join(get_data_dir(), "maf", "baseline_summary.h5")
+
+# exception classes
+
+# interface functions
+
+# classes
+
+
+class TestArchive(unittest.TestCase):
+    num_runs = 11
+    num_metrics = 7
+
+    def test_get_family_runs(self):
+        runs = maf.get_family_runs(FAMILY_SOURCE)
+
+        self.assertIsInstance(runs, pd.DataFrame)
+
+        columns = ("run", "brief", "filepath", "url")
+        for column in columns:
+            self.assertIn(column, runs.columns)
+
+        self.assertEqual(runs.index.name, "family")
+
+    def test_get_runs(self):
+        runs = maf.get_runs(FAMILY_SOURCE)
+        self.assertEqual(runs.index.name, "run")
+
+        columns = ("family", "brief", "filepath", "url")
+        for column in columns:
+            self.assertIn(column, runs.columns)
+
+        self.assertEqual(runs.index.name, "run")
+
+    @unittest.skip("skipping long running test.")
+    def test_download_runs(self):
+        run = maf.get_runs(FAMILY_SOURCE).index.values[0]
+
+        temp_dir_itself = TemporaryDirectory()
+        temp_dir = temp_dir_itself.name
+
+        dest_fnames = maf.download_runs(run, dest_dir=temp_dir)
+        self.assertTrue(path.exists(dest_fnames[0]))
+
+        temp_dir_itself.cleanup()
+
+    def test_get_metric_subsets(self):
+        metric_subsets = maf.get_metric_subsets(METRIC_SUBSET_SOURCE)
+        self.assertIsInstance(metric_subsets, pd.DataFrame)
+        self.assertIn("metric subset", metric_subsets.index.names)
+        self.assertIn("metric", metric_subsets.index.names)
+
+        columns = ("style", "invert", "mag")
+        for column in columns:
+            self.assertIn(column, metric_subsets.columns)
+
+    def test_get_metric_summaries(self):
+        summary = maf.get_metric_summaries(summary_source=SUMMARY_SOURCE)
+        self.assertIsInstance(summary, pd.DataFrame)
+        self.assertEqual(summary.columns.name, "metric")
+        self.assertEqual(summary.index.name, "run")
+
+        my_summary = maf.get_metric_summaries(
+            runs=["baseline_v3.0_10yrs", "baseline_v3.4_10yrs"],
+            metrics=[
+                "Rms Max normairmass All sky all bands HealpixSlicer",
+                "Median Max normairmass All sky all bands HealpixSlicer",
+                "Max Max normairmass All sky all bands HealpixSlicer",
+            ],
+            summary_source=summary,
+        )
+        self.assertEqual(len(my_summary), 2)
+        self.assertEqual(len(my_summary.columns), 3)
+
+        srd_sim = maf.get_metric_summaries(
+            metric_subsets="SRD",
+            summary_source=summary,
+            metric_subset_source=METRIC_SUBSET_SOURCE,
+        )
+        self.assertGreater(len(srd_sim.columns), 3)
+        self.assertLess(len(srd_sim.columns), len(summary.columns))
+
+        srdnvis_sim = maf.get_metric_summaries(
+            metric_subsets=["SRD", "N Visits"],
+            summary_source=summary,
+            metric_subset_source=METRIC_SUBSET_SOURCE,
+        )
+        self.assertGreater(len(srdnvis_sim.columns), len(srd_sim.columns))
+        self.assertLess(len(srdnvis_sim.columns), len(summary.columns))
+
+    def test_get_family_descriptions(self):
+        families = maf.get_family_descriptions(FAMILY_SOURCE)
+        self.assertIsInstance(families, pd.DataFrame)
+        self.assertEqual(families.index.name, "family")
+
+    def test_describe_families(self):
+        if "IPython" in sys.modules:
+            with (
+                patch("IPython.display.display_markdown") as _,
+                patch("IPython.display.HTML") as _,
+                patch("IPython.display") as _,
+            ):
+                self.perform_describe_families_test()
+        else:
+            with patch("builtins.print") as _:
+                self.perform_describe_families_test()
+
+    @unittest.skip("Skipping; family descriptions out of sync with summary")
+    def perform_describe_families_test(self):
+        families = maf.get_family_descriptions(FAMILY_SOURCE)
+        disp_families = families[:2]
+
+        fig, ax = maf.describe_families(families)
+        self.assertIsNone(fig)
+        self.assertIsNone(ax)
+
+        all_metric_subsets = maf.get_metric_subsets(METRIC_SUBSET_SOURCE)
+        summary = maf.get_metric_summaries(summary_source=SUMMARY_SOURCE)
+
+        table_metric_subset = all_metric_subsets.loc["SRD"]
+        fig, ax = maf.describe_families(
+            disp_families, summary=summary, table_metric_subset=table_metric_subset
+        )
+        self.assertIsNone(fig)
+        self.assertIsNone(ax)
+
+        plot_metric_subset = all_metric_subsets.loc["N Visits"]
+        fig, ax = maf.describe_families(disp_families, summary=summary, plot_metric_subset=plot_metric_subset)
+
+    def test_create_metric_subset(self):
+        metrics = ["Urania", "Thalia", "Calliope", "Terpsichore"]
+        metric_subset_name = "Muses"
+        metric_subset = maf.create_metric_subset(metric_subset_name, metrics)
+        self.assertSequenceEqual(metrics, metric_subset.metric.tolist())
+        self.assertSequenceEqual(
+            metric_subset.columns.tolist(),
+            ["metric", "short_name", "style", "invert", "mag"],
+        )
+        self.assertSequenceEqual(metric_subset.index.names, ["metric subset", "metric"])
+
+
+run_tests_now = __name__ == "__main__"
+if run_tests_now:
+    unittest.main()

tests/maf/test_basemetrics.py

@@ -0,0 +1,99 @@
+import unittest
+
+import rubin_sim.maf.metrics as metrics
+
+
+class TestBaseMetric(unittest.TestCase):
+    def test_reduce_dict(self):
+        """Test that reduce dictionary is created."""
+        testmetric = metrics.BaseMetric("testcol")
+        self.assertEqual(list(testmetric.reduce_funcs.keys()), [])
+
+    def test_metric_name(self):
+        """Test that metric name is set appropriately automatically
+        and when explicitly passed.
+        """
+        # Test automatic setting of metric name
+        testmetric = metrics.BaseMetric("testcol")
+        self.assertEqual(testmetric.name, "Base testcol")
+        testmetric = metrics.BaseMetric(["testcol1", "testcol2"])
+        self.assertEqual(testmetric.name, "Base testcol1, testcol2")
+        # Test explicit setting of metric name
+        testmetric = metrics.BaseMetric("testcol", metric_name="Test")
+        self.assertEqual(testmetric.name, "Test")
+
+    def test_col_registry(self):
+        """Test column registry adds to colRegistry as expected"""
+        # Clear the registry to make sure we start clear
+        colreg = metrics.ColRegistry()
+        colreg.clear_reg()
+
+        cols = "onecolumn"
+        colset = set()
+        colset.add(cols)
+        testmetric = metrics.BaseMetric(cols)
+        # Class registry should have dictionary with values =
+        # set of columns for metric class
+        for item in colset:
+            self.assertIn(item, testmetric.col_registry.col_set)
+        cols = ["onecolumn", "twocolumn"]
+        colset.add("twocolumn")
+        testmetric = metrics.BaseMetric(cols)
+        for item in colset:
+            self.assertIn(item, testmetric.col_registry.col_set)
+        # Test with additional (different) metric
+        cols = "twocolumn"
+        testmetric2 = metrics.MeanMetric(cols)
+        for item in colset:
+            self.assertIn(item, testmetric2.col_registry.col_set)
+
+        # test that the registry can be cleared
+        colreg.clear_reg()
+        assert len(colreg.col_set) == 0
+        assert len(colreg.db_set) == 0
+        assert len(colreg.stacker_dict) == 0
+
+    def test_metric_dtype(self):
+        """Test that base metric data value type set appropriately"""
+        cols = "onecolumn"
+        testmetric = metrics.BaseMetric(cols)
+        self.assertEqual(testmetric.metric_dtype, "float")
+        testmetric = metrics.BaseMetric(cols, metric_dtype="object")
+        self.assertEqual(testmetric.metric_dtype, "object")
+
+    def test_units(self):
+        """Test unit setting (including units set by utils.getColInfo)"""
+        cols = "onecolumn"
+        # Test for column not in colInfo, units not set by hand.
+        testmetric = metrics.BaseMetric(cols)
+        self.assertEqual(testmetric.units, "")
+        # Test for column not in colInfo, units set by hand.
+        testmetric = metrics.BaseMetric(cols, units="Test")
+        self.assertEqual(testmetric.units, "Test")
+        # Test for column in colInfo (looking up units in colInfo)
+        cols = "finSeeing"
+        testmetric = metrics.BaseMetric(cols)
+        self.assertEqual(testmetric.units, "arcsec")
+        # Test for column in colInfo but units overriden
+        testmetric = metrics.BaseMetric(cols, units="Test")
+        self.assertEqual(testmetric.units, "Test")
+        # Test for multiple columns not in colInfo
+        cols = ["onecol", "twocol"]
+        testmetric = metrics.BaseMetric(cols)
+        self.assertEqual(testmetric.units, "")
+        # Test for multiple columns in colInfo
+        cols = ["finSeeing", "filtSkyBrightness"]
+        testmetric = metrics.BaseMetric(cols)
+        self.assertEqual(testmetric.units, "arcsec mag/sq arcsec")
+        # Test for multiple columns, only one in colInfo
+        cols = ["finSeeing", "twocol"]
+        testmetric = metrics.BaseMetric(cols)
+        self.assertEqual(testmetric.units, "arcsec ")
+        # Test for multiple columns both in colInfo but repeated
+        cols = ["finSeeing", "finSeeing"]
+        testmetric = metrics.BaseMetric(cols)
+        self.assertEqual(testmetric.units, "arcsec arcsec")
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/maf/test_batchcommon.py

@@ -0,0 +1,36 @@
+import unittest
+
+import rubin_sim.maf.batches as batches
+
+
+class TestCommon(unittest.TestCase):
+    def test_col_map(self):
+        colmap = batches.col_map_dict("opsimv4")
+        self.assertEqual(colmap["raDecDeg"], True)
+        self.assertEqual(colmap["ra"], "fieldRA")
+        colmap = batches.col_map_dict("fbs")
+        self.assertEqual(colmap["raDecDeg"], True)
+        self.assertEqual(colmap["skyBrightness"], "skyBrightness")
+
+    def test_filter_list(self):
+        filterlist, colors, orders, sqls, info_label = batches.common.filter_list(all=False, extra_sql=None)
+        self.assertEqual(len(filterlist), 6)
+        self.assertEqual(len(colors), 6)
+        self.assertEqual(sqls["u"], "band = 'u'")
+        filterlist, colors, orders, sqls, info_label = batches.common.filter_list(all=True, extra_sql=None)
+        self.assertIn("all", filterlist)
+        self.assertEqual(sqls["all"], "")
+        filterlist, colors, orders, sqls, info_label = batches.common.filter_list(
+            all=True, extra_sql="night=3"
+        )
+        self.assertEqual(sqls["all"], "night=3")
+        self.assertEqual(sqls["u"], "(night=3) and (band = 'u')")
+        self.assertEqual(info_label["u"], "night=3 u band")
+        filterlist, colors, orders, sqls, info_label = batches.common.filter_list(
+            all=True, extra_sql="night=3", extra_info_label="night 3"
+        )
+        self.assertEqual(info_label["u"], "night 3 u band")
+
+
+if __name__ == "__main__":
+    unittest.main()