rubin-sim 2.6.1.dev2__py3-none-any.whl

tests/maf/test_hgplotters.py

@@ -0,0 +1,145 @@
+import os
+import sqlite3
+import unittest
+from tempfile import TemporaryDirectory
+
+import pandas as pd
+
+import rubin_sim.maf.db as db
+import rubin_sim.maf.metric_bundles as metric_bundles
+import rubin_sim.maf.metrics as metrics
+import rubin_sim.maf.plots as plots
+import rubin_sim.maf.slicers as slicers
+
+
+class TestScalarHourglassPlots(unittest.TestCase):
+    def setUp(self):
+        self._data_dir_itself = TemporaryDirectory()
+        opsdb_fname = create_sample_visit_db(self._data_dir_itself.name)
+        self._out_dir_itself = TemporaryDirectory()
+        self.out_dir = self._out_dir_itself.name
+        self.results_db = db.ResultsDb(out_dir=self.out_dir)
+        self.ops_db = db.OpsimDatabase(opsdb_fname)
+        self.slicer = slicers.TimeIntervalSlicer(interval_seconds=3600)
+        self.metric = metrics.MedianMetric("seeingFwhmEff")
+
+    @unittest.skip("This is now throwing astropy.utils.iers.iers.IERSRangeError, but only sometimes?")
+    def test_month_plot(self):
+        sql_constraint = ""
+        metric_bundle = metric_bundles.MetricBundle(self.metric, self.slicer, sql_constraint)
+        bundle_dict = {1: metric_bundle}
+        bundle_group = metric_bundles.MetricBundleGroup(
+            bundle_dict,
+            self.ops_db,
+            out_dir=self.out_dir,
+            results_db=self.results_db,
+        )
+        _ = bundle_group.run_all()
+        plot_func = plots.YearHourglassPlot(2023)
+        _ = metric_bundle.plot(plotFunc=plot_func)
+
+
+class TestTimeUseHourglassPlots(unittest.TestCase):
+    def setUp(self):
+        self._data_dir_itself = TemporaryDirectory()
+        opsdb_fname = create_sample_visit_db(self._data_dir_itself.name)
+
+        self._out_dir_itself = TemporaryDirectory()
+        self.out_dir = self._out_dir_itself.name
+        self.results_db = db.ResultsDb(out_dir=self.out_dir)
+        self.ops_db = db.OpsimDatabase(opsdb_fname)
+        self.slicer = slicers.BlockIntervalSlicer()
+        self.metric = metrics.UseMetric()
+
+    @unittest.skip("XXX-seems like this was erroneously passing before?")
+    def test_year_plot(self):
+        sql_constraint = ""
+        metric_bundle = metric_bundles.MetricBundle(self.metric, self.slicer, sql_constraint)
+        bundle_dict = {1: metric_bundle}
+        bundle_group = metric_bundles.MetricBundleGroup(
+            bundle_dict,
+            self.ops_db,
+            out_dir=self.out_dir,
+            results_db=self.results_db,
+        )
+        _ = bundle_group.run_all()
+        plot_func = plots.YearHourglassUsePlot(2023)
+        _ = metric_bundle.plot(plotFunc=plot_func)
+
+
+def create_sample_visit_db(dir_name):
+    fname = os.path.join(dir_name, "opsim.sqlite3")
+
+    sample_visits_dict = {
+        "observationStartMJD": {
+            75831: 59974.1403819843,
+            88243: 59991.13314155574,
+            84877: 59986.16509162588,
+            73474: 59971.038614223085,
+            91024: 59994.3667217889,
+            94567: 60001.38094520713,
+            88867: 59992.06039742178,
+            91397: 59995.17212771153,
+            88282: 59991.151247547146,
+            82396: 59983.0460827712,
+        },
+        "visitTime": {
+            75831: 34.0,
+            88243: 34.0,
+            84877: 34.0,
+            73474: 34.0,
+            91024: 34.0,
+            94567: 34.0,
+            88867: 34.0,
+            91397: 34.0,
+            88282: 34.0,
+            82396: 34.0,
+        },
+        "seeingFwhmEff": {
+            75831: 0.9580660562409485,
+            88243: 1.2816139136535794,
+            84877: 0.8711626732163685,
+            73474: 1.138078694389459,
+            91024: 0.6851416920629385,
+            94567: 0.9526705179882091,
+            88867: 1.0749841095670385,
+            91397: 0.9092889658087193,
+            88282: 1.3069907982221605,
+            82396: 0.8034796448061345,
+        },
+        "filter": {
+            75831: "y",
+            88243: "g",
+            84877: "z",
+            73474: "r",
+            91024: "r",
+            94567: "y",
+            88867: "r",
+            91397: "r",
+            88282: "g",
+            82396: "r",
+        },
+        "scheduler_note": {
+            75831: "blob, zy, b",
+            88243: "blob, ug, b",
+            84877: "blob, iz, b",
+            73474: "greedy",
+            91024: "greedy",
+            94567: "greedy",
+            88867: "blob, ur, b",
+            91397: "blob, ri, a",
+            88282: "blob, gr, a",
+            82396: "DD:ECDFS",
+        },
+    }
+
+    visits = pd.DataFrame(sample_visits_dict)
+    with sqlite3.connect(fname) as con:
+        visits.to_sql("observations", con=con)
+
+    return fname
+
+
+run_tests_now = __name__ == "__main__"
+if run_tests_now:
+    unittest.main()

tests/maf/test_hourglassmetric.py

@@ -0,0 +1,33 @@
+import unittest
+
+import numpy as np
+
+import rubin_sim.maf.metrics as metrics
+
+
+class TestHourglassmetric(unittest.TestCase):
+    def test_hourglass_metric(self):
+        """Test the hourglass metric"""
+        names = ["observationStartMJD", "night", "band"]
+        types = [float, float, str]
+        npts = 50
+        data = np.zeros(npts, dtype=list(zip(names, types)))
+        day0 = 59000
+        data["observationStartMJD"] = np.arange(0, 10, 0.2)[:npts] + day0
+        data["night"] = np.floor(data["observationStartMJD"] - day0)
+        data["band"] = "r"
+        data["band"][-1] = "g"
+        slice_point = [0]
+        metric = metrics.HourglassMetric()
+        result = metric.run(data, slice_point)
+        pernight = result["pernight"]
+        perfilter = result["perfilter"]
+
+        # All the gaps are larger than 2 min.
+        assert np.size(perfilter) == 2 * data.size
+        # Check that the format is right at least
+        assert len(pernight.dtype.names) == 9
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/maf/test_io.py

@@ -0,0 +1,187 @@
+import unittest
+from tempfile import TemporaryFile
+
+import healpy as hp
+import numpy as np
+import numpy.ma as ma
+
+import rubin_sim.maf.slicers as slicers
+
+
+class TestSlicers(unittest.TestCase):
+    def setUp(self):
+        self.filenames = []
+        self.baseslicer = slicers.BaseSlicer()
+
+    def test_healpix_slicer_obj(self):
+        rng = np.random.RandomState(8121)
+        nside = 32
+        slicer = slicers.HealpixSlicer(nside=nside)
+        metric_values = rng.rand(hp.nside2npix(nside)).astype("object")
+        metric_values = ma.MaskedArray(
+            data=metric_values,
+            mask=np.where(metric_values < 0.1, True, False),
+            fill_value=slicer.badval,
+        )
+        with TemporaryFile() as filename:
+            info_label = "testdata"
+            slicer.write_data(filename, metric_values, info_label=info_label)
+            _ = filename.seek(0)
+            metric_values_back, slicer_back, header = self.baseslicer.read_data(filename)
+            np.testing.assert_almost_equal(metric_values_back, metric_values)
+            assert slicer == slicer_back
+            assert info_label == header["info_label"]
+            attr2check = ["nside", "nslice", "columns_needed", "lon_col", "lat_col"]
+            for att in attr2check:
+                assert getattr(slicer, att) == getattr(slicer_back, att)
+
+    def test_healpix_slicer_floats(self):
+        rng = np.random.RandomState(71231)
+        nside = 32
+        slicer = slicers.HealpixSlicer(nside=nside)
+        metric_values = rng.rand(hp.nside2npix(nside))
+        with TemporaryFile() as filename:
+            slicer.write_data(filename, metric_values, info_label="testdata")
+            _ = filename.seek(0)
+            metric_values_back, slicer_back, header = self.baseslicer.read_data(filename)
+            np.testing.assert_almost_equal(metric_values_back, metric_values)
+            assert slicer == slicer_back
+            attr2check = ["nside", "nslice", "columns_needed", "lon_col", "lat_col"]
+            for att in attr2check:
+                assert getattr(slicer, att) == getattr(slicer_back, att)
+
+    def test_healpix_slicer_masked(self):
+        rng = np.random.RandomState(712551)
+        nside = 32
+        slicer = slicers.HealpixSlicer(nside=nside)
+        metric_values = rng.rand(hp.nside2npix(nside))
+        metric_values = ma.MaskedArray(
+            data=metric_values,
+            mask=np.where(metric_values < 0.1, True, False),
+            fill_value=slicer.badval,
+        )
+        with TemporaryFile() as filename:
+            slicer.write_data(filename, metric_values, info_label="testdata")
+            _ = filename.seek(0)
+            metric_values_back, slicer_back, header = self.baseslicer.read_data(filename)
+            np.testing.assert_almost_equal(metric_values_back, metric_values)
+            assert slicer == slicer_back
+            attr2check = ["nside", "nslice", "columns_needed", "lon_col", "lat_col"]
+            for att in attr2check:
+                assert getattr(slicer, att) == getattr(slicer_back, att)
+
+    def test_one_d_slicer(self):
+        rng = np.random.RandomState(71111)
+        slicer = slicers.OneDSlicer(slice_col_name="testdata")
+        data_values = np.zeros(10000, dtype=[("testdata", "float")])
+        data_values["testdata"] = rng.rand(10000)
+        slicer.setup_slicer(data_values)
+        with TemporaryFile() as filename:
+            slicer.write_data(filename, data_values[:100])
+            _ = filename.seek(0)
+            data_back, slicer_back, header = self.baseslicer.read_data(filename)
+            assert slicer == slicer_back
+            # np.testing.assert_almost_equal(data_back,data_values[:100])
+            attr2check = ["nslice", "columns_needed"]
+            for att in attr2check:
+                if type(getattr(slicer, att)).__module__ == "numpy":
+                    np.testing.assert_almost_equal(getattr(slicer, att), getattr(slicer_back, att))
+                else:
+                    assert getattr(slicer, att) == getattr(slicer_back, att)
+
+    def test_userpoints_slicer(self):
+        rng = np.random.RandomState(7442)
+
+        names = ["fieldRA", "fieldDec", "fieldId"]
+        dt = ["float", "float", "int"]
+        metric_values = rng.rand(100)
+        field_data = np.zeros(100, dtype=list(zip(names, dt)))
+        field_data["fieldRA"] = rng.rand(100)
+        field_data["fieldDec"] = rng.rand(100)
+        field_data["fieldId"] = np.arange(100)
+
+        slicer = slicers.UserPointsSlicer(ra=field_data["fieldRA"], dec=field_data["fieldDec"])
+
+        names = ["data1", "data2", "fieldId"]
+        sim_data = np.zeros(100, dtype=list(zip(names, dt)))
+        sim_data["data1"] = rng.rand(100)
+        sim_data["fieldId"] = np.arange(100)
+        with TemporaryFile() as filename:
+            slicer.write_data(filename, metric_values)
+            _ = filename.seek(0)
+            metric_back, slicer_back, header = self.baseslicer.read_data(filename)
+            assert slicer == slicer_back
+            np.testing.assert_almost_equal(metric_back, metric_values)
+            attr2check = [
+                "nslice",
+                "columns_needed",
+                "lon_col",
+                "lat_col",
+                "shape",
+            ]
+            for att in attr2check:
+                if type(getattr(slicer, att)).__name__ == "dict":
+                    for key in getattr(slicer, att):
+                        np.testing.assert_almost_equal(
+                            getattr(slicer, att)[key], getattr(slicer_back, att)[key]
+                        )
+                else:
+                    assert getattr(slicer, att) == getattr(slicer_back, att)
+
+    def test_unislicer(self):
+        rng = np.random.RandomState(34229)
+        slicer = slicers.UniSlicer()
+        data = np.zeros(1, dtype=[("testdata", "float")])
+        data[:] = rng.rand(1)
+        slicer.setup_slicer(data)
+        with TemporaryFile() as filename:
+            metric_value = np.array([25.0])
+            slicer.write_data(filename, metric_value)
+            _ = filename.seek(0)
+            data_back, slicer_back, header = self.baseslicer.read_data(filename)
+            assert slicer == slicer_back
+            np.testing.assert_almost_equal(data_back, metric_value)
+            attr2check = ["nslice", "columns_needed"]
+            for att in attr2check:
+                assert getattr(slicer, att) == getattr(slicer_back, att)
+
+    def test_complex(self):
+        # Test case where there is a complex metric
+        rng = np.random.RandomState(5442)
+        nside = 8
+        slicer = slicers.HealpixSlicer(nside=nside)
+        data = np.zeros(slicer.nslice, dtype="object")
+        for i, ack in enumerate(data):
+            n_el = rng.rand(1) * 4  # up to 4 elements
+            data[i] = np.arange(n_el[0])
+        with TemporaryFile() as filename:
+            slicer.write_data(filename, data)
+            _ = filename.seek(0)
+            data_back, slicer_back, header = self.baseslicer.read_data(filename)
+            assert slicer == slicer_back
+            # This is a crazy slow loop!
+            for i, ack in enumerate(data):
+                np.testing.assert_almost_equal(data_back[i], data[i])
+
+    def test_n_d_slicer(self):
+        rng = np.random.RandomState(621)
+        colnames = ["test1", "test2", "test3"]
+        n = 1000
+        dv = np.zeros(n, dtype=list(zip(colnames, [float] * 3)))
+        for c in colnames:
+            dv[c] = rng.rand(n)
+        slicer = slicers.NDSlicer(colnames, bins_list=10)
+        slicer.setup_slicer(dv)
+        with TemporaryFile() as filename:
+            metricdata = np.zeros(slicer.nslice, dtype="float")
+            for i, s in enumerate(slicer):
+                metricdata[i] = i
+            slicer.write_data(filename, metricdata)
+            _ = filename.seek(0)
+            data_back, slicer_back, header = self.baseslicer.read_data(filename)
+            assert slicer == slicer_back
+            np.testing.assert_almost_equal(data_back, metricdata)
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/maf/test_json.py

@@ -0,0 +1,169 @@
+import json
+import unittest
+
+import numpy as np
+import numpy.ma as ma
+
+import rubin_sim.maf.slicers as slicers
+
+
+def make_data_values(size=100, min=0.0, max=1.0, random=-1):
+    """Generate a simple array of numbers, evenly arranged between min/max,
+    but (optional) random order."""
+    datavalues = np.arange(0, size, dtype="float")
+    datavalues *= (float(max) - float(min)) / (datavalues.max() - datavalues.min())
+    datavalues += min
+    if random > 0:
+        rng = np.random.RandomState(random)
+        randorder = rng.rand(size)
+        randind = np.argsort(randorder)
+        datavalues = datavalues[randind]
+    datavalues = np.array(list(zip(datavalues)), dtype=[("testdata", "float")])
+    return datavalues
+
+
+def make_metric_data(slicer, dtype="float", seed=8800):
+    rng = np.random.RandomState(seed)
+    metric_values = rng.rand(len(slicer)).astype(dtype)
+    metric_values = ma.MaskedArray(
+        data=metric_values, mask=np.zeros(len(slicer), "bool"), fill_value=slicer.badval
+    )
+    return metric_values
+
+
+def make_field_data():
+    """Set up sample field data."""
+    # These are a subset of the fields from opsim.
+    field_id = [2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010]
+    ra_rad = [
+        1.4961071750760884,
+        4.009380232682723,
+        2.2738050744968632,
+        2.7527439701957053,
+        6.043715459855715,
+        0.23946974745438585,
+        3.4768050063149119,
+        2.8063803008646744,
+        4.0630173623005916,
+        2.2201678117208452,
+    ]
+    dec_rad = [
+        -0.25205231807872636,
+        -0.25205228478831621,
+        -0.25205228478831621,
+        -0.25205228478831621,
+        -0.25205145255075168,
+        -0.25205145255075168,
+        -0.24630904473998308,
+        -0.24630904473998308,
+        -0.24630894487049795,
+        -0.24630894487049795,
+    ]
+    field_id = np.array(field_id, "int")
+    ra_rad = np.array(ra_rad, "float")
+    dec_rad = np.array(dec_rad, "float")
+    field_data = np.core.records.fromarrays(
+        [field_id, ra_rad, dec_rad], names=["field_id", "fieldRA", "fieldDec"]
+    )
+    return field_data
+
+
+def make_opsim_data_values(field_data, size=10000, min=0.0, max=1.0, random=88):
+    """Generate a simple array of numbers, evenly arranged between min/max,
+    but (optional) random order."""
+    datavalues = np.arange(0, size, dtype="float")
+    datavalues *= (float(max) - float(min)) / (datavalues.max() - datavalues.min())
+    datavalues += min
+    rng = np.random.RandomState(random)
+    randorder = rng.rand(size)
+    randind = np.argsort(randorder)
+    datavalues = datavalues[randind]
+    # Add valid field_id values to match data values
+    field_id = np.zeros(len(datavalues), "int")
+    idxs = rng.rand(size) * len(field_data["field_id"])
+    for i, d in enumerate(datavalues):
+        field_id[i] = field_data[int(idxs[i])][0]
+    sim_data = np.core.records.fromarrays([field_id, datavalues], names=["field_id", "testdata"])
+    return sim_data
+
+
+class TestJSONoutUniSlicer(unittest.TestCase):
+    def setUp(self):
+        self.testslicer = slicers.UniSlicer()
+
+    def tearDown(self):
+        del self.testslicer
+
+    def test(self):
+        metric_val = make_metric_data(self.testslicer, "float", seed=88102231)
+        io = self.testslicer.output_json(
+            metric_val,
+            metric_name="testMetric",
+            sim_data_name="testSim",
+            info_label="testmeta",
+        )
+        jsn = json.loads(io.getvalue())
+        jsn_header = jsn[0]
+        jsn_data = jsn[1]
+        self.assertEqual(jsn_header["metric_name"], "testMetric")
+        self.assertEqual(jsn_header["sim_data_name"], "testSim")
+        self.assertEqual(jsn_header["info_label"], "testmeta")
+        self.assertEqual(jsn_header["slicer_name"], "UniSlicer")
+        self.assertEqual(jsn_header["slicer_len"], 1)
+        self.assertEqual(len(jsn_data), 1)
+
+
+class TestJSONoutOneDSlicer2(unittest.TestCase):
+    def setUp(self):
+        # Set up a slicer and some metric data for that slicer.
+        dv = make_data_values(1000, random=40082)
+        self.testslicer = slicers.OneDSlicer(slice_col_name="testdata")
+        self.testslicer.setup_slicer(dv)
+
+    def tearDown(self):
+        del self.testslicer
+
+    def test(self):
+        metric_val = make_metric_data(self.testslicer, "float", seed=18)
+        io = self.testslicer.output_json(metric_val)
+        jsn = json.loads(io.getvalue())
+        jsn_header = jsn[0]
+        jsn_data = jsn[1]
+        self.assertEqual(jsn_header["slicer_name"], "OneDSlicer")
+        self.assertEqual(jsn_header["slicer_len"], len(self.testslicer))
+        self.assertEqual(len(jsn_data), len(metric_val) + 1)
+        for jsndat, bin_left, mval in zip(jsn_data, self.testslicer.slice_points["bins"], metric_val.data):
+            self.assertEqual(jsndat[0], bin_left)
+            self.assertEqual(jsndat[1], mval)
+
+
+class TestJSONoutHealpixSlicer(unittest.TestCase):
+    def setUp(self):
+        # Set up a slicer and some metric data for that slicer.
+        self.testslicer = slicers.HealpixSlicer(nside=4, verbose=False)
+
+    def tearDown(self):
+        del self.testslicer
+
+    def test(self):
+        metric_val = make_metric_data(self.testslicer, "float", seed=452)
+        io = self.testslicer.output_json(metric_val)
+        jsn = json.loads(io.getvalue())
+        jsn_header = jsn[0]
+        jsn_data = jsn[1]
+        self.assertEqual(jsn_header["slicer_name"], "HealpixSlicer")
+        self.assertEqual(jsn_header["slicer_len"], len(self.testslicer))
+        self.assertEqual(len(jsn_data), len(metric_val))
+        for jsndat, ra, dec, mval in zip(
+            jsn_data,
+            self.testslicer.slice_points["ra"],
+            self.testslicer.slice_points["dec"],
+            metric_val.data,
+        ):
+            self.assertAlmostEqual(jsndat[0], ra / np.pi * 180.0)
+            self.assertAlmostEqual(jsndat[1], dec / np.pi * 180.0)
+            self.assertEqual(jsndat[2], mval)
+
+
+if __name__ == "__main__":
+    unittest.main()