astro-otter 0.3.2__tar.gz → 0.3.4__tar.gz

{astro_otter-0.3.2/src/astro_otter.egg-info → astro_otter-0.3.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: astro-otter
-Version: 0.3.2
+Version: 0.3.4
 Author-email: Noah Franz <nfranz@arizona.edu>
 License: MIT License
         
@@ -54,7 +54,6 @@ Requires-Dist: skypatrol
 Requires-Dist: fundamentals
 Requires-Dist: astro-datalab
 Requires-Dist: sparclclient
-Requires-Dist: astro-ghost
 Requires-Dist: pydantic
 Requires-Dist: pyreadline3; platform_system == "Windows"
 Provides-Extra: docs

{astro_otter-0.3.2 → astro_otter-0.3.4}/pyproject.toml

@@ -46,7 +46,6 @@ dependencies = [
   "fundamentals",
   "astro-datalab",
   "sparclclient",
-  "astro-ghost",
 
   # for the schema validation
   "pydantic",

{astro_otter-0.3.2 → astro_otter-0.3.4/src/astro_otter.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: astro-otter
-Version: 0.3.2
+Version: 0.3.4
 Author-email: Noah Franz <nfranz@arizona.edu>
 License: MIT License
         
@@ -54,7 +54,6 @@ Requires-Dist: skypatrol
 Requires-Dist: fundamentals
 Requires-Dist: astro-datalab
 Requires-Dist: sparclclient
-Requires-Dist: astro-ghost
 Requires-Dist: pydantic
 Requires-Dist: pyreadline3; platform_system == "Windows"
 Provides-Extra: docs

{astro_otter-0.3.2 → astro_otter-0.3.4}/src/astro_otter.egg-info/SOURCES.txt

@@ -24,5 +24,7 @@ tests/test_exceptions.py
 tests/test_host.py
 tests/test_otter.py
 tests/test_package.py
+tests/test_plotter.py
+tests/test_schema.py
 tests/test_transient.py
 tests/test_util.py

{astro_otter-0.3.2 → astro_otter-0.3.4}/src/astro_otter.egg-info/requires.txt

@@ -13,7 +13,6 @@ skypatrol
 fundamentals
 astro-datalab
 sparclclient
-astro-ghost
 pydantic
 
 [:platform_system == "Windows"]

{astro_otter-0.3.2 → astro_otter-0.3.4}/src/otter/_version.py

@@ -2,4 +2,4 @@
 Just define the package version in one place
 """
 
-__version__ = "0.3.2"
+__version__ = "0.3.4"

{astro_otter-0.3.2 → astro_otter-0.3.4}/src/otter/io/data_finder.py

@@ -602,8 +602,16 @@ class DataFinder(object):
         cone_search_res = qc.query(adql=adql, fmt="pandas")
 
         # then retrieve all of the spectra corresponding to those sparcl_ids
-        sparcl_ids = cone_search_res.sparcl_id.tolist()
-        res = client.retrieve(uuid_list=sparcl_ids, include=include)
+        spec_ids = cone_search_res.targetid.tolist()
+        if len(spec_ids) == 0:
+            logger.warn("Object not found in Sparcl!")
+            return
+
+        res = client.retrieve_by_specid(spec_ids, include=include)
+        if res.count == 0:
+            logger.warn("No Spectra available in sparcl!")
+            return
+
         all_spec = pd.concat([pd.DataFrame([record]) for record in res.records])
         return Table.from_pandas(all_spec)
 

{astro_otter-0.3.2 → astro_otter-0.3.4}/src/otter/io/otter.py

@@ -154,6 +154,7 @@ class Otter(Database):
         keep_raw=False,
         wave_unit="nm",
         freq_unit="GHz",
+        deduplicate=None,
         **kwargs,
     ) -> Table:
         """
@@ -176,6 +177,11 @@ class Otter(Database):
                              is False.
             wave_unit (str): The astropy wavelength unit to return with
             freq_unit (str): The astropy frequency unit to return with`
+            deduplicate (Callable|None|False): if we should deduplicate the dataset
+                                               using the deduplicate Callable. Set to
+                                               False if you don't want this to happen.
+                                               None defaults to
+                                               Transient.deduplicate_photometry
             **kwargs : Arguments to pass to Otter.query(). Can be::
 
                        names (list[str]): A list of names to get the metadata for
@@ -215,6 +221,7 @@ class Otter(Database):
                     wave_unit=wave_unit,
                     freq_unit=freq_unit,
                     obs_type=obs_type,
+                    deduplicate=deduplicate,
                 )
 
                 phot["name"] = [default_name] * len(phot)
@@ -291,8 +298,13 @@ class Otter(Database):
         radius: float = 5,
         minz: float = None,
         maxz: float = None,
+        mindec: float = -90,
+        maxdec: float = 90,
         refs: list[str] = None,
         hasphot: bool = False,
+        has_radio_phot: bool = False,
+        has_uvoir_phot: bool = False,
+        has_xray_phot: bool = False,
         hasspec: bool = False,
         spec_classed: bool = False,
         unambiguous: bool = False,
@@ -316,9 +328,14 @@ class Otter(Database):
             radius (float): The radius in arcseconds for a cone search, default is 0.05"
             minz (float): The minimum redshift to search for
             maxz (float): The maximum redshift to search for
+            mindec (float): The minimum declination in degrees
+            maxdec (float): Tje maximum declination in degrees
             refs (list[str]): A list of ads bibcodes to match to. Will only return
                               metadata for transients that have this as a reference.
             hasphot (bool): if True, only returns transients which have photometry.
+            has_radio_phot (bool): if True, only returns transients with radio phot.
+            has_uvoir_phot (bool): if True, only returns transients with uvoir phot.
+            has_xray_phot (bool): if True, only returns transients with X-ray phot.
             hasspec (bool): NOT IMPLEMENTED! Will return False for all targets!
             spec_classed (bool): If True, only returns transients that have been
                                  specotroscopically classified/confirmed
@@ -337,9 +354,18 @@ class Otter(Database):
         # write some AQL filters based on the inputs
         query_filters = ""
 
-        if hasphot is True:
+        if hasphot or has_radio_phot or has_xray_phot or has_uvoir_phot:
             query_filters += "FILTER 'photometry' IN ATTRIBUTES(transient)\n"
 
+        if has_radio_phot:
+            query_filters += "FILTER 'radio' IN transient.photometry[*].obs_type\n"
+
+        if has_uvoir_phot:
+            query_filters += "FILTER 'uvoir' IN transient.photometry[*].obs_type\n"
+
+        if has_xray_phot:
+            query_filters += "FILTER 'xray' IN transient.photometry[*].obs_type\n"
+
         if hasspec is True:
             query_filters += "FILTER 'spectra' IN ATTRIBUTES(transient)\n"
 
@@ -451,6 +477,11 @@ class Otter(Database):
         else:
             arango_query_results = [Transient(res) for res in result.result]
 
+        # filter based on the min and max declination query options
+        decs = np.array([t.get_skycoord().dec.deg for t in arango_query_results])
+        where_dec = np.where((decs > mindec) * (decs < maxdec))[0]
+        arango_query_results = [arango_query_results[i] for i in where_dec]
+
         if not query_private:
             return arango_query_results
 
@@ -665,7 +696,10 @@ class Otter(Database):
             if len(res) > 1:
                 raise OtterLimitationError("Some objects in Otter are too close!")
 
-            elif len(res) == 1:
+            elif len(res) == 1 and collection != "vetting":
+                # if the collection is the vetting collection we don't want to do the
+                # merging yet, even if the object already exists in OTTER
+
                 # this object exists in otter already, let's grab the transient data and
                 # merge the files
                 merged = t + res[0]
@@ -674,13 +708,6 @@ class Otter(Database):
                 merged["_key"] = res[0]["_key"]
                 merged["_id"] = res[0]["_id"]
 
-                # we also have to delete the document from the OTTER database
-                doc = self.fetchDocument(merged["_id"])
-                if not testing:
-                    doc.delete()
-                else:
-                    print(f"Would delete\n{doc}")
-
             else:
                 # this means the object doesn't exist in otter already
                 merged = t
@@ -832,7 +859,7 @@ class Otter(Database):
     def from_csvs(
         metafile: str,
         photfile: str = None,
-        local_outpath: Optional[str] = None,
+        local_outpath: Optional[str] = os.path.join(os.getcwd(), "private-data"),
         db: Otter = None,
     ) -> Otter:
         """
@@ -973,11 +1000,13 @@ class Otter(Database):
                 pd.isna(tde["luminosity_distance"])
             ):
                 json["distance"].append(
-                    value=tde.luminosity_distance[0],
-                    reference=[tde.luminosity_distance_bibcode[0]],
-                    unit=tde.luminosity_distance_unit[0],
-                    computed=False,
-                    distance_type="luminosity",
+                    dict(
+                        value=tde.luminosity_distance[0],
+                        reference=[tde.luminosity_distance_bibcode[0]],
+                        unit=tde.luminosity_distance_unit[0],
+                        computed=False,
+                        distance_type="luminosity",
+                    )
                 )
 
             # comoving distance
@@ -985,11 +1014,13 @@ class Otter(Database):
                 pd.isna(tde["comoving_distance"])
             ):
                 json["distance"].append(
-                    value=tde.comoving_distance[0],
-                    reference=[tde.comoving_distance_bibcode[0]],
-                    unit=tde.comoving_distance_unit[0],
-                    computed=False,
-                    distance_type="comoving",
+                    dict(
+                        value=tde.comoving_distance[0],
+                        reference=[tde.comoving_distance_bibcode[0]],
+                        unit=tde.comoving_distance_unit[0],
+                        computed=False,
+                        distance_type="comoving",
+                    )
                 )
 
             # remove the distance list if it is empty still
@@ -1081,11 +1112,6 @@ class Otter(Database):
                     if src not in phot_sources:
                         phot_sources.append(src)
 
-                    if len(np.unique(p.flux_unit)) == 1:
-                        raw_units = p.flux_unit.tolist()[0]
-                    else:
-                        raw_units = p.flux_unit.tolist()
-
                     # add a column to phot with the unique filter key
                     if obstype == "radio":
                         filter_uq_key = (
@@ -1105,17 +1131,73 @@ class Otter(Database):
                     if "upperlimit" not in p:
                         p["upperlimit"] = False
 
-                    json_phot = dict(
-                        reference=src,
-                        raw=p.flux.astype(float).tolist(),
-                        raw_err=p.flux_err.astype(float).tolist(),
-                        raw_units=raw_units,
-                        date=p.date.tolist(),
-                        date_format=p.date_format.tolist(),
-                        upperlimit=p.upperlimit.tolist(),
-                        filter_key=filter_uq_key,
-                        obs_type=obstype,
-                    )
+                    if "raw" in p.columns and "flux" in p.columns:
+                        if len(np.unique(p.raw_unit)) == 1:
+                            raw_units = p.raw_unit.tolist()[0]
+                        else:
+                            raw_units = p.raw_unit.tolist()
+
+                        if len(np.unique(p.flux_unit)) == 1:
+                            val_units = p.flux_unit.tolist()[0]
+                        else:
+                            val_units = p.flux_unit.tolist()
+
+                        # treat "raw" as the "raw" keyword and "flux" as the "value"
+                        json_phot = dict(
+                            reference=src,
+                            raw=p.raw.astype(float).tolist(),
+                            raw_err=p.raw_err.astype(float).tolist(),
+                            raw_units=raw_units,
+                            value=p.flux.astype(float).tolist(),
+                            value_err=p.flux_err.astype(float).tolist(),
+                            value_units=val_units,
+                            date=p.date.tolist(),
+                            date_format=p.date_format.tolist(),
+                            upperlimit=p.upperlimit.tolist(),
+                            filter_key=filter_uq_key,
+                            obs_type=obstype,
+                        )
+
+                    elif "flux" in p.columns and "raw" not in p.columns:
+                        if len(np.unique(p.flux_unit)) == 1:
+                            raw_units = p.flux_unit.tolist()[0]
+                        else:
+                            raw_units = p.flux_unit.tolist()
+
+                        # treat "flux" as the "raw" keyword
+                        json_phot = dict(
+                            reference=src,
+                            raw=p.flux.astype(float).tolist(),
+                            raw_err=p.flux_err.astype(float).tolist(),
+                            raw_units=raw_units,
+                            date=p.date.tolist(),
+                            date_format=p.date_format.tolist(),
+                            upperlimit=p.upperlimit.tolist(),
+                            filter_key=filter_uq_key,
+                            obs_type=obstype,
+                        )
+
+                    elif "raw" in p.columns and "flux" not in p.columns:
+                        if len(np.unique(p.raw_unit)) == 1:
+                            raw_units = p.raw_unit.tolist()[0]
+                        else:
+                            raw_units = p.raw_unit.tolist()
+
+                        # treat "raw" as the "raw" keyword
+                        json_phot = dict(
+                            reference=src,
+                            raw=p.raw.astype(float).tolist(),
+                            raw_err=p.raw_err.astype(float).tolist(),
+                            raw_units=raw_units,
+                            date=p.date.tolist(),
+                            date_format=p.date_format.tolist(),
+                            upperlimit=p.upperlimit.tolist(),
+                            filter_key=filter_uq_key,
+                            obs_type=obstype,
+                        )
+
+                    else:
+                        raise ValueError("`raw` and/or `flux` key(s) must be provided!")
 
                     if not pd.isna(tele):
                         json_phot["telescope"] = tele
@@ -1131,6 +1213,13 @@ class Otter(Database):
                             "Minimum and maximum filters required for X-ray data!"
                         )
 
+                    if ("date_min" in p and "date_max" not in p) or (
+                        "date_min" not in p and "date_max" in p
+                    ):
+                        raise ValueError(
+                            "If date_min/date_max is provided, the other must be too!"
+                        )
+
                     # check optional keys
                     optional_keys = [
                         "date_err",
@@ -1142,6 +1231,8 @@ class Otter(Database):
                         "observer",
                         "reducer",
                         "pipeline",
+                        "date_min",
+                        "date_max",
                     ]
                     for k in optional_keys:
                         if k in p and not np.all(pd.isna(p[k])):
@@ -1154,7 +1245,13 @@ class Otter(Database):
 
                     # handle more detailed uncertainty information
                     raw_err_detail = {}
-                    for key in ["statistical_err", "systematic_err", "iss_err"]:
+                    for key in [
+                        "statistical_err",
+                        "systematic_err",
+                        "iss_err",
+                        "upper_err",
+                        "lower_err",
+                    ]:
                         if key in p and not np.all(pd.isna(p[key])):
                             k = key.split("_")[0]
 
@@ -1183,6 +1280,126 @@ class Otter(Database):
                             json_phot[c] = p[c].tolist()
                             json_phot[bool_v_key] = [v != "null" for v in json_phot[c]]
 
+                    # deal with the xray model
+                    if "xray_model_name" not in p and obstype == "xray":
+                        raise ValueError(
+                            "You must provide the xray model for xray data!"
+                        )
+                    if obstype == "xray" and "xray_model_name" in p:
+                        # get various sets of keywords
+                        model_val_cols = list(
+                            p.columns[p.columns.str.contains("xray_model_param_value")]
+                        )
+                        model_up_err_cols = list(
+                            p.columns[p.columns.str.contains("xray_model_param_up_err")]
+                        )
+                        model_lo_err_cols = list(
+                            p.columns[p.columns.str.contains("xray_model_param_lo_err")]
+                        )
+                        model_val_units_cols = list(
+                            p.columns[p.columns.str.contains("xray_model_param_unit")]
+                        )
+                        model_uplim_cols = list(
+                            p.columns[
+                                p.columns.str.contains("xray_model_param_upperlimit")
+                            ]
+                        )
+
+                        param_names = [v.split("::")[-1] for v in model_val_cols]
+
+                        xray_model_info = p[
+                            model_val_cols
+                            + model_up_err_cols
+                            + model_lo_err_cols
+                            + model_val_units_cols
+                            + model_uplim_cols
+                            + [
+                                "xray_model_name",
+                                "xray_model_reference",
+                                "filter_min",
+                                "filter_max",
+                                "filter_eff_units",
+                            ]
+                        ]
+
+                        if len(model_uplim_cols) == 0:
+                            # assume they are all false
+                            for param_name in param_names:
+                                colname = f"xray_model_param_upperlimit::{param_name}"
+                                xray_model_info[colname] = False
+                                model_uplim_cols.append(colname)
+
+                        if not all(
+                            len(model_val_cols) == len(p)
+                            for p in [
+                                model_up_err_cols,
+                                model_lo_err_cols,
+                                model_val_units_cols,
+                                model_uplim_cols,
+                            ]
+                        ):
+                            raise ValueError(
+                                "Missing a column for one of the X-ray parameters!"
+                            )
+
+                        xray_models = []
+                        for _, row in xray_model_info.iterrows():
+                            energy1 = (
+                                (row["filter_min"] * u.Unit(row["filter_eff_units"]))
+                                .to("keV", equivalencies=u.spectral())
+                                .value
+                            )
+                            energy2 = (
+                                (row["filter_max"] * u.Unit(row["filter_eff_units"]))
+                                .to("keV", equivalencies=u.spectral())
+                                .value
+                            )
+
+                            if energy1 > energy2:
+                                min_energy = energy2
+                                max_energy = energy1
+                            else:
+                                min_energy = energy1
+                                max_energy = energy2
+
+                            param_names_not_na = []
+                            for n in param_names:
+                                if not pd.isna(row[f"xray_model_param_value::{n}"]):
+                                    param_names_not_na.append(n)
+
+                            xray_models.append(
+                                {
+                                    "model_name": row.xray_model_name,
+                                    "param_names": [n for n in param_names_not_na],
+                                    "param_values": [
+                                        row[f"xray_model_param_value::{n}"]
+                                        for n in param_names_not_na
+                                    ],
+                                    "param_value_upper_err": [
+                                        row[f"xray_model_param_up_err::{n}"]
+                                        for n in param_names_not_na
+                                    ],
+                                    "param_value_lower_err": [
+                                        row[f"xray_model_param_lo_err::{n}"]
+                                        for n in param_names_not_na
+                                    ],
+                                    "param_upperlimit": [
+                                        row[f"xray_model_param_upperlimit::{n}"]
+                                        for n in param_names_not_na
+                                    ],
+                                    "param_units": [
+                                        row[f"xray_model_param_unit::{n}"]
+                                        for n in param_names_not_na
+                                    ],
+                                    "model_reference": row["xray_model_reference"],
+                                    "min_energy": min_energy,
+                                    "max_energy": max_energy,
+                                    "energy_units": "keV",
+                                }
+                            )
+
+                        json_phot["xray_model"] = xray_models
+
                     json["photometry"].append(json_phot)
 
                 tde["filter_uq_key"] = pd.Series(
@@ -1191,14 +1408,20 @@ class Otter(Database):
 
                 # filter alias
                 # radio filters first
-                filter_keys1 = ["filter_uq_key", "band_eff_wave", "band_eff_wave_unit"]
+                filter_keys1 = [
+                    "filter_uq_key",
+                    "band_eff_wave",
+                    "band_eff_wave_unit",
+                    "filter_eff_units",
+                ]
                 if "filter_min" in tde:
                     filter_keys1.append("filter_min")
                 if "filter_max" in tde:
                     filter_keys1.append("filter_max")
 
-                filter_map = (
-                    tde[filter_keys1].drop_duplicates().set_index("filter_uq_key")
+                filt_info = tde[filter_keys1]
+                filter_map = filt_info.drop_duplicates().set_index(
+                    "filter_uq_key"
                 )  # .to_dict(orient='index')
                 try:
                     filter_map_radio = filter_map.to_dict(orient="index")
@@ -1229,22 +1452,24 @@ class Otter(Database):
                         wave_units=val["band_eff_wave_unit"],
                     )
 
-                    if "filter_min" in val:
+                    if "filter_min" in val and not pd.isna(val["filter_min"]):
                         filter_alias_dict["wave_min"] = (
-                            val["filter_min"] * u.Unit(phot.filter_eff_units)
-                        ).to(
-                            u.Unit(
-                                val["band_eff_wave_unit"], equivalencies=u.spectral()
+                            (val["filter_min"] * u.Unit(val["filter_eff_units"]))
+                            .to(
+                                u.Unit(val["band_eff_wave_unit"]),
+                                equivalencies=u.spectral(),
                             )
+                            .value
                         )
 
-                    if "filter_max" in val:
+                    if "filter_max" in val and not pd.isna(val["filter_max"]):
                         filter_alias_dict["wave_max"] = (
-                            val["filter_max"] * u.Unit(phot.filter_eff_units)
-                        ).to(
-                            u.Unit(
-                                val["band_eff_wave_unit"], equivalencies=u.spectral()
+                            (val["filter_max"] * u.Unit(val["filter_eff_units"]))
+                            .to(
+                                u.Unit(val["band_eff_wave_unit"]),
+                                equivalencies=u.spectral(),
                             )
+                            .value
                         )
 
                     json["filter_alias"].append(filter_alias_dict)