astro-otter 0.3.0__tar.gz → 0.3.3__tar.gz

{astro_otter-0.3.0/src/astro_otter.egg-info → astro_otter-0.3.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: astro-otter
-Version: 0.3.0
+Version: 0.3.3
 Author-email: Noah Franz <nfranz@arizona.edu>
 License: MIT License
         
@@ -45,6 +45,7 @@ Requires-Dist: pandas
 Requires-Dist: synphot
 Requires-Dist: typing-extensions
 Requires-Dist: pyarango
+Requires-Dist: tabulate
 Requires-Dist: matplotlib
 Requires-Dist: plotly
 Requires-Dist: astroquery
@@ -53,7 +54,6 @@ Requires-Dist: skypatrol
 Requires-Dist: fundamentals
 Requires-Dist: astro-datalab
 Requires-Dist: sparclclient
-Requires-Dist: astro-ghost
 Requires-Dist: pydantic
 Requires-Dist: pyreadline3; platform_system == "Windows"
 Provides-Extra: docs
@@ -117,6 +117,7 @@ python3 -m pip install astro-otter
    ```
    git clone https://github.com/astro-otter/otter.git $OTTER_ROOT/otter
    git clone https://github.com/astro-otter/otterdb.git $OTTER_ROOT/otterdb
+   git clone https://github.com/astro-otter/otter-web.git $OTTER_ROOT/otter-web
    ```
 3. Install the NASA ADS Python API by following the instructions at https://ads.readthedocs.io/en/latest/#getting-started
 4. Install otter, the API for this database. From
@@ -125,13 +126,7 @@ python3 -m pip install astro-otter
    cd $OTTER_ROOT/otter
    python -m pip install -e .
    ```
-5. Process the data to build the local "database" (although it is really just a directory).
-   Then, you can build the "database" by running the
-   following commands:
-   ```
-   cd $OTTER_ROOT/otter/scripts/
-   python3 gen_summary_table.py --otterroot $OTTER_ROOT
-   ```
+5. Process the data to build the local copy of the database. Follow the instructions in the otterdb repo README.
 6. Easily access the data using the Otter code! In python:
   ```
   import os

{astro_otter-0.3.0 → astro_otter-0.3.3}/README.md

@@ -45,6 +45,7 @@ python3 -m pip install astro-otter
    ```
    git clone https://github.com/astro-otter/otter.git $OTTER_ROOT/otter
    git clone https://github.com/astro-otter/otterdb.git $OTTER_ROOT/otterdb
+   git clone https://github.com/astro-otter/otter-web.git $OTTER_ROOT/otter-web
    ```
 3. Install the NASA ADS Python API by following the instructions at https://ads.readthedocs.io/en/latest/#getting-started
 4. Install otter, the API for this database. From
@@ -53,13 +54,7 @@ python3 -m pip install astro-otter
    cd $OTTER_ROOT/otter
    python -m pip install -e .
    ```
-5. Process the data to build the local "database" (although it is really just a directory).
-   Then, you can build the "database" by running the
-   following commands:
-   ```
-   cd $OTTER_ROOT/otter/scripts/
-   python3 gen_summary_table.py --otterroot $OTTER_ROOT
-   ```
+5. Process the data to build the local copy of the database. Follow the instructions in the otterdb repo README.
 6. Easily access the data using the Otter code! In python:
   ```
   import os

{astro_otter-0.3.0 → astro_otter-0.3.3}/pyproject.toml

@@ -33,6 +33,7 @@ dependencies = [
   "synphot",
   "typing-extensions",
   "pyarango",
+  "tabulate",
 
   # for the plotting
   "matplotlib",
@@ -45,7 +46,6 @@ dependencies = [
   "fundamentals",
   "astro-datalab",
   "sparclclient",
-  "astro-ghost",
 
   # for the schema validation
   "pydantic",

{astro_otter-0.3.0 → astro_otter-0.3.3/src/astro_otter.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: astro-otter
-Version: 0.3.0
+Version: 0.3.3
 Author-email: Noah Franz <nfranz@arizona.edu>
 License: MIT License
         
@@ -45,6 +45,7 @@ Requires-Dist: pandas
 Requires-Dist: synphot
 Requires-Dist: typing-extensions
 Requires-Dist: pyarango
+Requires-Dist: tabulate
 Requires-Dist: matplotlib
 Requires-Dist: plotly
 Requires-Dist: astroquery
@@ -53,7 +54,6 @@ Requires-Dist: skypatrol
 Requires-Dist: fundamentals
 Requires-Dist: astro-datalab
 Requires-Dist: sparclclient
-Requires-Dist: astro-ghost
 Requires-Dist: pydantic
 Requires-Dist: pyreadline3; platform_system == "Windows"
 Provides-Extra: docs
@@ -117,6 +117,7 @@ python3 -m pip install astro-otter
    ```
    git clone https://github.com/astro-otter/otter.git $OTTER_ROOT/otter
    git clone https://github.com/astro-otter/otterdb.git $OTTER_ROOT/otterdb
+   git clone https://github.com/astro-otter/otter-web.git $OTTER_ROOT/otter-web
    ```
 3. Install the NASA ADS Python API by following the instructions at https://ads.readthedocs.io/en/latest/#getting-started
 4. Install otter, the API for this database. From
@@ -125,13 +126,7 @@ python3 -m pip install astro-otter
    cd $OTTER_ROOT/otter
    python -m pip install -e .
    ```
-5. Process the data to build the local "database" (although it is really just a directory).
-   Then, you can build the "database" by running the
-   following commands:
-   ```
-   cd $OTTER_ROOT/otter/scripts/
-   python3 gen_summary_table.py --otterroot $OTTER_ROOT
-   ```
+5. Process the data to build the local copy of the database. Follow the instructions in the otterdb repo README.
 6. Easily access the data using the Otter code! In python:
   ```
   import os

{astro_otter-0.3.0 → astro_otter-0.3.3}/src/astro_otter.egg-info/requires.txt

@@ -4,6 +4,7 @@ pandas
 synphot
 typing-extensions
 pyarango
+tabulate
 matplotlib
 plotly
 astroquery
@@ -12,7 +13,6 @@ skypatrol
 fundamentals
 astro-datalab
 sparclclient
-astro-ghost
 pydantic
 
 [:platform_system == "Windows"]

{astro_otter-0.3.0 → astro_otter-0.3.3}/src/otter/_version.py

@@ -2,4 +2,4 @@
 Just define the package version in one place
 """
 
-__version__ = "0.3.0"
+__version__ = "0.3.3"

{astro_otter-0.3.0 → astro_otter-0.3.3}/src/otter/io/otter.py

@@ -3,11 +3,12 @@ This is the primary class for user interaction with the catalog
 """
 
 from __future__ import annotations
+from typing import Optional
 import os
 import json
 import glob
-from warnings import warn
 from copy import deepcopy
+import logging
 
 from pyArango.connection import Connection
 from pyArango.database import Database
@@ -22,7 +23,7 @@ from astropy import units as u
 
 from .transient import Transient
 from ..exceptions import FailedQueryError, OtterLimitationError, TransientMergeError
-from ..util import bibcode_to_hrn, freq_to_obstype, freq_to_band
+from ..util import bibcode_to_hrn, freq_to_obstype, freq_to_band, _DuplicateFilter
 
 import warnings
 
@@ -30,6 +31,8 @@ warnings.simplefilter("once", RuntimeWarning)
 warnings.simplefilter("once", UserWarning)
 warnings.simplefilter("once", u.UnitsWarning)
 
+logger = logging.getLogger(__name__)
+
 
 def _np_encoder(object):
     """
@@ -44,23 +47,34 @@ class Otter(Database):
     This is the primary class for users to access the otter backend database
 
     Args:
+        url (str): The url where the database api endpoints are located
+        username (str): The username to log into the database with
+        password (str): The password to log into the database with
+        gen_summary (bool): Generate a local summary table, this should generally be
+                            left as False!
         datadir (str): Path to the data directory with the otter data. If not provided
                        will default to a ".otter" directory in the CWD where you call
                        this class from.
         debug (bool): If we should just debug and not do anything serious.
 
+    Returns:
+        An Otter object that is connected to the otter database
     """
 
     def __init__(
         self,
         url: str = "http://127.0.0.1:8529",
-        username: str = "user-guest",
-        password: str = "",
+        username: str = os.environ.get("ARANGO_USER_USERNAME", "user-guest"),
+        password: str = os.environ.get("ARANGO_USER_PASSWORD", ""),
         gen_summary: bool = False,
         datadir: str = None,
         debug: bool = False,
         **kwargs,
     ) -> None:
+        print("Attempting to login with the following credentials:")
+        print(f"username: {username}")
+        print(f"password: {password}")
+
         # save inputs
         if datadir is None:
             self.CWD = os.path.dirname(os.path.abspath("__FILE__"))
@@ -79,7 +93,7 @@ class Otter(Database):
             try:
                 os.makedirs(self.DATADIR)
             except FileExistsError:
-                warn(
+                logger.warning(
                     "Directory was created between the if statement and trying "
                     + "to create the directory!"
                 )
@@ -93,7 +107,8 @@ class Otter(Database):
         Get the metadata of the objects matching the arguments
 
         Args:
-            **kwargs : Arguments to pass to Otter.query()
+            **kwargs : Arguments to pass to Otter.query(). See that documentation with
+                       `help(otter.Otter.query)`.
         Return:
            The metadata for the transients that match the arguments. Will be an astropy
            Table by default, if raw=True will be a dictionary.
@@ -146,9 +161,9 @@ class Otter(Database):
         unit conversion for you!
 
         Args:
-            flux_units (astropy.unit.Unit): Either a valid string to convert
+            flux_unit (astropy.unit.Unit): Either a valid string to convert
                                             or an astropy.unit.Unit
-            date_units (astropy.unit.Unit): Either a valid string to convert to a date
+            date_unit (astropy.unit.Unit): Either a valid string to convert to a date
                                             or an astropy.unit.Unit
             return_type (str): Either 'astropy' or 'pandas'. If astropy, returns an
                                astropy Table. If pandas, returns a pandas DataFrame.
@@ -159,6 +174,8 @@ class Otter(Database):
             keep_raw (bool): If True, keep the raw flux/date/freq/wave associated with
                              the dataset. Else, just keep the converted data. Default
                              is False.
+            wave_unit (str): The astropy wavelength unit to return with
+            freq_unit (str): The astropy frequency unit to return with`
             **kwargs : Arguments to pass to Otter.query(). Can be::
 
                        names (list[str]): A list of names to get the metadata for
@@ -181,6 +198,9 @@ class Otter(Database):
             FailedQueryError: When the query returns no results
             IOError: if one of your inputs is incorrect
         """
+        warn_filt = _DuplicateFilter()
+        logger.addFilter(warn_filt)
+
         queryres = self.query(hasphot=True, **kwargs)
 
         dicts = []
@@ -239,6 +259,7 @@ class Otter(Database):
             else:
                 fullphot = fullphot[keys_to_keep]
 
+        logger.removeFilter(warn_filt)
         if return_type == "astropy":
             return Table.from_pandas(fullphot)
         elif return_type == "pandas":
@@ -252,6 +273,9 @@ class Otter(Database):
 
         Args:
             filename (str): The path to the OTTER JSON file to load
+
+        Returns:
+            dictionary with the otter JSON file contents
         """
 
         # read in files from summary
@@ -270,6 +294,8 @@ class Otter(Database):
         refs: list[str] = None,
         hasphot: bool = False,
         hasspec: bool = False,
+        spec_classed: bool = False,
+        unambiguous: bool = False,
         classification: str = None,
         class_confidence_threshold: float = 0,
         query_private=False,
@@ -293,10 +319,17 @@ class Otter(Database):
             refs (list[str]): A list of ads bibcodes to match to. Will only return
                               metadata for transients that have this as a reference.
             hasphot (bool): if True, only returns transients which have photometry.
-            hasspec (bool): if True, only return transients that have spectra.
+            hasspec (bool): NOT IMPLEMENTED! Will return False for all targets!
+            spec_classed (bool): If True, only returns transients that have been
+                                 specotroscopically classified/confirmed
+            unambiguous (bool): If True, only returns transients that only have a single
+                                published classification in OTTER. If classifications
+                                disagree for a transient, it will be filtered out.
             classification (str): A classification string to search for
             class_confidence_threshold (float): classification confidence cutoff for
                                                 query, between 0 and 1. Default is 0.
+            query_private (bool): Set to True if you would like to also query the
+                                  dataset located at whatever you set datadir to
 
         Return:
            Get all of the raw (unconverted!) data for objects that match the criteria.
@@ -310,9 +343,15 @@ class Otter(Database):
         if hasspec is True:
             query_filters += "FILTER 'spectra' IN ATTRIBUTES(transient)\n"
 
+        if spec_classed:
+            query_filters += "FILTER transient.classification.spec_classed >= 1"
+
+        if unambiguous:
+            query_filters += "FILTER transient.classification.unambiguous"
+
         if classification is not None:
             query_filters += f"""
-            FOR subdoc IN transient.classification
+            FOR subdoc IN transient.classification.value
                 FILTER subdoc.confidence > TO_NUMBER({class_confidence_threshold})
                 FILTER subdoc.object_class LIKE '%{classification}%'
             """
@@ -573,7 +612,7 @@ class Otter(Database):
 
     def upload(self, json_data, collection="vetting", testing=False) -> Document:
         """
-        Upload json_data to collection
+        Upload json_data to collection WITHOUT deduplication!
 
         Args:
             json_data [dict] : A dictionary of the json data to upload to Otter
@@ -793,7 +832,7 @@ class Otter(Database):
     def from_csvs(
         metafile: str,
         photfile: str = None,
-        local_outpath: str = "private_otter_data",
+        local_outpath: Optional[str] = None,
         db: Otter = None,
     ) -> Otter:
         """
@@ -815,6 +854,7 @@ class Otter(Database):
         """
         # read in the metadata and photometry file
         meta = pd.read_csv(metafile)
+        meta.columns = meta.columns.str.strip()  # clean up the col names
         phot = None
 
         required_phot_cols = [
@@ -831,17 +871,20 @@ class Otter(Database):
 
         if photfile is not None:
             phot_unclean = pd.read_csv(photfile)
+            phot_unclean.columns = phot_unclean.columns.str.strip()  # cleanup colnames
 
             phot = phot_unclean.dropna(subset=required_phot_cols)
             if len(phot) != len(phot_unclean):
-                warn("""
+                logger.warning("""
                 Filtered out rows with nan in the photometry file! Make sure you
                 expect this behaviour!
                 """)
 
             if "bibcode" not in phot:
                 phot["bibcode"] = "private"
-                warn("Setting the bibcode column to the special keyword 'private'!")
+                logger.warning("""
+                Setting the bibcode column to the special keyword 'private'!
+                """)
 
             # we need to generate columns of wave_eff and freq_eff
             wave_eff = []
@@ -866,7 +909,7 @@ class Otter(Database):
             phot["band_eff_freq_unit"] = str(freq_eff_unit)
 
         if not os.path.exists(local_outpath):
-            os.mkdir(local_outpath)
+            os.makedirs(local_outpath)
 
         # drop duplicated names in meta and keep the first
         meta = meta.drop_duplicates(subset="name", keep="first")
@@ -907,7 +950,7 @@ class Otter(Database):
                     ra_units=tde.ra_unit[0],
                     dec_units=tde.dec_unit[0],
                     reference=[tde.coord_bibcode[0]],
-                    coordinate_type="equitorial",
+                    coordinate_type="equatorial",
                 )
             ]
 
@@ -956,13 +999,18 @@ class Otter(Database):
             ### Classification information that is in the csvs
             # classification
             if "classification" in tde:
-                json["classification"] = [
-                    dict(
-                        object_class=tde.classification[0],
-                        confidence=1,  # we know this is at least an tde
-                        reference=[tde.classification_bibcode[0]],
-                    )
-                ]
+                class_flag = 0
+                if "classification_flag" in tde:
+                    class_flag = tde.classification_flag[0]
+                json["classification"] = dict(
+                    value=[
+                        dict(
+                            object_class=tde.classification[0],
+                            confidence=class_flag,
+                            reference=[tde.classification_bibcode[0]],
+                        )
+                    ]
+                )
 
             # discovery date
             # print(tde)
@@ -1260,7 +1308,7 @@ class Otter(Database):
         if db is None:
             db = Otter(datadir=local_outpath)
         else:
-            db.datadir = local_outpath
+            db.DATADIR = local_outpath
 
         # always save this document as a new one
         db.save(all_jsons)