mpcq 0.2.3__py3-none-any.whl

mpcq/__init__.py

@@ -0,0 +1,5 @@
+from .client import MPCObservationsClient
+from .observation import Observation, ObservationStatus
+from .version import __version__
+
+__all__ = ["MPCObservationsClient", "Observation", "ObservationStatus", "__version__"]

mpcq/client.py

@@ -0,0 +1,221 @@
+import json
+import logging
+from typing import Iterator, Optional
+
+import astropy.time
+import google.cloud.secretmanager
+import sqlalchemy as sq
+from google.cloud.sql.connector import Connector
+
+from .observation import Observation, ObservationStatus
+from .submission import Submission
+
+log = logging.getLogger("mpcq.client")
+
+_DB_DRIVER = "pg8000"
+
+
+class MPCObservationsClient:
+    def __init__(self, dbconn: sq.engine.Connection):
+        self._dbconn = dbconn
+
+    @classmethod
+    def connect(cls, engine: sq.engine.Engine):
+        return cls(dbconn=engine.connect())
+
+    @classmethod
+    def connect_using_gcloud(
+        cls,
+        cloudsql_connection_name: str = "moeyens-thor-dev:us-west1:mpc-replica",
+        credentials_uri: str = "projects/moeyens-thor-dev/secrets/mpc-replica-readonly-credentials/versions/latest",  # noqa: E501
+    ):
+        log.info("loading database credentials")
+        client = google.cloud.secretmanager.SecretManagerServiceClient()
+        secret = client.access_secret_version(name=credentials_uri)
+        creds = json.loads(secret.payload.data)
+        log.info("database credentials loaded successfully")
+
+        connector = Connector()
+
+        def make_connection():
+            conn = connector.connect(
+                cloudsql_connection_name,
+                _DB_DRIVER,
+                user=creds["username"],
+                password=creds["password"],
+                db="mpc_sbn",
+            )
+            return conn
+
+        engine = sq.create_engine(
+            f"postgresql+{_DB_DRIVER}://", creator=make_connection
+        )
+        return cls.connect(engine)
+
+    def close(self):
+        self._dbconn.close()
+
+    def get_object_observations(
+        self,
+        object_provisional_designation: str,
+        obscode: Optional[str] = None,
+        filter_band: Optional[str] = None,
+    ) -> Iterator[Observation]:
+        stmt = self._observations_select_stmt(
+            object_provisional_designation, obscode, filter_band
+        )
+        result = self._dbconn.execute(stmt)
+        for r in result:
+            yield self._parse_obs_sbn_row(r)
+
+    def get_object_submissions(
+        self,
+        object_provisional_designation: str,
+    ) -> Iterator[Submission]:
+
+        stmt = self._submissions_select_stmt(object_provisional_designation)
+        result = self._dbconn.execute(stmt)
+        for r in result:
+            yield self._parse_submissions_sbn_row(r)
+
+    @staticmethod
+    def _parse_obs_sbn_row(row: sq.engine.Row) -> Observation:
+
+        if row.created_at is None:
+            created_at = None
+        else:
+            created_at = astropy.time.Time(row.created_at, scale="utc")
+
+        if row.updated_at is None:
+            updated_at = None
+        else:
+            updated_at = astropy.time.Time(row.updated_at, scale="utc")
+
+        return Observation(
+            mpc_id=row.id,
+            status=ObservationStatus._from_db_value(row.status),
+            obscode=row.stn,
+            filter_band=row.band,
+            unpacked_provisional_designation=row.provid,
+            timestamp=astropy.time.Time(row.obstime, scale="utc"),
+            ra=row.ra,
+            dec=row.dec,
+            mag=row.mag,
+            ra_rms=row.rmsra,
+            dec_rms=row.rmsdec,
+            mag_rms=row.rmsmag,
+            submission_id=row.submission_id,
+            created_at=created_at,
+            updated_at=updated_at,
+        )
+
+    @staticmethod
+    def _parse_submissions_sbn_row(row: sq.engine.Row) -> Submission:
+        return Submission(
+            id=row.submission_id,
+            num_observations=row.num_observations,
+        )
+
+    def _observations_select_stmt(
+        self,
+        provisional_designation: str,
+        obscode: Optional[str],
+        filter_band: Optional[str],
+    ) -> sq.sql.expression.Select:
+        """Construct a database select statement to fetch observations for
+        given object (named by provisional designation, eg "2022 AJ2").
+
+        obscode and filter_band can optionally be provided to limit the
+        result set.
+
+        """
+        log.info("loading observations for %s", provisional_designation)
+        stmt = (
+            sq.select(
+                sq.column("id"),
+                sq.column("stn"),
+                sq.column("status"),
+                sq.column("ra"),
+                sq.column("dec"),
+                sq.column("obstime"),
+                sq.column("provid"),
+                sq.column("rmsra"),
+                sq.column("rmsdec"),
+                sq.column("mag"),
+                sq.column("rmsmag"),
+                sq.column("band"),
+                sq.column("submission_id"),
+                sq.column("created_at"),
+                sq.column("updated_at"),
+            )
+            .select_from(sq.table("obs_sbn"))
+            .where(
+                sq.column("provid") == provisional_designation,
+            )
+        )
+        if obscode is not None:
+            stmt = stmt.where(sq.column("stn") == obscode)
+        if filter_band is not None:
+            stmt = stmt.where(sq.column("band") == filter_band)
+
+        return stmt
+
+    def _submissions_select_stmt(
+        self,
+        provisional_designation: str,
+    ) -> sq.sql.expression.Select:
+        """
+        Construct a database select statement to fetch submission IDs for
+        the given object (named by provisional designation, eg "2022 AJ2").
+
+        Parameters
+        ----------
+        provisional_designation : str
+            The provisional designation of the object to fetch submissions for.
+
+        Returns
+        -------
+        sq.sql.expression.Select
+            The select statement to fetch the submission IDs for the given object.
+        """
+        log.info("loading submissions for %s", provisional_designation)
+        stmt = (
+            sq.select(
+                sq.column("submission_id"),
+                sq.func.count(sq.column("submission_id")).label("num_observations"),
+            )
+            .select_from(sq.table("obs_sbn"))
+            .where(
+                sq.column("provid") == provisional_designation,
+            )
+            .group_by(sq.column("submission_id"))
+        )
+
+        return stmt
+
+    def object_counts_for_submission_id(
+        self, submission_id: str
+    ) -> list[tuple[str, int]]:
+        """
+        Queries for the number of observations for each object associated with a given submission ID.
+        """
+        log.info("loading object counts for submission ID %s", submission_id)
+        stmt = (
+            sq.select(
+                sq.column("provid"),
+                sq.func.count(sq.column("provid")).label("num_observations"),
+            )
+            .select_from(sq.table("obs_sbn"))
+            .where(
+                # HACK: the database currently only has an index for
+                # submission_block_id, even though we'd like to use
+                # submission_id.
+                sq.column("submission_block_id") == submission_id + "_01",
+                # We'd prefer this:
+                # sq.column("submission_id") == submission_id,
+                # but that index is still being built.
+            )
+            .group_by(sq.column("provid"))
+        )
+        result = self._dbconn.execute(stmt)
+        return [(row.provid, row.num_observations) for row in result]

mpcq/observation.py

@@ -0,0 +1,78 @@
+import dataclasses
+import decimal
+from enum import Enum
+from typing import Optional
+
+import astropy.time
+
+
+class ObservationStatus(Enum):
+    Published = 1
+    Pending = 2
+    IsolatedTrackletFile = 3
+
+    @classmethod
+    def _from_db_value(cls, val: str):
+        return {
+            "P": ObservationStatus.Published,
+            "p": ObservationStatus.Pending,
+            "I": ObservationStatus.IsolatedTrackletFile,
+        }[val]
+
+
+@dataclasses.dataclass
+class Observation:
+    """
+    Represents a single entry in the MPC Observations 'obs_sbn' table.
+    """
+
+    # The internal primary key, an opaque integer.
+    mpc_id: int
+
+    # See ObservationStatus.
+    status: ObservationStatus
+
+    # The three-character code which identifies the observatory which
+    # made the observation.
+    obscode: str
+
+    # The single-character code which identifies which filter was used
+    # when observing.
+    filter_band: str
+
+    # An identifier for the object being observed, for example '2022 AJ2'.
+    unpacked_provisional_designation: str
+
+    # The time of the observation. This is submitted to MPC, and does
+    # not have a well-defined meaning. In general, it ought to be the
+    # midpoint of the exposure that the observation came from, but
+    # sometimes it seems to be the start of the exposure. For
+    # observations which come from stacks of multiple exposures it is
+    # even less well-defined.
+    timestamp: astropy.time.Time
+
+    # Right ascension of the observation in the J2000 frame.
+    ra: decimal.Decimal
+    # Error in the right ascension.
+    ra_rms: Optional[decimal.Decimal]
+
+    # Declination of the observation in the J2000 frame.
+    dec: decimal.Decimal
+    # Error in the declination.
+    dec_rms: Optional[decimal.Decimal]
+
+    # Observed magnitude of the object.
+    mag: Optional[decimal.Decimal]
+    # Error in the magnitude estimate.
+    mag_rms: Optional[decimal.Decimal]
+
+    # Submission Properties
+    # MPC-assigned ID for the submission
+    submission_id: str
+
+    # Time when the observation was first submitted to MPC and ingested
+    # Apparently this can be None... maybe for older observations???
+    created_at: Optional[astropy.time.Time]
+    # Time when the last update to this observations was made, None if
+    # never updated beyond initial creation
+    updated_at: Optional[astropy.time.Time]

mpcq/submission.py

@@ -0,0 +1,23 @@
+from dataclasses import dataclass, field
+
+from astropy.time import Time
+
+
+def timestamp_from_submission_id(submission_id: str):
+    timestamp_isot = submission_id.split("_")[0]
+    return Time(timestamp_isot, format="isot")
+
+
+@dataclass
+class Submission:
+    # MPC-assigned submission ID (same as submission_id in Observation)
+    id: str
+
+    # Added-value (number of observations in submission)
+    num_observations: int
+
+    # Added-value (extracted from submission ID)
+    timestamp: Time = field(init=False)
+
+    def __post_init__(self):
+        self.timestamp = timestamp_from_submission_id(self.id)

mpcq/tests/test_integration.py

@@ -0,0 +1,58 @@
+import os
+
+import pytest
+
+from ..client import MPCObservationsClient
+
+
+@pytest.mark.integration
+@pytest.mark.skipif(
+    "MPCQ_INTEGRATION_TESTS" not in os.environ,
+    reason="MPCQ_INTEGRATION_TESTS must be set",
+)
+class TestIntegration:
+    def test_connection(self):
+        # Test that a connection can be established
+        MPCObservationsClient.connect_using_gcloud()
+
+    def test_get_object_observations(self, mpc_client):
+        observations = list(mpc_client.get_object_observations("2022 AJ2"))
+        assert len(observations) >= 10
+        for o in observations:
+            assert o.unpacked_provisional_designation == "2022 AJ2"
+
+    def test_get_object_observations_filter_by_obscode(self, mpc_client):
+        observations = mpc_client.get_object_observations("2022 AJ2", obscode="I52")
+        observations = list(observations)
+        assert len(observations) > 0
+        for o in observations:
+            assert o.unpacked_provisional_designation == "2022 AJ2"
+            assert o.obscode == "I52"
+
+    def test_get_object_observations_filter_by_filter_band(self, mpc_client):
+        observations = mpc_client.get_object_observations("2022 AJ2", filter_band="G")
+        observations = list(observations)
+        assert len(observations) > 0
+        for o in observations:
+            assert o.unpacked_provisional_designation == "2022 AJ2"
+            assert o.filter_band == "G"
+
+    def test_get_object_submissions(self, mpc_client):
+        submissions = list(mpc_client.get_object_submissions("2022 AJ2"))
+        assert len(submissions) >= 1
+        for s in submissions:
+            assert s.num_observations >= 1
+
+        observations = mpc_client.get_object_observations("2022 AJ2")
+        observations = list(observations)
+
+        num_observations_from_submissions = sum(s.num_observations for s in submissions)
+        num_observations = len(observations)
+        assert num_observations_from_submissions == num_observations
+
+
+@pytest.fixture
+def mpc_client():
+    client = MPCObservationsClient.connect_using_gcloud()
+    yield client
+    client.close()

mpcq/tests/test_submission.py

@@ -0,0 +1,11 @@
+from astropy.time import Time
+
+from mpcq.submission import timestamp_from_submission_id
+
+
+def test_timestamp_from_submission_id():
+
+    submission_id = "2020-10-23T06:39:01.200_0000DvZH"
+    desired_timestamp = Time("2020-10-23T06:39:01.200", format="isot")
+
+    assert timestamp_from_submission_id(submission_id) == desired_timestamp

mpcq/tests/test_utils.py

@@ -0,0 +1,69 @@
+import decimal
+
+import pandas as pd
+from astropy.time import Time
+
+from mpcq.observation import Observation, ObservationStatus
+from mpcq.submission import Submission
+from mpcq.utils import observations_to_dataframe, submissions_to_dataframe
+
+
+def test_observations_to_dataframe():
+
+    observation = Observation(
+        mpc_id=1000,
+        status=ObservationStatus.Published,
+        obscode="I41",
+        filter_band="R",
+        unpacked_provisional_designation="2022 AJ2",
+        timestamp=Time("2021-01-01T00:00:00.000", format="isot"),
+        ra=decimal.Decimal(1.0),
+        ra_rms=decimal.Decimal(0.001),
+        dec=decimal.Decimal(-10.0),
+        dec_rms=decimal.Decimal(0.004),
+        mag=decimal.Decimal(20.0),
+        mag_rms=decimal.Decimal(0.1),
+        submission_id="2020-10-23T06:39:01.200_0000DvZH",
+        created_at=Time("2020-10-23T06:39:01.200", format="isot"),
+        updated_at=Time("2020-10-23T06:39:01.200", format="isot"),
+    )
+
+    desired_df = pd.DataFrame(
+        {
+            "mpc_id": [1000],
+            "status": ["Published"],
+            "obscode": ["I41"],
+            "filter_band": ["R"],
+            "unpacked_provisional_designation": ["2022 AJ2"],
+            "timestamp": [Time("2021-01-01T00:00:00.000", format="isot").datetime],
+            "ra": [decimal.Decimal(1.0)],
+            "ra_rms": [decimal.Decimal(0.001)],
+            "dec": [decimal.Decimal(-10.0)],
+            "dec_rms": [decimal.Decimal(0.004)],
+            "mag": [decimal.Decimal(20.0)],
+            "mag_rms": [decimal.Decimal(0.1)],
+            "submission_id": ["2020-10-23T06:39:01.200_0000DvZH"],
+            "created_at": [Time("2020-10-23T06:39:01.200", format="isot").datetime],
+            "updated_at": [Time("2020-10-23T06:39:01.200", format="isot").datetime],
+        }
+    )
+
+    pd.testing.assert_frame_equal(observations_to_dataframe([observation]), desired_df)
+
+
+def test_submissions_to_dataframe():
+
+    submission = Submission(
+        id="2020-10-23T06:39:01.200_0000DvZH",
+        num_observations=5,
+    )
+
+    desired_df = pd.DataFrame(
+        {
+            "id": ["2020-10-23T06:39:01.200_0000DvZH"],
+            "num_observations": [5],
+            "timestamp": [Time("2020-10-23T06:39:01.200", format="isot").datetime],
+        }
+    )
+
+    pd.testing.assert_frame_equal(submissions_to_dataframe([submission]), desired_df)

mpcq/utils.py

@@ -0,0 +1,57 @@
+from dataclasses import asdict
+from typing import List
+
+import pandas as pd
+
+from .observation import Observation
+from .submission import Submission
+
+
+def observations_to_dataframe(observations: List[Observation]) -> pd.DataFrame:
+    """
+    Convert a list of Observation objects to a pandas DataFrame.
+
+    Parameters
+    ----------
+    observations : List[Observation]
+        The observations to convert.
+
+    Returns
+    -------
+    observations : `~pd.DataFrame`
+    """
+    data = [asdict(obs) for obs in observations]
+
+    # Convert ObservationStatus objects to strings.
+    # Convert Time objects to datetime objects.
+    for row in data:
+        row["status"] = row["status"].name
+        row["timestamp"] = row["timestamp"].datetime
+        if row["created_at"] is not None:
+            row["created_at"] = row["created_at"].datetime
+        if row["updated_at"] is not None:
+            row["updated_at"] = row["updated_at"].datetime
+
+    return pd.DataFrame(data)
+
+
+def submissions_to_dataframe(submissions: List[Submission]) -> pd.DataFrame:
+    """
+    Convert a list of Submission objects to a pandas DataFrame.
+
+    Parameters
+    ----------
+    submissions : List[Submission]
+        The submissions to convert.
+
+    Returns
+    -------
+    submissions : `~pd.DataFrame`
+    """
+    data = [asdict(sub) for sub in submissions]
+
+    # Convert Time objects to datetime objects.
+    for row in data:
+        row["timestamp"] = row["timestamp"].datetime
+
+    return pd.DataFrame(data)

mpcq/version.py

@@ -0,0 +1,7 @@
+from importlib.metadata import PackageNotFoundError, version
+
+try:
+    __version__ = version("mpcq")
+except PackageNotFoundError:
+    # package is not installed
+    __version__ = "unknown"

mpcq-0.2.3.dist-info/METADATA

@@ -0,0 +1,76 @@
+Metadata-Version: 2.1
+Name: mpcq
+Version: 0.2.3
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+Requires-Dist: sqlalchemy (<2)
+Requires-Dist: pandas
+Requires-Dist: astropy
+Requires-Dist: google-cloud-secret-manager
+Requires-Dist: cloud-sql-python-connector[pg8000]
+Requires-Dist: pg8000
+Provides-Extra: tests
+Requires-Dist: pre-commit ; extra == 'tests'
+Requires-Dist: pytest ; extra == 'tests'
+
+# mpcq
+#### A Python package by the Asteroid Institute, a program of the B612 Foundation  
+
+This is a client library for interacting with an MPC observations database.
+
+## Usage
+
+To connect to the Asteroid Institute's clone of the Small Bodies Node MPC database:
+```python
+from mpcq.client import MPCObservationsClient
+
+client = MPCObservationsClient.connect_using_gcloud()
+```
+
+With a client initialized, you can get all observations of a particular object using its provisional designation:
+```python
+observations = client.get_object_observations("2013 RR165")
+observations = list(observations)
+```
+
+These can be converted into a dataframe as follows:
+
+```python
+from mpcq.utils import observations_to_dataframe
+
+observations_df = observations_to_dataframe(observations)
+print(observations_df.head(5))
+```
+```
+	mpc_id	status	obscode	filter_band	unpacked_provisional_designation	timestamp	ra	ra_rms	dec	dec_rms	mag	mag_rms	submission_id	created_at	updated_at
+0	174511900	Published	F51	w	2013 RR165	2011-01-30 11:15:26	123.884679	None	19.820047	None	22.2	None	2011-04-12T00:57:19.000_00005L9j	2017-07-10 00:00:00.000000	2022-06-15 17:17:33.421485
+1	174511901	Published	F51	w	2013 RR165	2011-01-30 11:37:23	123.880767	None	19.820603	None	22.4	None	2011-04-12T00:57:19.000_00005L9j	2017-07-10 00:00:00.000000	2022-06-15 17:17:33.427512
+2	174511902	Published	F51	w	2013 RR165	2011-01-30 12:22:35	123.872683	None	19.821694	None	22.3	None	2011-04-12T00:57:19.000_00005L9j	2017-07-10 00:00:00.000000	2022-06-15 17:17:33.431369
+3	394474985	Published	W84	g	2013 RR165	2013-09-02 05:49:09	354.49745627	0.097	1.17373181	0.100	21.66	0.07	2022-05-23T23:16:35.633_0000EfpX	2022-05-23 23:18:28.963374	2022-06-15 17:17:33.434170
+4	175542203	Published	F51	w	2013 RR165	2013-09-03 10:20:19	354.259229	None	1.099064	None	21.6	None	2013-09-04T00:37:51.000_00005cQy	2017-07-10 00:00:00.000000	2022-06-15 17:17:33.436820
+```
+
+Getting the submission ID and the number of observations per submission of an object:
+```python
+submissions = client.get_object_submissions("2013 RR165")
+submissions = list(submissions)
+```
+
+As before, these can be converted to a dataframe:
+```python
+from mpcq.utils import submissions_to_dataframe
+
+submissions_df = submissions_to_dataframe(submissions)
+print(submissions_df)
+```
+```
+	id	num_observations	timestamp
+0	2017-07-05T20:57:57.001_00006dS8	3	2017-07-05 20:57:57.001
+1	2013-09-04T00:37:51.000_00005cQy	3	2013-09-04 00:37:51.000
+2	2017-09-13T19:53:09.000_0000CdSX	2	2017-09-13 19:53:09.000
+3	2022-05-23T23:16:35.633_0000EfpX	19	2022-05-23 23:16:35.633
+4	2013-09-12T11:43:25.001_00005cpl	4	2013-09-12 11:43:25.001
+5	2011-04-12T00:57:19.000_00005L9j	3	2011-04-12 00:57:19.000
+6	2016-04-03T19:54:29.000_00006IpM	3	2016-04-03 19:54:29.000
+7	2015-01-16T23:14:02.000_00005wg5	3	2015-01-16 23:14:02.000
+```

mpcq-0.2.3.dist-info/RECORD

@@ -0,0 +1,14 @@
+mpcq/__init__.py,sha256=w7f2XUymlsZSzrKSmez08cBBM3HF2xznNnDfKevVRe0,219
+mpcq/client.py,sha256=wUEWu2oJm5hVI7LwA5lVKWMVk0jpTDnIngaUL380o-U,7335
+mpcq/observation.py,sha256=n1eQ8hFWh1HsFhejlX2Wqb-cqMVkQWAqWs3FrmHSm-A,2354
+mpcq/submission.py,sha256=wkxU-PavFnHOhD14UwPWXFsx8ScK7U6N4EbECG3cw8M,606
+mpcq/utils.py,sha256=IHFUUaWBGo1xXGJbrfxdFuWh-luNLaIpzKvvwHLyQuM,1499
+mpcq/version.py,sha256=gfIUsZ8--RIaUj-9NvEiDCoybvDLeoOILCvAnJ9F8Yg,189
+mpcq/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+mpcq/tests/test_integration.py,sha256=avnBZXk1fEZP1kt2KbOt0sZ00GQDyUkC29s1M7OyYo4,2093
+mpcq/tests/test_submission.py,sha256=RbwnihJ8UAihd4FZbErW7Rpq0jzvI1gru3mVSvsb0Eo,335
+mpcq/tests/test_utils.py,sha256=gJUtA9NOJn130OhBEe18feyljEU8fajiolYR7q9jUuk,2374
+mpcq-0.2.3.dist-info/METADATA,sha256=8vaSYGIp2Of4zJvgEunhiwImiAkmEuxnhX7W-4QR53Y,3225
+mpcq-0.2.3.dist-info/WHEEL,sha256=pkctZYzUS4AYVn6dJ-7367OJZivF2e8RA9b_ZBjif18,92
+mpcq-0.2.3.dist-info/top_level.txt,sha256=vldAS3-4ybSuV_Y3YQJbdEQ16MmwUrhvCt_5uU-AUPg,5
+mpcq-0.2.3.dist-info/RECORD,,

mpcq-0.2.3.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: bdist_wheel (0.40.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

mpcq-0.2.3.dist-info/top_level.txt

@@ -0,0 +1 @@
+mpcq