datanommer.models 1.0.4__py3-none-any.whl → 1.3.0__py3-none-any.whl

datanommer/models/__init__.py

@@ -14,6 +14,7 @@
 # You should have received a copy of the GNU General Public License along
 # with this program.  If not, see <http://www.gnu.org/licenses/>.
 import datetime
+import importlib.metadata
 import json
 import logging
 import math
@@ -21,7 +22,6 @@ import traceback
 import uuid
 from warnings import warn
 
-import pkg_resources
 from sqlalchemy import (
     and_,
     between,
@@ -31,11 +31,14 @@ from sqlalchemy import (
     DDL,
     event,
     ForeignKey,
+    func,
     Integer,
     not_,
     or_,
+    select,
     String,
     Table,
+    text,
     TypeDecorator,
     Unicode,
     UnicodeText,
@@ -61,6 +64,9 @@ except ImportError:  # pragma: no cover
     UniqueViolation = lookup_error("23505")
 
 
+__version__ = importlib.metadata.version("datanommer.models")
+
+
 log = logging.getLogger("datanommer")
 
 maker = sessionmaker()
@@ -80,7 +86,7 @@ def init(uri=None, alembic_ini=None, engine=None, create=False):
         raise ValueError("One of uri or engine must be specified")
 
     if uri and not engine:
-        engine = create_engine(uri)
+        engine = create_engine(uri, future=True)
 
     # We need to hang our own attribute on the sqlalchemy session to stop
     # ourselves from initializing twice.  That is only a problem if the code
@@ -90,11 +96,12 @@ def init(uri=None, alembic_ini=None, engine=None, create=False):
         return
     session._datanommer_initialized = True
 
-    session.configure(bind=engine)
+    maker.configure(bind=engine)
     DeclarativeBase.query = session.query_property()
 
     if create:
-        session.execute("CREATE EXTENSION IF NOT EXISTS timescaledb")
+        with engine.begin() as connection:
+            connection.execute(text("CREATE EXTENSION IF NOT EXISTS timescaledb"))
         DeclarativeBase.metadata.create_all(engine)
         # Loads the alembic configuration and generates the version table, with
         # the most recent revision stamped as head
@@ -122,7 +129,7 @@ def add(message):
             log.exception("Failed to parse sent-at timestamp value")
             return
     else:
-        sent_at = datetime.datetime.utcnow()
+        sent_at = datetime.datetime.now(tz=datetime.timezone.utc)
 
     # Workaround schemas misbehaving
     try:
@@ -158,15 +165,10 @@ def add(message):
     session.commit()
 
 
-def source_version_default(context):
-    dist = pkg_resources.get_distribution("datanommer.models")
-    return dist.version
-
-
 # https://docs.sqlalchemy.org/en/14/core/custom_types.html#marshal-json-strings
 
 
-class JSONEncodedDict(TypeDecorator):
+class _JSONEncodedDict(TypeDecorator):
     """Represents an immutable structure as a json-encoded string."""
 
     impl = UnicodeText
@@ -224,8 +226,8 @@ class Message(DeclarativeBase):
     username = Column(Unicode)
     crypto = Column(UnicodeText)
     source_name = Column(Unicode, default="datanommer")
-    source_version = Column(Unicode, default=source_version_default)
-    msg = Column(JSONEncodedDict, nullable=False)
+    source_version = Column(Unicode, default=lambda context: __version__)
+    msg = Column(_JSONEncodedDict, nullable=False)
     headers = Column(postgresql.JSONB(none_as_null=True))
     users = relationship(
         "User",
@@ -314,7 +316,7 @@ class Message(DeclarativeBase):
 
     @classmethod
     def from_msg_id(cls, msg_id):
-        return cls.query.filter(cls.msg_id == msg_id).first()
+        return session.execute(select(cls).where(cls.msg_id == msg_id)).scalar_one_or_none()
 
     def as_dict(self, request=None):
         return dict(
@@ -337,13 +339,12 @@ class Message(DeclarativeBase):
     def as_fedora_message_dict(self):
         headers = self.headers or {}
         if "sent-at" not in headers:
-            headers["sent-at"] = self.timestamp.astimezone(
-                datetime.timezone.utc
-            ).isoformat()
+            headers["sent-at"] = self.timestamp.astimezone(datetime.timezone.utc).isoformat()
         return dict(
             body=self.msg,
             headers=headers,
             id=self.msg_id,
+            priority=headers.get("priority", 0),
             queue=None,
             topic=self.topic,
         )
@@ -353,17 +354,15 @@ class Message(DeclarativeBase):
             "The __json__() method has been renamed to as_dict(), and will be removed "
             "in the next major version",
             DeprecationWarning,
+            stacklevel=2,
         )
         return self.as_dict(request)
 
     @classmethod
-    def grep(
+    def make_query(
         cls,
         start=None,
         end=None,
-        page=1,
-        rows_per_page=100,
-        order="asc",
         msg_id=None,
         users=None,
         not_users=None,
@@ -374,7 +373,6 @@ class Message(DeclarativeBase):
         topics=None,
         not_topics=None,
         contains=None,
-        defer=False,
     ):
         """Flexible query interface for messages.
 
@@ -402,11 +400,6 @@ class Message(DeclarativeBase):
 
             (user == 'ralph') AND
             NOT (category == 'bodhi' OR category == 'wiki')
-
-        ----
-
-        If the `defer` argument evaluates to True, the query won't actually
-        be executed, but a SQLAlchemy query object returned instead.
         """
 
         users = users or []
@@ -419,87 +412,137 @@ class Message(DeclarativeBase):
         not_topics = not_topics or []
         contains = contains or []
 
-        query = Message.query
+        Message = cls
+        query = select(Message)
 
         # A little argument validation.  We could provide some defaults in
         # these mixed cases.. but instead we'll just leave it up to our caller.
         if (start is not None and end is None) or (end is not None and start is None):
             raise ValueError(
-                "Either both start and end must be specified "
-                "or neither must be specified"
+                "Either both start and end must be specified or neither must be specified"
             )
 
         if start and end:
-            query = query.filter(between(Message.timestamp, start, end))
+            query = query.where(between(Message.timestamp, start, end))
 
         if msg_id:
-            query = query.filter(Message.msg_id == msg_id)
+            query = query.where(Message.msg_id == msg_id)
 
         # Add the four positive filters as necessary
         if users:
-            query = query.filter(
-                or_(*(Message.users.any(User.name == u) for u in users))
-            )
+            query = query.where(or_(*(Message.users.any(User.name == u) for u in users)))
 
         if packages:
-            query = query.filter(
-                or_(*(Message.packages.any(Package.name == p) for p in packages))
-            )
+            query = query.where(or_(*(Message.packages.any(Package.name == p) for p in packages)))
 
         if categories:
-            query = query.filter(
-                or_(*(Message.category == category for category in categories))
-            )
+            query = query.where(or_(*(Message.category == category for category in categories)))
 
         if topics:
-            query = query.filter(or_(*(Message.topic == topic for topic in topics)))
+            query = query.where(or_(*(Message.topic == topic for topic in topics)))
 
         if contains:
-            query = query.filter(
-                or_(*(Message.msg.like(f"%{contain}%") for contain in contains))
-            )
+            query = query.where(or_(*(Message.msg.like(f"%{contain}%") for contain in contains)))
 
         # And then the four negative filters as necessary
         if not_users:
-            query = query.filter(
-                not_(or_(*(Message.users.any(User.name == u) for u in not_users)))
-            )
+            query = query.where(not_(or_(*(Message.users.any(User.name == u) for u in not_users))))
 
         if not_packs:
-            query = query.filter(
+            query = query.where(
                 not_(or_(*(Message.packages.any(Package.name == p) for p in not_packs)))
             )
 
         if not_cats:
-            query = query.filter(
-                not_(or_(*(Message.category == category for category in not_cats)))
-            )
+            query = query.where(not_(or_(*(Message.category == category for category in not_cats))))
 
         if not_topics:
-            query = query.filter(
-                not_(or_(*(Message.topic == topic for topic in not_topics)))
-            )
+            query = query.where(not_(or_(*(Message.topic == topic for topic in not_topics))))
+
+        return query
+
+    @classmethod
+    def grep(
+        cls,
+        *,
+        page=1,
+        rows_per_page=100,
+        order="asc",
+        defer=False,
+        **kwargs,
+    ):
+        """Flexible query interface for messages.
+
+        Arguments are filters.  start and end should be :mod:`datetime` objs.
+
+        Other filters should be lists of strings.  They are applied in a
+        conjunctive-normal-form (CNF) kind of way
 
+        for example, the following::
+
+          users = ['ralph', 'lmacken']
+          categories = ['bodhi', 'wiki']
+
+        should return messages where
+
+          (user=='ralph' OR user=='lmacken') AND
+          (category=='bodhi' OR category=='wiki')
+
+        Furthermore, you can use a negative version of each argument.
+
+            users = ['ralph']
+            not_categories = ['bodhi', 'wiki']
+
+        should return messages where
+
+            (user == 'ralph') AND
+            NOT (category == 'bodhi' OR category == 'wiki')
+
+        ----
+
+        The ``jsons`` argument is a list of jsonpath filters, please refer to
+        `PostgreSQL's documentation
+        <https://www.postgresql.org/docs/current/functions-json.html#FUNCTIONS-SQLJSON-PATH>`_
+        on the matter to learn how to build the jsonpath expression.
+
+        The ``jsons_and`` argument is similar to the ``jsons`` argument, but all
+        the values must match for a message to be returned.
+        """
+        query = cls.make_query(**kwargs)
         # Finally, tag on our pagination arguments
-        total = query.count()
+        Message = cls
+
+        query_total = query.with_only_columns(func.count(Message.id))
+        total = None
         query = query.order_by(getattr(Message.timestamp, order)())
 
         if not rows_per_page:
             pages = 1
         else:
+            total = session.scalar(query_total)
             pages = int(math.ceil(total / float(rows_per_page)))
             query = query.offset(rows_per_page * (page - 1)).limit(rows_per_page)
 
         if defer:
-            return total, page, query
+            if total is None:
+                total = session.scalar(query_total)
+            return total, pages, query
         else:
             # Execute!
-            messages = query.all()
+            messages = session.scalars(query).all()
+            if pages == 1:
+                total = len(messages)
             return total, pages, messages
 
+    @classmethod
+    def get_first(cls, *, order="asc", **kwargs):
+        """Get the first message matching the regular grep filters."""
+        query = cls.make_query(**kwargs)
+        query = query.order_by(getattr(Message.timestamp, order)())
+        return session.scalars(query).first()
 
-class NamedSingleton:
 
+class NamedSingleton:
     id = Column(Integer, primary_key=True, autoincrement=True)
     name = Column(UnicodeText, index=True, unique=True)
 
@@ -513,8 +556,8 @@ class NamedSingleton:
         if name in cls._cache:
             # If we cache the instance, SQLAlchemy will run this query anyway because the instance
             # will be from a different transaction. So just cache the id.
-            return cls.query.get(cls._cache[name])
-        obj = cls.query.filter_by(name=name).one_or_none()
+            return session.get(cls, cls._cache[name])
+        obj = session.execute(select(cls).where(cls.name == name)).scalar_one_or_none()
         if obj is None:
             obj = cls(name=name)
             session.add(obj)
@@ -546,16 +589,3 @@ def _setup_hypertable(table_class):
 
 
 _setup_hypertable(Message)
-
-
-# Set the version
-try:  # pragma: no cover
-    import importlib.metadata
-
-    __version__ = importlib.metadata.version("datanommer.models")
-except ImportError:  # pragma: no cover
-    try:
-        __version__ = pkg_resources.get_distribution("datanommer.models").version
-    except pkg_resources.DistributionNotFound:
-        # The app is not installed, but the flask dev server can run it nonetheless.
-        __version__ = None

datanommer/models/testing/__init__.py

@@ -1,46 +1,56 @@
-import os
-
 import pytest
+import sqlalchemy as sa
 from pytest_postgresql import factories
+from pytest_postgresql.janitor import DatabaseJanitor
 from sqlalchemy.orm import scoped_session
 
 import datanommer.models as dm
 
 
-@pytest.fixture(scope="session")
-def postgresql_proc_with_timescaledb(postgresql_proc):
-    with open(os.path.join(postgresql_proc.datadir, "postgresql.conf"), "a") as pgconf:
-        pgconf.write("\nshared_preload_libraries = 'timescaledb'\n")
-        pgconf.write("timescaledb.telemetry_level=off\n")
-    postgresql_proc.stop()
-    postgresql_proc.start()
-    yield postgresql_proc
-
-
-_inital_sql = os.path.abspath(os.path.join(os.path.dirname(__file__), "startup.sql"))
-pgsql = factories.postgresql(
-    "postgresql_proc_with_timescaledb",
-    load=[_inital_sql],
+postgresql_proc = factories.postgresql_proc(
+    postgres_options="-c shared_preload_libraries=timescaledb -c timescaledb.telemetry_level=off",
 )
 
 
-@pytest.fixture()
-def datanommer_db_url(pgsql):
+@pytest.fixture(scope="session")
+def datanommer_db_url(postgresql_proc):
     return (
-        f"postgresql+psycopg2://{pgsql.info.user}:@"
-        f"{pgsql.info.host}:{pgsql.info.port}"
-        f"/{pgsql.info.dbname}"
+        f"postgresql+psycopg2://{postgresql_proc.user}:@"
+        f"{postgresql_proc.host}:{postgresql_proc.port}"
+        f"/{postgresql_proc.dbname}"
     )
 
 
+@pytest.fixture(scope="session")
+def datanommer_db_engine(postgresql_proc, datanommer_db_url):
+    with DatabaseJanitor(
+        user=postgresql_proc.user,
+        host=postgresql_proc.host,
+        port=postgresql_proc.port,
+        dbname=postgresql_proc.dbname,
+        # Don't use a template database
+        # template_dbname=postgresql_proc.template_dbname,
+        version=postgresql_proc.version,
+    ):
+        engine = sa.create_engine(datanommer_db_url, future=True)
+        # Renew the global object, dm.init checks a custom attribute
+        dm.session = scoped_session(dm.maker)
+        dm.init(engine=engine, create=True)
+        yield engine
+        engine.dispose()
+
+
+@pytest.fixture()
+def datanommer_db(datanommer_db_url, datanommer_db_engine):
+    for table in reversed(dm.DeclarativeBase.metadata.sorted_tables):
+        dm.session.execute(table.delete())
+    dm.session.commit()
+    yield datanommer_db_engine
+
+
 @pytest.fixture()
-def datanommer_models(datanommer_db_url):
-    dm.session = scoped_session(dm.maker)
-    dm.init(datanommer_db_url, create=True)
+def datanommer_models(datanommer_db):
     dm.User.clear_cache()
     dm.Package.clear_cache()
     yield dm.session
     dm.session.rollback()
-    # engine = dm.session.get_bind()
-    dm.session.close()
-    # dm.DeclarativeBase.metadata.drop_all(engine)

datanommer_models-1.3.0.dist-info/METADATA

@@ -0,0 +1,55 @@
+Metadata-Version: 2.1
+Name: datanommer.models
+Version: 1.3.0
+Summary: SQLAlchemy models for datanommer
+Home-page: https://github.com/fedora-infra/datanommer
+License: GPL-3.0-or-later
+Author: Fedora Infrastructure
+Author-email: admin@fedoraproject.org
+Requires-Python: >=3.10,<4.0
+Classifier: License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Provides-Extra: schemas
+Requires-Dist: SQLAlchemy (>=1.3.24,<3.0.0)
+Requires-Dist: alembic (>=1.6.5,<2.0.0)
+Requires-Dist: anitya-schema ; extra == "schemas"
+Requires-Dist: bodhi-messages ; extra == "schemas"
+Requires-Dist: bugzilla2fedmsg-schema ; extra == "schemas"
+Requires-Dist: ci-messages ; extra == "schemas"
+Requires-Dist: copr-messaging ; extra == "schemas"
+Requires-Dist: discourse2fedmsg-messages ; extra == "schemas"
+Requires-Dist: fedocal-messages ; extra == "schemas"
+Requires-Dist: fedora-elections-messages ; extra == "schemas"
+Requires-Dist: fedora-messaging (>=2.1.0)
+Requires-Dist: fedora-messaging-git-hook-messages ; extra == "schemas"
+Requires-Dist: fedora-messaging-the-new-hotness-schema ; extra == "schemas"
+Requires-Dist: fedora-planet-messages ; extra == "schemas"
+Requires-Dist: fedorainfra-ansible-messages ; extra == "schemas"
+Requires-Dist: fmn-messages ; extra == "schemas"
+Requires-Dist: kerneltest-messages (>=1.0.0,<2.0.0) ; extra == "schemas"
+Requires-Dist: koji-fedoramessaging-messages (>=1.2.2,<2.0.0) ; extra == "schemas"
+Requires-Dist: koschei-messages ; extra == "schemas"
+Requires-Dist: maubot-fedora-messages ; extra == "schemas"
+Requires-Dist: mdapi-messages ; extra == "schemas"
+Requires-Dist: mediawiki-messages ; extra == "schemas"
+Requires-Dist: meetbot-messages ; extra == "schemas"
+Requires-Dist: noggin-messages ; extra == "schemas"
+Requires-Dist: nuancier-messages ; extra == "schemas"
+Requires-Dist: pagure-messages ; extra == "schemas"
+Requires-Dist: psycopg2 (>=2.9.1,<3.0.0)
+Requires-Dist: tahrir-messages ; extra == "schemas"
+Project-URL: Repository, https://github.com/fedora-infra/datanommer
+Description-Content-Type: text/x-rst
+
+datanommer.models
+=================
+
+This package contains the SQLAlchemy data model for datanommer.
+
+Datanommer is a storage consumer for the Fedora Infrastructure Message Bus
+(fedmsg).  It is comprised of a `fedmsg <http://fedmsg.com>`_ consumer that
+stuffs every message into a sqlalchemy database.
+

datanommer_models-1.3.0.dist-info/RECORD

@@ -0,0 +1,10 @@
+datanommer/models/__init__.py,sha256=j3ZIgc2XnXoHaTuC2edq9RoVEqx0g8dc9-3dKi3ctBU,18628
+datanommer/models/alembic/env.py,sha256=WNTimgnH70CakhvuV5QCilCnOcjTy7kcx0nD7hryYx0,2793
+datanommer/models/alembic/script.py.mako,sha256=D8kFI44_9vBJZrAYSkZxDTX2-S5Y-oEetEd9BKlo9S8,412
+datanommer/models/alembic/versions/5db25abc63be_init.py,sha256=xMD7WGCOqeVNFroCZds_aS_jta2yTrAHc_XhtmZLZRs,249
+datanommer/models/alembic/versions/951c40020acc_unique.py,sha256=GwKDhppKW7y5BUV8BqILZCAiR7GsNyIvoTXUu2A1ZMI,843
+datanommer/models/testing/__init__.py,sha256=wwAZ-s1U4M7nYNxHgJsn5ZIqLuHMzHrJwGJOwgHAFgc,1693
+datanommer_models-1.3.0.dist-info/LICENSE,sha256=jOtLnuWt7d5Hsx6XXB2QxzrSe2sWWh3NgMfFRetluQM,35147
+datanommer_models-1.3.0.dist-info/METADATA,sha256=wKhrCYMD6cECmPOoosFOEIZdtAezEhw1SxwDxlotE54,2561
+datanommer_models-1.3.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+datanommer_models-1.3.0.dist-info/RECORD,,

{datanommer.models-1.0.4.dist-info → datanommer_models-1.3.0.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: poetry 1.0.4
+Generator: poetry-core 1.9.0
 Root-Is-Purelib: true
 Tag: py3-none-any

datanommer/models/testing/startup.sql

@@ -1 +0,0 @@
-CREATE EXTENSION IF NOT EXISTS timescaledb;

datanommer.models-1.0.4.dist-info/METADATA

@@ -1,43 +0,0 @@
-Metadata-Version: 2.1
-Name: datanommer.models
-Version: 1.0.4
-Summary: SQLAlchemy models for datanommer
-Home-page: https://github.com/fedora-infra/datanommer
-License: GPL-3.0-or-later
-Author: Fedora Infrastructure
-Author-email: admin@fedoraproject.org
-Requires-Python: >=3.7,<4.0
-Classifier: License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)
-Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.7
-Classifier: Programming Language :: Python :: 3.8
-Classifier: Programming Language :: Python :: 3.9
-Provides-Extra: schemas
-Requires-Dist: SQLAlchemy (>=1.3.24,<2.0.0)
-Requires-Dist: alembic (>=1.6.5,<2.0.0)
-Requires-Dist: anitya-schema; extra == "schemas"
-Requires-Dist: bodhi-messages; extra == "schemas"
-Requires-Dist: copr-messaging; extra == "schemas"
-Requires-Dist: discourse2fedmsg-messages; extra == "schemas"
-Requires-Dist: fedocal-messages; extra == "schemas"
-Requires-Dist: fedora-messaging (>=2.1.0)
-Requires-Dist: fedora-messaging-the-new-hotness-schema; extra == "schemas"
-Requires-Dist: fedora-planet-messages; extra == "schemas"
-Requires-Dist: fedorainfra-ansible-messages; extra == "schemas"
-Requires-Dist: mdapi-messages; extra == "schemas"
-Requires-Dist: noggin-messages; extra == "schemas"
-Requires-Dist: nuancier-messages; extra == "schemas"
-Requires-Dist: pagure-messages; extra == "schemas"
-Requires-Dist: psycopg2 (>=2.9.1,<3.0.0)
-Project-URL: Repository, https://github.com/fedora-infra/datanommer
-Description-Content-Type: text/x-rst
-
-datanommer.models
-=================
-
-This package contains the SQLAlchemy data model for datanommer.
-
-Datanommer is a storage consumer for the Fedora Infrastructure Message Bus
-(fedmsg).  It is comprised of a `fedmsg <http://fedmsg.com>`_ consumer that
-stuffs every message into a sqlalchemy database.
-

datanommer.models-1.0.4.dist-info/RECORD

@@ -1,11 +0,0 @@
-datanommer/models/__init__.py,sha256=4smdWjO0DSxKQeOn0UAfH93eUKQPtTKklqnBdWbGP0Q,17280
-datanommer/models/alembic/env.py,sha256=WNTimgnH70CakhvuV5QCilCnOcjTy7kcx0nD7hryYx0,2793
-datanommer/models/alembic/script.py.mako,sha256=D8kFI44_9vBJZrAYSkZxDTX2-S5Y-oEetEd9BKlo9S8,412
-datanommer/models/alembic/versions/5db25abc63be_init.py,sha256=xMD7WGCOqeVNFroCZds_aS_jta2yTrAHc_XhtmZLZRs,249
-datanommer/models/alembic/versions/951c40020acc_unique.py,sha256=GwKDhppKW7y5BUV8BqILZCAiR7GsNyIvoTXUu2A1ZMI,843
-datanommer/models/testing/__init__.py,sha256=4aNMycVGrYA6Ljo74ePweDrzSr9oaA7K_W2gtn0YZ74,1271
-datanommer/models/testing/startup.sql,sha256=tdjQMZcM7bVDoLnfSg1-fpKKiAPihiVPy-syF0H8mvo,44
-datanommer.models-1.0.4.dist-info/LICENSE,sha256=jOtLnuWt7d5Hsx6XXB2QxzrSe2sWWh3NgMfFRetluQM,35147
-datanommer.models-1.0.4.dist-info/WHEEL,sha256=N0LZrBtofpkS5mJXgVHTCEy52Sam4D6PHQWC8HnMeTs,83
-datanommer.models-1.0.4.dist-info/METADATA,sha256=P6Jrz1-dqJlXlhbsANOKWcXNiVpJI4-W0jOiLB5vW0g,1828
-datanommer.models-1.0.4.dist-info/RECORD,,