datanommer.models 1.2.0__py3-none-any.whl → 1.4.0__py3-none-any.whl

datanommer/models/__init__.py

@@ -32,6 +32,7 @@ from sqlalchemy import (
     event,
     ForeignKey,
     func,
+    Index,
     Integer,
     not_,
     or_,
@@ -158,6 +159,7 @@ def add(message):
         timestamp=sent_at,
         msg=message.body,
         headers=headers,
+        agent_name=getattr(message, "agent_name", None),
         users=usernames,
         packages=packages,
     )
@@ -168,7 +170,7 @@ def add(message):
 # https://docs.sqlalchemy.org/en/14/core/custom_types.html#marshal-json-strings
 
 
-class JSONEncodedDict(TypeDecorator):
+class _JSONEncodedDict(TypeDecorator):
     """Represents an immutable structure as a json-encoded string."""
 
     impl = UnicodeText
@@ -213,7 +215,15 @@ packages_assoc_table = Table(
 
 class Message(DeclarativeBase):
     __tablename__ = "messages"
-    __table_args__ = (UniqueConstraint("msg_id", "timestamp"),)
+    __table_args__ = (
+        UniqueConstraint("msg_id", "timestamp"),
+        Index(
+            "ix_messages_headers",
+            "headers",
+            postgresql_using="gin",
+            postgresql_ops={"headers": "jsonb_path_ops"},
+        ),
+    )
 
     id = Column(Integer, primary_key=True, autoincrement=True)
     msg_id = Column(Unicode, nullable=True, default=None, index=True)
@@ -223,11 +233,11 @@ class Message(DeclarativeBase):
     certificate = Column(UnicodeText)
     signature = Column(UnicodeText)
     category = Column(Unicode, nullable=False, index=True)
-    username = Column(Unicode)
+    agent_name = Column(Unicode, index=True)
     crypto = Column(UnicodeText)
     source_name = Column(Unicode, default="datanommer")
     source_version = Column(Unicode, default=lambda context: __version__)
-    msg = Column(JSONEncodedDict, nullable=False)
+    msg = Column(_JSONEncodedDict, nullable=False)
     headers = Column(postgresql.JSONB(none_as_null=True))
     users = relationship(
         "User",
@@ -326,7 +336,8 @@ class Message(DeclarativeBase):
             timestamp=self.timestamp,
             certificate=self.certificate,
             signature=self.signature,
-            username=self.username,
+            agent_name=self.agent_name,
+            username=self.agent_name,  # DEPRECATED
             crypto=self.crypto,
             msg=self.msg,
             headers=self.headers,
@@ -358,14 +369,21 @@ class Message(DeclarativeBase):
         )
         return self.as_dict(request)
 
+    @property
+    def username(self):
+        warn(
+            "The username attribute has been renamed to agent_name, and will be removed "
+            "in the next major version",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        return self.agent_name
+
     @classmethod
-    def grep(
+    def make_query(
         cls,
         start=None,
         end=None,
-        page=1,
-        rows_per_page=100,
-        order="asc",
         msg_id=None,
         users=None,
         not_users=None,
@@ -375,8 +393,9 @@ class Message(DeclarativeBase):
         not_categories=None,
         topics=None,
         not_topics=None,
+        agents=None,
+        not_agents=None,
         contains=None,
-        defer=False,
     ):
         """Flexible query interface for messages.
 
@@ -405,10 +424,6 @@ class Message(DeclarativeBase):
             (user == 'ralph') AND
             NOT (category == 'bodhi' OR category == 'wiki')
 
-        ----
-
-        If the `defer` argument evaluates to True, the query won't actually
-        be executed, but a SQLAlchemy query object returned instead.
         """
 
         users = users or []
@@ -419,6 +434,8 @@ class Message(DeclarativeBase):
         not_cats = not_categories or []
         topics = topics or []
         not_topics = not_topics or []
+        agents = agents or []
+        not_agents = not_agents or []
         contains = contains or []
 
         Message = cls
@@ -450,6 +467,9 @@ class Message(DeclarativeBase):
         if topics:
             query = query.where(or_(*(Message.topic == topic for topic in topics)))
 
+        if agents:
+            query = query.where(or_(*(Message.agent_name == agent for agent in agents)))
+
         if contains:
             query = query.where(or_(*(Message.msg.like(f"%{contain}%") for contain in contains)))
 
@@ -468,23 +488,86 @@ class Message(DeclarativeBase):
         if not_topics:
             query = query.where(not_(or_(*(Message.topic == topic for topic in not_topics))))
 
+        if not_agents:
+            query = query.where(not_(or_(*(Message.agent_name == agent for agent in not_agents))))
+
+        return query
+
+    @classmethod
+    def grep(
+        cls,
+        *,
+        page=1,
+        rows_per_page=100,
+        order="asc",
+        defer=False,
+        **kwargs,
+    ):
+        """Flexible query interface for messages.
+
+        Arguments are filters.  start and end should be :mod:`datetime` objs.
+
+        Other filters should be lists of strings.  They are applied in a
+        conjunctive-normal-form (CNF) kind of way
+
+        for example, the following::
+
+          users = ['ralph', 'lmacken']
+          categories = ['bodhi', 'wiki']
+
+        should return messages where
+
+          (user=='ralph' OR user=='lmacken') AND
+          (category=='bodhi' OR category=='wiki')
+
+        Furthermore, you can use a negative version of each argument.
+
+            users = ['ralph']
+            not_categories = ['bodhi', 'wiki']
+
+        should return messages where
+
+            (user == 'ralph') AND
+            NOT (category == 'bodhi' OR category == 'wiki')
+
+        ----
+
+        If the `defer` argument evaluates to True, the query won't actually
+        be executed, but a SQLAlchemy query object returned instead.
+        """
+        query = cls.make_query(**kwargs)
         # Finally, tag on our pagination arguments
-        total = session.scalar(query.with_only_columns(func.count(Message.id)))
+        Message = cls
+
+        query_total = query.with_only_columns(func.count(Message.id))
+        total = None
         query = query.order_by(getattr(Message.timestamp, order)())
 
         if not rows_per_page:
             pages = 1
         else:
+            total = session.scalar(query_total)
             pages = int(math.ceil(total / float(rows_per_page)))
             query = query.offset(rows_per_page * (page - 1)).limit(rows_per_page)
 
         if defer:
-            return total, page, query
+            if total is None:
+                total = session.scalar(query_total)
+            return total, pages, query
         else:
             # Execute!
             messages = session.scalars(query).all()
+            if pages == 1:
+                total = len(messages)
             return total, pages, messages
 
+    @classmethod
+    def get_first(cls, *, order="asc", **kwargs):
+        """Get the first message matching the regular grep filters."""
+        query = cls.make_query(**kwargs)
+        query = query.order_by(getattr(Message.timestamp, order)()).limit(1)
+        return session.scalars(query).first()
+
 
 class NamedSingleton:
     id = Column(Integer, primary_key=True, autoincrement=True)

datanommer/models/alembic/versions/429e6f2cba6f_message_agent_name.py

@@ -0,0 +1,24 @@
+"""Message.username → Message.agent_name
+
+Revision ID: 429e6f2cba6f
+Revises: 951c40020acc
+Create Date: 2024-06-07 09:12:33.393757
+
+"""
+
+from alembic import op
+
+
+# revision identifiers, used by Alembic.
+revision = "429e6f2cba6f"
+down_revision = "f6918385051f"
+
+
+def upgrade():
+    op.alter_column("messages", "username", new_column_name="agent_name")
+    op.create_index(op.f("ix_messages_agent_name"), "messages", ["agent_name"], unique=False)
+
+
+def downgrade():
+    op.drop_index(op.f("ix_messages_agent_name"), table_name="messages")
+    op.alter_column("messages", "agent_name", new_column_name="username")

datanommer/models/alembic/versions/f6918385051f_messages_headers_index.py

@@ -0,0 +1,29 @@
+"""Messages.headers index
+
+Revision ID: f6918385051f
+Revises: 951c40020acc
+Create Date: 2024-05-07 16:05:05.344863
+
+"""
+
+from alembic import op
+
+
+# revision identifiers, used by Alembic.
+revision = "f6918385051f"
+down_revision = "951c40020acc"
+
+
+def upgrade():
+    op.create_index(
+        "ix_messages_headers",
+        "messages",
+        ["headers"],
+        unique=False,
+        postgresql_using="gin",
+        postgresql_ops={"headers": "jsonb_path_ops"},
+    )
+
+
+def downgrade():
+    op.drop_index("ix_messages_headers", table_name="messages", postgresql_using="gin")

datanommer/models/testing/__init__.py

@@ -21,8 +21,8 @@ def datanommer_db_url(postgresql_proc):
     )
 
 
-@pytest.fixture()
-def datanommer_db(postgresql_proc, datanommer_db_url):
+@pytest.fixture(scope="session")
+def datanommer_db_engine(postgresql_proc, datanommer_db_url):
     with DatabaseJanitor(
         user=postgresql_proc.user,
         host=postgresql_proc.host,
@@ -32,12 +32,20 @@ def datanommer_db(postgresql_proc, datanommer_db_url):
         # template_dbname=postgresql_proc.template_dbname,
         version=postgresql_proc.version,
     ):
-        engine = sa.create_engine(datanommer_db_url, future=True, poolclass=sa.pool.NullPool)
+        engine = sa.create_engine(datanommer_db_url, future=True)
         # Renew the global object, dm.init checks a custom attribute
         dm.session = scoped_session(dm.maker)
         dm.init(engine=engine, create=True)
         yield engine
-        dm.session.close()
+        engine.dispose()
+
+
+@pytest.fixture()
+def datanommer_db(datanommer_db_url, datanommer_db_engine):
+    for table in reversed(dm.DeclarativeBase.metadata.sorted_tables):
+        dm.session.execute(table.delete())
+    dm.session.commit()
+    yield datanommer_db_engine
 
 
 @pytest.fixture()

{datanommer_models-1.2.0.dist-info → datanommer_models-1.4.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datanommer.models
-Version: 1.2.0
+Version: 1.4.0
 Summary: SQLAlchemy models for datanommer
 Home-page: https://github.com/fedora-infra/datanommer
 License: GPL-3.0-or-later
@@ -32,6 +32,7 @@ Requires-Dist: fmn-messages ; extra == "schemas"
 Requires-Dist: kerneltest-messages (>=1.0.0,<2.0.0) ; extra == "schemas"
 Requires-Dist: koji-fedoramessaging-messages (>=1.2.2,<2.0.0) ; extra == "schemas"
 Requires-Dist: koschei-messages ; extra == "schemas"
+Requires-Dist: maubot-fedora-messages ; extra == "schemas"
 Requires-Dist: mdapi-messages ; extra == "schemas"
 Requires-Dist: mediawiki-messages ; extra == "schemas"
 Requires-Dist: meetbot-messages ; extra == "schemas"

datanommer_models-1.4.0.dist-info/RECORD

@@ -0,0 +1,12 @@
+datanommer/models/__init__.py,sha256=5By2KNqRy1d7abDFMT3DvhcQZ4UVrx92XokAYevjKnk,19322
+datanommer/models/alembic/env.py,sha256=WNTimgnH70CakhvuV5QCilCnOcjTy7kcx0nD7hryYx0,2793
+datanommer/models/alembic/script.py.mako,sha256=D8kFI44_9vBJZrAYSkZxDTX2-S5Y-oEetEd9BKlo9S8,412
+datanommer/models/alembic/versions/429e6f2cba6f_message_agent_name.py,sha256=JT_q5oweCN8soqMTul1vEspWvqx5TXOfG3ifB9DYmgs,612
+datanommer/models/alembic/versions/5db25abc63be_init.py,sha256=xMD7WGCOqeVNFroCZds_aS_jta2yTrAHc_XhtmZLZRs,249
+datanommer/models/alembic/versions/951c40020acc_unique.py,sha256=GwKDhppKW7y5BUV8BqILZCAiR7GsNyIvoTXUu2A1ZMI,843
+datanommer/models/alembic/versions/f6918385051f_messages_headers_index.py,sha256=vj-yzMOH3MNdQDufPLAxhix74fV_2tzBbEc6JNWt9Og,575
+datanommer/models/testing/__init__.py,sha256=wwAZ-s1U4M7nYNxHgJsn5ZIqLuHMzHrJwGJOwgHAFgc,1693
+datanommer_models-1.4.0.dist-info/LICENSE,sha256=jOtLnuWt7d5Hsx6XXB2QxzrSe2sWWh3NgMfFRetluQM,35147
+datanommer_models-1.4.0.dist-info/METADATA,sha256=F6nCFWELLinPdPJVmvxwGN4a9wgB7RodZAOlPlA0KLM,2561
+datanommer_models-1.4.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+datanommer_models-1.4.0.dist-info/RECORD,,

datanommer_models-1.2.0.dist-info/RECORD

@@ -1,10 +0,0 @@
-datanommer/models/__init__.py,sha256=ImCJYqPGAPBjx5OoVu6Lbo0mC6Zu9buGfrdgbectsTc,16874
-datanommer/models/alembic/env.py,sha256=WNTimgnH70CakhvuV5QCilCnOcjTy7kcx0nD7hryYx0,2793
-datanommer/models/alembic/script.py.mako,sha256=D8kFI44_9vBJZrAYSkZxDTX2-S5Y-oEetEd9BKlo9S8,412
-datanommer/models/alembic/versions/5db25abc63be_init.py,sha256=xMD7WGCOqeVNFroCZds_aS_jta2yTrAHc_XhtmZLZRs,249
-datanommer/models/alembic/versions/951c40020acc_unique.py,sha256=GwKDhppKW7y5BUV8BqILZCAiR7GsNyIvoTXUu2A1ZMI,843
-datanommer/models/testing/__init__.py,sha256=T_uy2jBp5TG507WAVTx4A96_SsFgy4UrY6WKOuG_l1Q,1453
-datanommer_models-1.2.0.dist-info/LICENSE,sha256=jOtLnuWt7d5Hsx6XXB2QxzrSe2sWWh3NgMfFRetluQM,35147
-datanommer_models-1.2.0.dist-info/METADATA,sha256=KxGFi0ZMyjdK7oslLs8ImIlD9oHogCWF4MXOj2Yyg0k,2502
-datanommer_models-1.2.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-datanommer_models-1.2.0.dist-info/RECORD,,