morp 7.0.0__py3-none-any.whl

morp/message.py

@@ -0,0 +1,354 @@
+# -*- coding: utf-8 -*-
+from contextlib import asynccontextmanager, AbstractAsyncContextManager
+import logging
+import inspect
+import typing
+
+from datatypes import (
+    ReflectClass,
+    ReflectName,
+    ReflectType,
+    make_dict,
+    classproperty,
+)
+
+from .compat import *
+from .interface import get_interface
+from .exception import ReleaseMessage, AckMessage
+from .config import environ
+
+
+logger = logging.getLogger(__name__)
+
+
+class Message(object):
+    """
+    this is the base class for sending and handling a message
+
+    to add a new message to your application, just subclass this class
+
+    :example:
+        import morp
+
+        class CustomMessage(morp.Message):
+            # message fields
+            foo: int
+            bar: str
+
+            # class fields start with an underscore
+            _ignored: bool = False
+
+            def handle(self):
+                # target will be called when the message is consumed using
+                # the CustomMessage.handle method
+                pass
+
+        m1 = CustomMessage(foo=1, bar="one")
+        await m1.send() # m1 is sent with foo and bar fields
+
+        m2 = await CustomMessage.create(foo=2, bar="two")
+        # m2 was created and sent with foo and bar fields
+
+        await CustomMessage.process(2)
+        # both m1 and m2 were consumed and their .target methods called
+
+    By default, all subclasses will go to the same queue and then when the
+    queue is consumed the correct child class will be created and consume the
+    message with its `.handle` method.
+
+    If you would like your subclass to use a different queue then just set the
+    `._name` property on the class and it will use a different queue
+    """
+    _connection_name: str = ""
+    """the name of the connection to use to retrieve the interface"""
+
+    _name: str = "morp-messages"
+    """The queue name, see .get_name()"""
+
+    _classpath_key: str = "_classpath"
+    """The key that will be used to hold the Message's child class's full
+    classpath, see `._to_interface` and `.from_interface`"""
+
+    _message_classes: dict = {}
+    """Holds all the children message classes. See `__init_subclass__`"""
+
+    @classproperty
+    def interface(cls):
+        return get_interface(cls._connection_name)
+
+    @classproperty
+    def schema(cls):
+        schema = {}
+
+        for field_name, field_type in typing.get_type_hints(cls).items():
+            if field_name.startswith("_"):
+                continue
+
+            schema[field_name] = ReflectType(field_type)
+
+        # cache the value so we don't need to generate it again
+        cls.schema = schema
+        return cls.schema
+
+    @property
+    def fields(self):
+        fields = {}
+
+        for field_name in self.schema.keys():
+            v = getattr(self, field_name, typing.NoReturn)
+            if v is not typing.NoReturn:
+                fields[field_name] = v
+
+        return fields
+
+    def __init__(self, fields=None, **fields_kwargs):
+        fields = make_dict(fields, fields_kwargs)
+        self._from_interface(fields)
+
+    def __init_subclass__(cls):
+        """When a child class is loaded into memory it will be saved into
+        .orm_classes, this way every orm class knows about all the other orm
+        classes, this is the method that makes that possible magically
+
+        https://peps.python.org/pep-0487/
+        """
+        cls._message_classes[f"{cls.__module__}:{cls.__qualname__}"] = cls
+
+    def __contains__(self, field_name):
+        v = getattr(self, field_name, typing.NoReturn)
+        return v is not typing.NoReturn
+        #return hasattr(self, key)
+        #return key in self.fields
+
+    async def send(self, **kwargs):
+        """send the message using the configured interface for this class
+
+        :param **kwargs:
+        :keyword delay_seconds: int, how many seconds before the message can
+            be processed. The max value is interface specific, (eg, it can
+            only be 900s max (15 minutes) per SQS docs)
+        """
+        queue_off = environ.DISABLED
+        name = self.get_name()
+        fields = self._to_interface()
+        if queue_off:
+            logger.warning("DISABLED - Would have sent {} to {}".format(
+                fields,
+                name,
+            ))
+
+        else:
+            logger.info("Sending message with '{}' keys to '{}'".format(
+                "', '".join(fields.keys()),
+                name
+            ))
+
+            hydrate_fields = await self.interface.send(
+                name=name,
+                fields=fields,
+                **kwargs,
+            )
+            # we mimic hydration
+            self._from_interface(hydrate_fields)
+            self._hydrate_fields = hydrate_fields
+
+    @classmethod
+    def get_name(cls):
+        """This is what's used as the official queue name, it takes `.name`
+        and combines it with the `MORP_PREFIX` environment variable
+
+        :returns: str, the queue name
+        """
+        name = cls._name
+        if env_name := environ.PREFIX:
+            name = "{}-{}".format(env_name, name)
+        return name
+
+    @classmethod
+    async def process(cls, count=0, **kwargs):
+        """wait for messages to come in and handle them by calling the incoming
+        message's `handle` method
+
+        :example:
+            # handle 10 messages by receiving them and calling `handle` on the
+            # message instance
+            Message.process(10)
+
+        :param count: int, if you only want to handle N messages, pass in count
+        :param **kwargs: any other params will get passed to underlying 
+            `._recv` methods
+        """
+        max_count = count
+        count = 0
+        while not max_count or count < max_count:
+            count += 1
+            logger.debug("Receiving {}/{}".format(
+                count,
+                max_count if max_count else "Infinity"
+            ))
+
+            async with cls._recv(**kwargs) as m:
+                r = m.handle()
+                while inspect.iscoroutine(r):
+                    r = await r
+
+                if r is False:
+                    raise ReleaseMessage()
+
+    @classmethod
+    @asynccontextmanager
+    async def _recv(cls, **kwargs) -> AbstractAsyncContextManager:
+        """Internal context manager that receives a message to be processed
+
+        Called from `.process` to receive messages and repeatedly calls
+        `._recv_for` to actually receive a message
+
+        :keyword timeout: int, how long to wait before yielding None
+        """
+        m = None
+        # 20 is the max long polling timeout per Amazon
+        kwargs.setdefault('timeout', 20)
+        while not m:
+            async with cls._recv_for(**kwargs) as m:
+                if m:
+                    yield m
+
+    @classmethod
+    @asynccontextmanager
+    async def _recv_for(
+        cls,
+        timeout: int|float,
+        **kwargs,
+    ) -> AbstractAsyncContextManager:
+        """Internal context manager. Try and receive a message, return None
+        if a message is not received within timeout
+
+        :param timeout: float|int, how many seconds before yielding None
+        :returns: generator[Message]
+        """
+        i = cls.interface
+        name = cls.get_name()
+        ack_on_recv = kwargs.pop('ack_on_recv', False)
+        logger.debug(
+            "Waiting to receive on {} for {} seconds".format(name, timeout)
+        )
+
+        async with i.connection(**kwargs) as connection:
+            kwargs["connection"] = connection
+
+            fields = await i.recv(name, timeout=timeout, **kwargs)
+            if fields:
+                try:
+                    yield cls._hydrate(fields)
+
+                except ReleaseMessage as e:
+                    await i.release(
+                        name,
+                        fields,
+                        delay_seconds=e.delay_seconds,
+                        **kwargs
+                    )
+
+                except AckMessage as e:
+                    await i.ack(name, fields, **kwargs)
+
+                except Exception as e:
+                    if ack_on_recv:
+                        await i.ack(name, fields, **kwargs)
+
+                    else:
+                        await i.release(name, fields, **kwargs)
+
+                    raise
+
+                else:
+                    await i.ack(name, fields, **kwargs)
+
+            else:
+                yield None
+
+    @classmethod
+    async def create(cls, *args, **kwargs):
+        """create an instance of cls with the passed in fields and send it off
+
+        Since this passed *args and **kwargs directly to .__init__, you can
+        override the .__init__ method and customize it and this method will
+        inherit the child class's changes. And the signature of this method
+        should always match .__init__
+
+        :param *args: list[Any], passed directly to .__init__
+        :param **kwargs: dict[str, Any], passed directly to .__init__
+        """
+        connection = kwargs.pop("connection", None)
+        instance = cls(*args, **kwargs)
+        await instance.send(connection=connection)
+        return instance
+
+    @classmethod
+    async def count(cls):
+        """how many messages total (approximately) are in the whole message
+        queue"""
+        n = cls.get_name()
+        return await cls.interface.count(n)
+
+    @classmethod
+    def _hydrate(cls, fields):
+        """This is used by the interface to populate an instance with
+        information received from the interface
+
+        :param fields: InterfaceMessage, the message freshly received from
+            the interface, see Interface.create_imessage()
+        """
+        message_class = cls
+        if cls is Message:
+            # When a generic Message instance is used to consume messages it
+            # will use the passed in classpath to create the correct Message
+            # child
+            rn = ReflectName(fields.get(cls._classpath_key))
+            message_class = rn.get_class()
+
+        instance = message_class()
+        instance._from_interface(fields)
+        instance._hydrate_fields = fields
+
+        return instance
+
+    def _to_interface(self):
+        """When sending a message to the interface this method will be called
+
+        :returns: dict, the fields
+        """
+        fields = {}
+        schema = self.schema
+        for field_name, rt in schema.items():
+            fields[field_name] = rt.cast(getattr(self, field_name))
+
+        if self._classpath_key not in fields:
+            fields[self._classpath_key] = ReflectClass(self).classpath
+
+        return fields
+
+    def _from_interface(self, fields):
+        """When receiving a message from the interface this method will
+        be called
+
+        you can see it in action with `._hydrate`
+
+        :param fields: dict, the fields received from the interface
+        """
+        for field_name in self.schema.keys():
+            if field_name in fields:
+                setattr(self, field_name, fields[field_name])
+
+    async def handle(self) -> bool|None:
+        """This method will be called from `.process` and can handle any
+        processing of the message, it should be defined in the child classes
+
+        :returns:
+            - if False, the message will be released back to be processed
+              again
+            - any other value is considered a success and the message is
+              acked/consumed
+        """
+        raise NotImplementedError()
+

morp-7.0.0.dist-info/METADATA

@@ -0,0 +1,212 @@
+Metadata-Version: 2.4
+Name: morp
+Version: 7.0.0
+Summary: Send and receive messages without thinking about it
+Author-email: Jay Marcyes <jay@marcyes.com>
+License-Expression: MIT
+Project-URL: Homepage, http://github.com/Jaymon/morp
+Project-URL: Repository, https://github.com/Jaymon/morp
+Classifier: Development Status :: 4 - Beta
+Classifier: Environment :: Web Environment
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: OS Independent
+Classifier: Topic :: Database
+Classifier: Topic :: Software Development :: Libraries
+Classifier: Topic :: Utilities
+Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE.txt
+Requires-Dist: dsnparse
+Requires-Dist: datatypes
+Provides-Extra: tests
+Requires-Dist: testdata; extra == "tests"
+Provides-Extra: postgres
+Requires-Dist: psycopg; extra == "postgres"
+Provides-Extra: sqs
+Requires-Dist: boto3; extra == "sqs"
+Provides-Extra: encryption
+Requires-Dist: cryptography; extra == "encryption"
+Dynamic: license-file
+
+# Morp
+
+Simple message processing without really thinking about it. Morp can use dropfiles (simple text files), Postgres, and [Amazon SQS](http://aws.amazon.com/sqs/).
+
+
+## Installation
+
+Use pip to install the latest stable version:
+
+    pip install morp
+    
+Morp only supports the dropfiles interface out of the box, you'll need to install certain dependencies depending on what interface you want to use:
+
+    pip install morp[sqs]
+    pip install morp[postgres]
+    
+To install the development version:
+
+    pip install -U "git+https://github.com/Jaymon/morp#egg=morp"
+
+
+## 1 Minute Getting Started
+
+Send and receive a `Foo` message.
+
+First, let's set our environment variable to use dropfiles (local files suitable for development and prototyping) interface:
+
+    export MORP_DSN=dropfile:${TMPDIR}
+
+Then, let's create three files in our working directory:
+
+* `tasks.py` - We'll define our `Message` classes here.
+* `send.py` - We'll send messages from this script.
+* `recv.py` - We'll receive messages from this script.
+
+
+Let's create our `Message` class in `tasks.py`:
+
+```python
+# tasks.py
+from morp import Message
+
+class Foo(Message):
+    some_field: int
+    some_other_field: str
+    
+    def handle(self):
+        # this will be run when a Foo message is consumed
+        print(self.fields)
+```
+
+Now, let's flesh out our `recv.py` file:
+
+```python
+# recv.py
+
+import asyncio
+
+# import our Foo message class from our tasks.py file
+from tasks import Foo
+
+# Foo's `process` method will call `Foo.handle` for each Foo instance received
+asyncio.run(Foo.process())
+```
+
+And start it up:
+
+```
+$ python recv.py
+```
+
+
+Finally, let's send some messages by fleshing out `send.py`:
+
+```python
+# send.py
+
+import asyncio
+
+from tasks import Foo
+
+async def send_messages():
+    # create a message and send it manually
+    f = Foo()
+    f.some_field = 1
+    f.some_other_field = "one"
+    f.ignored_field = True
+    await f.send()
+
+    # quickly send a message
+    await Foo.create(
+        some_field=2,
+        some_other_field="two",
+    )
+    
+asyncio.run(send_messages())
+```
+
+And running it in a separate shell from the shell running our `recv.py` script (it should send two messages):
+
+```
+$ python send.py
+```
+
+That's it! Our running `recv.py` script should've received the messages we sent when we ran our `send.py` script.
+
+
+## DSN
+
+You configure your connection using a dsn in the form:
+
+    InterfaceName://username:password@host:port/path?param1=value1&param2=value2
+
+So, to connect to [Amazon SQS](http://aws.amazon.com/sqs/), you would do:
+
+    sqs://${AWS_ACCESS_KEY_ID}:${AWS_SECRET_ACCESS_KEY}@
+
+You can also override some default values like `region` and `read_lock`:
+
+    sqs://${AWS_ACCESS_KEY_ID}:${AWS_SECRET_ACCESS_KEY}@?region=${AWS_DEFAULT_REGION}&read_lock=120
+
+
+### Serializers
+
+* `pickle` (default)
+* `json`
+
+```
+MORP_DSN="sqs://x:x@?serializer=json"
+```
+
+
+## Encryption
+
+You might need to install some dependencies:
+
+```
+pip install morp[encryption]
+```
+
+If you would like to encrypt all your messages, you can pass in a `key` argument to your DSN and Morp will take care of encrypting and decrypting the messages for you transparently.
+
+Let's just modify our DSN to pass in our key:
+
+    sqs://${AWS_ACCESS_KEY_ID}:${AWS_SECRET_ACCESS_KEY}@?key=jy4XWRuEsrH98RD2VeLG62uVLCPWpdUh
+
+That's it, every message will now be encrypted on send and decrypted on receive. If you're using SQS you can also use [Amazon's key management service](https://github.com/Jaymon/morp/blob/master/docs/KMS.md) to handle the encryption for you.
+
+
+## Environment configuration
+
+### MORP_DISABLED
+
+By default every message will be sent, if you just want to test functionality without actually sending the message you can set this environment variable to turn off all the queues.
+
+    MORP_DISABLED = 1 # queue is off
+    MORP_DISABLED = 0 # queue is on
+
+
+### MORP_PREFIX
+
+If you would like to have your queue names prefixed with something (eg, `prod` or `dev`) then you can set this environment variable and it will be prefixed to the queue name.
+
+
+### MORP_DSN
+
+Set this environment variable with your connection DSN so Morp can automatically configure itself when the interface is first requested.
+
+
+## FAQ
+
+### I would like to have multiple queues
+
+By default, Morp will send any message from any `morp.Message` derived class to `Message.get_name()`, you can override this behavior by giving your child class a `._name` property:
+
+```python
+from morp import Message
+
+class childMsg(Message):
+    _name = "custom-queue-name"
+```

morp-7.0.0.dist-info/RECORD

@@ -0,0 +1,15 @@
+morp/__init__.py,sha256=2V8Qep-iSh73325gx-hqAwRJPh5CcpWb7uPhAB6iBCw,347
+morp/compat.py,sha256=KPZYveHPyXYU4fKeU3fQuvBSFi-mcHK7Uqk9O_kNA-k,97
+morp/config.py,sha256=kifJp-KL_tdDHeu5V17o25f95boQZtEpg3fR9RvOSio,4854
+morp/exception.py,sha256=9CSZWYVNyqt7gxqApG4k3YXzrbmy_EJ_EPqw_SM8Wpk,1022
+morp/message.py,sha256=B2ZLt83lzp-CN3dA0OIxzCegeKgYIwZ3t_FM41hW67w,11406
+morp/interface/__init__.py,sha256=EMrMTF84Rp9V6KlnTaw4uYouLDf6jtb26CwGRX7zvts,2674
+morp/interface/base.py,sha256=fi__yOjKw6J311BPoYAHo9RZ0Yi9MpJ5ZTUg7nZbE1M,14813
+morp/interface/dropfile.py,sha256=oQhUVsTHIJlq_bR53SJ9W-5YZI7JOt_2FgEbx5Uc-AM,5191
+morp/interface/postgres.py,sha256=Ov8NUPEhk63fXa5CZbXueKdmYXoIryfVZwdSQcJRnb8,12415
+morp/interface/sqs.py,sha256=vvUdSVNqSGeM9nrMGUAPX1RVj9kZRMYet0xLht89ws4,14129
+morp-7.0.0.dist-info/licenses/LICENSE.txt,sha256=NdwjhMrenM_sXvd3REQVjswgOw-pOITlnHCNLVp4AwI,1079
+morp-7.0.0.dist-info/METADATA,sha256=gANIuXGUvMNlAKCZTbU6jZKCdsG3HU-Dt78N6sk9HT4,5586
+morp-7.0.0.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+morp-7.0.0.dist-info/top_level.txt,sha256=GjC8bZE2FMSjNYMuYb-lrYy42OkOytWTV2pokuwpfDM,10
+morp-7.0.0.dist-info/RECORD,,

morp-7.0.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.10.2)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

morp-7.0.0.dist-info/licenses/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2013+ Jay Marcyes
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

morp-7.0.0.dist-info/top_level.txt

@@ -0,0 +1,2 @@
+dist
+morp